rpfm_extensions/diagnostics/
table.rs

1//---------------------------------------------------------------------------//
2// Copyright (c) 2017-2026 Ismael Gutiérrez González. All rights reserved.
3//
4// This file is part of the Rusted PackFile Manager (RPFM) project,
5// which can be found here: https://github.com/Frodo45127/rpfm.
6//
7// This file is licensed under the MIT license, which can be found here:
8// https://github.com/Frodo45127/rpfm/blob/master/LICENSE.
9//---------------------------------------------------------------------------//
10
11//! Module with the structs and functions specific for `Table` diagnostics.
12
13use getset::{Getters, MutGetters};
14use itertools::Itertools;
15use serde_derive::{Serialize, Deserialize};
16
17use std::{fmt, fmt::Display};
18
19use rpfm_lib::files::pack::DiagnosticIgnoreEntry;
20use rpfm_lib::files::table::DecodedData;
21use rpfm_lib::schema::{DefinitionPatch, Field};
22
23use crate::diagnostics::*;
24
25/// Multi-column key index used by the DB duplicate-row check.
26///
27/// Maps each key-tuple to the list of `((cell_row, cell_column) list, file_index)` occurrences
28/// so duplicates across multiple files can be flagged.
29type DbKeyIndex<'a> = HashMap<Vec<&'a DecodedData>, Vec<(Vec<(i32, i32)>, usize)>>;
30
31/// Single-column key index used by the Loc duplicate-row check.
32///
33/// Maps each key cell to the list of `((row, column), file_index)` occurrences.
34type LocKeyIndex<'a> = HashMap<&'a DecodedData, Vec<((i32, i32), usize)>>;
35
36//-------------------------------------------------------------------------------//
37//                              Enums & Structs
38//-------------------------------------------------------------------------------//
39
40/// This struct contains the results of a Table diagnostic.
41#[derive(Debug, Clone, Default, Getters, MutGetters, Serialize, Deserialize)]
42#[getset(get = "pub", get_mut = "pub")]
43pub struct TableDiagnostic {
44    path: String,
45    pack: String,
46    results: Vec<TableDiagnosticReport>
47}
48
49/// This struct defines an individual table diagnostic result.
50#[derive(Debug, Clone, Getters, MutGetters, Serialize, Deserialize)]
51#[getset(get = "pub", get_mut = "pub")]
52pub struct TableDiagnosticReport {
53
54    /// List of cells, in "row, column" format.
55    ///
56    /// If the full row or full column are affected, use -1.
57    cells_affected: Vec<(i32, i32)>,
58
59    /// Name of the columns that corresponds to the affected cells.
60    column_names: Vec<String>,
61    report_type: TableDiagnosticReportType,
62}
63
64#[derive(Debug, Clone, Serialize, Deserialize)]
65pub enum TableDiagnosticReportType {
66    OutdatedTable,
67    InvalidReference(String, String),
68    EmptyRow,
69    EmptyKeyField(String),
70    EmptyKeyFields,
71    DuplicatedCombinedKeys(String),
72    NoReferenceTableFound(String),
73    NoReferenceTableNorColumnFoundPak(String),
74    NoReferenceTableNorColumnFoundNoPak(String),
75    InvalidEscape,
76    DuplicatedRow(String),
77    InvalidKey,
78    TableNameEndsInNumber,
79    TableNameHasSpace,
80    TableIsDataCoring,
81    FieldWithPathNotFound(Vec<String>),
82    BannedTable,
83    ValueCannotBeEmpty(String),
84    AlteredTable,
85}
86
87/// Internal struct with cached data of tables.
88///
89/// Used to cache multiple tables and having them being aware of each other on check.
90struct TableInfo<'a> {
91    path: &'a str,
92    pack_key: &'a str,
93    fields_processed: Vec<Field>,
94    patches: Option<&'a DefinitionPatch>,
95    key_amount: usize,
96    table_data: Cow<'a, [Vec<DecodedData>]>,
97    default_row: Vec<DecodedData>,
98    ignored_fields: Vec<String>,
99    ignored_diagnostics: HashSet<String>,
100    ignored_diagnostics_for_fields: HashMap<String, Vec<String>>,
101}
102
103//-------------------------------------------------------------------------------//
104//                             Implementations
105//-------------------------------------------------------------------------------//
106
107impl TableDiagnosticReport {
108    pub fn new(report_type: TableDiagnosticReportType, cells_affected: &[(i32, i32)], fields: &[Field]) -> Self {
109        let mut fields_affected = cells_affected.iter().map(|(_, column)| *column).collect::<Vec<_>>();
110        fields_affected.sort();
111        fields_affected.dedup();
112
113        if fields_affected.contains(&-1) {
114            fields_affected = vec![-1];
115        }
116
117        Self {
118            cells_affected: cells_affected.to_vec(),
119            column_names: fields_affected.iter().flat_map(|index| {
120                if index == &-1 {
121                    fields.iter().map(|field| field.name().to_owned()).collect()
122                } else {
123                    vec![fields[*index as usize].name().to_owned()]
124                }
125            }).collect(),
126            report_type
127        }
128    }
129}
130
131impl DiagnosticReport for TableDiagnosticReport {
132    fn message(&self) -> String {
133        match &self.report_type {
134            TableDiagnosticReportType::OutdatedTable => "Possibly outdated table".to_owned(),
135            TableDiagnosticReportType::InvalidReference(cell_data, field_name) => format!("Invalid reference \"{cell_data}\" in column \"{field_name}\"."),
136            TableDiagnosticReportType::EmptyRow => "Empty row.".to_owned(),
137            TableDiagnosticReportType::EmptyKeyField(field_name) => format!("Empty key for column \"{field_name}\"."),
138            TableDiagnosticReportType::EmptyKeyFields => "Empty key fields.".to_owned(),
139            TableDiagnosticReportType::DuplicatedCombinedKeys(combined_keys) => format!("Duplicated combined keys: {}.", &combined_keys),
140            TableDiagnosticReportType::NoReferenceTableFound(field_name) => format!("No reference table found for column \"{field_name}\"."),
141            TableDiagnosticReportType::NoReferenceTableNorColumnFoundPak(field_name) => format!("No reference column found in referenced table for column \"{field_name}\". Maybe a problem with the schema?"),
142            TableDiagnosticReportType::NoReferenceTableNorColumnFoundNoPak(field_name) => format!("No reference column found in referenced table for column \"{field_name}\". Did you forget to generate the Dependencies Cache, or did you generate it before installing the Assembly kit?"),
143            TableDiagnosticReportType::InvalidEscape => "Invalid line jump/tabulation detected in loc entry. Use \\\\n or \\\\t instead.".to_owned(),
144            TableDiagnosticReportType::DuplicatedRow(combined_keys) => format!("Duplicated row: {combined_keys}."),
145            TableDiagnosticReportType::InvalidKey => "Invalid key.".to_owned(),
146            TableDiagnosticReportType::TableNameEndsInNumber => "Table name ends in number.".to_owned(),
147            TableDiagnosticReportType::TableNameHasSpace => "Table name contains spaces.".to_owned(),
148            TableDiagnosticReportType::TableIsDataCoring => "Table is datacoring.".to_owned(),
149            TableDiagnosticReportType::FieldWithPathNotFound(paths) => format!("Path not found: {}.", paths.iter().join(" || ")),
150            TableDiagnosticReportType::BannedTable => "Banned table.".to_owned(),
151            TableDiagnosticReportType::ValueCannotBeEmpty(field_name) => format!("Empty value for column \"{field_name}\"."),
152            TableDiagnosticReportType::AlteredTable => "Altered Table".to_owned(),
153        }
154    }
155
156    fn level(&self) -> DiagnosticLevel {
157        match self.report_type {
158            TableDiagnosticReportType::OutdatedTable => DiagnosticLevel::Error,
159            TableDiagnosticReportType::InvalidReference(_,_) => DiagnosticLevel::Error,
160            TableDiagnosticReportType::EmptyRow => DiagnosticLevel::Warning,
161            TableDiagnosticReportType::EmptyKeyField(_) => DiagnosticLevel::Error,
162            TableDiagnosticReportType::EmptyKeyFields => DiagnosticLevel::Warning,
163            TableDiagnosticReportType::DuplicatedCombinedKeys(_) => DiagnosticLevel::Warning,
164            TableDiagnosticReportType::NoReferenceTableFound(_) => DiagnosticLevel::Info,
165            TableDiagnosticReportType::NoReferenceTableNorColumnFoundPak(_) => DiagnosticLevel::Info,
166            TableDiagnosticReportType::NoReferenceTableNorColumnFoundNoPak(_) => DiagnosticLevel::Warning,
167            TableDiagnosticReportType::InvalidEscape => DiagnosticLevel::Warning,
168            TableDiagnosticReportType::DuplicatedRow(_) => DiagnosticLevel::Warning,
169            TableDiagnosticReportType::InvalidKey => DiagnosticLevel::Error,
170            TableDiagnosticReportType::TableNameEndsInNumber => DiagnosticLevel::Error,
171            TableDiagnosticReportType::TableNameHasSpace => DiagnosticLevel::Error,
172            TableDiagnosticReportType::TableIsDataCoring => DiagnosticLevel::Warning,
173            TableDiagnosticReportType::FieldWithPathNotFound(_) => DiagnosticLevel::Warning,
174            TableDiagnosticReportType::BannedTable => DiagnosticLevel::Error,
175            TableDiagnosticReportType::ValueCannotBeEmpty(_) => DiagnosticLevel::Error,
176            TableDiagnosticReportType::AlteredTable => DiagnosticLevel::Error,
177        }
178    }
179}
180
181impl Display for TableDiagnosticReportType {
182    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
183        Display::fmt(match self {
184            Self::OutdatedTable => "OutdatedTable",
185            Self::InvalidReference(_,_) => "InvalidReference",
186            Self::EmptyRow => "EmptyRow",
187            Self::EmptyKeyField(_) => "EmptyKeyField",
188            Self::EmptyKeyFields => "EmptyKeyFields",
189            Self::DuplicatedCombinedKeys(_) => "DuplicatedCombinedKeys",
190            Self::NoReferenceTableFound(_) => "NoReferenceTableFound",
191            Self::NoReferenceTableNorColumnFoundPak(_) => "NoReferenceTableNorColumnFoundPak",
192            Self::NoReferenceTableNorColumnFoundNoPak(_) => "NoReferenceTableNorColumnFoundNoPak",
193            Self::InvalidEscape => "InvalidEscape",
194            Self::DuplicatedRow(_) => "DuplicatedRow",
195            Self::InvalidKey => "InvalidKey",
196            Self::TableNameEndsInNumber => "TableNameEndsInNumber",
197            Self::TableNameHasSpace => "TableNameHasSpace",
198            Self::TableIsDataCoring => "TableIsDataCoring",
199            Self::FieldWithPathNotFound(_) => "FieldWithPathNotFound",
200            Self::BannedTable => "BannedTable",
201            Self::ValueCannotBeEmpty(_) => "ValueCannotBeEmpty",
202            Self::AlteredTable => "AlteredTable",
203        }, f)
204    }
205}
206
207
208impl TableDiagnostic {
209    pub fn new(path: &str, pack: &str) -> Self {
210        Self {
211            path: path.to_owned(),
212            pack: pack.to_owned(),
213            results: vec![],
214        }
215    }
216
217    /// This function is used to check if a table is outdated or not.
218    fn is_table_outdated(table_name: &str, table_version: i32, dependencies: &Dependencies) -> bool {
219        if let Ok(vanilla_dbs) = dependencies.db_data(table_name, true, false) {
220            if let Some(max_version) = vanilla_dbs.iter()
221                .filter_map(|x| {
222                    if let Ok(RFileDecoded::DB(table)) = x.decoded() {
223                        Some(table.definition().version())
224                    } else {
225                        None
226                    }
227                }).max_by(|x, y| x.cmp(y)) {
228                if *max_version != table_version {
229                    return true
230                }
231            }
232        }
233
234        false
235    }
236
237    /// This function takes care of checking the db tables of your mod for errors.
238    #[allow(clippy::too_many_arguments)]
239    pub fn check_db(
240        files: &[(&str, &RFile)],
241        dependencies: &Dependencies,
242        global_ignored_diagnostics: &[String],
243        game_info: &GameInfo,
244        local_path_list: &HashMap<String, Vec<String>>,
245        check_ak_only_refs: bool,
246        files_to_ignore: &Option<Vec<DiagnosticIgnoreEntry>>,
247        packs: &BTreeMap<String, Pack>,
248        schema: &Schema,
249        loc_data: &Option<HashMap<Cow<str>, Cow<str>>>,
250        ca_packs: &HashSet<String>,
251    ) -> Vec<DiagnosticType> {
252        let mut diagnostics = vec![];
253
254        if files.is_empty() {
255            return diagnostics;
256        }
257
258        // Get the dependency data for tables once per batch. That way we can speed up this a lot.
259        let file = files.first().and_then(|(_, x)| x.decoded().ok());
260        let dependency_data = if let Some(RFileDecoded::DB(table)) = file {
261            dependencies.db_reference_data(schema, packs, table.table_name(), table.definition(), loc_data)
262        } else {
263            return diagnostics
264        };
265
266        // So, the way we do this semi-optimized, is we do a first loop getting all the cached data we're going to need,
267        // then do the real loop, having the data of all files available for checking diagnostics.
268        let mut table_infos = vec![];
269        for (pack_key, file) in files {
270            let (ignored_fields, ignored_diagnostics, ignored_diagnostics_for_fields) = Diagnostics::ignore_data_for_file(file, files_to_ignore).unwrap_or_default();
271            if let Ok(RFileDecoded::DB(table)) = file.decoded() {
272                let fields_processed = table.definition().fields_processed();
273                let patches = Some(table.definition().patches());
274                let table_data = table.data();
275
276                table_infos.push(TableInfo {
277                    path: file.path_in_container_raw(),
278                    pack_key,
279                    key_amount: fields_processed.iter().filter(|field| field.is_key(patches)).count(),
280                    fields_processed,
281                    patches,
282                    table_data,
283                    default_row: table.new_row(),
284                    ignored_fields,
285                    ignored_diagnostics,
286                    ignored_diagnostics_for_fields
287                });
288            }
289        }
290
291        let mut global_keys: DbKeyIndex = HashMap::with_capacity(table_infos.iter().map(|x| x.table_data.len()).sum());
292        let dec_files = files.iter()
293            .filter_map(|(_, x)| match x.decoded().ok() {
294                Some(RFileDecoded::DB(ref table)) => Some((table, *x)),
295                _ => None,
296            })
297            .collect::<Vec<_>>();
298
299        for (index, (table, file)) in dec_files.iter().enumerate() {
300            let is_twad_key_deletes = table.table_name().starts_with("twad_key_deletes");
301            let check_ak_only = check_ak_only_refs || table.table_name().starts_with("start_pos_");
302            if let Some(table_info) = table_infos.get(index) {
303                let mut diagnostic = TableDiagnostic::new(table_info.path, table_info.pack_key);
304
305                // Before anything else, check if the table is outdated.
306                if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("OutdatedTable"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && Self::is_table_outdated(table.table_name(), *table.definition().version(), dependencies) {
307                    let result = TableDiagnosticReport::new(TableDiagnosticReportType::OutdatedTable, &[], &[]);
308                    diagnostic.results_mut().push(result);
309                }
310
311                // Check if it's one of the banned tables for the game selected.
312                if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("BannedTable"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && game_info.is_file_banned(file.path_in_container_raw()) {
313                    let result = TableDiagnosticReport::new(TableDiagnosticReportType::BannedTable, &[], &[]);
314                    diagnostic.results_mut().push(result);
315                }
316
317                if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("AlteredTable"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && table.altered() {
318                    let result = TableDiagnosticReport::new(TableDiagnosticReportType::AlteredTable, &[], &[]);
319                    diagnostic.results_mut().push(result);
320                }
321
322                // Check if the table name has a number at the end, which causes very annoying bugs.
323                if let Some(name) = file.file_name() {
324                    if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("TableNameEndsInNumber"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && (name.ends_with('0') ||
325                        name.ends_with('1') ||
326                        name.ends_with('2') ||
327                        name.ends_with('3') ||
328                        name.ends_with('4') ||
329                        name.ends_with('5') ||
330                        name.ends_with('6') ||
331                        name.ends_with('7') ||
332                        name.ends_with('8') || name.ends_with('9')) {
333
334                        let result = TableDiagnosticReport::new(TableDiagnosticReportType::TableNameEndsInNumber, &[], &[]);
335                        diagnostic.results_mut().push(result);
336                    }
337
338                    if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("TableNameHasSpace"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && name.contains(' ') {
339                        let result = TableDiagnosticReport::new(TableDiagnosticReportType::TableNameHasSpace, &[], &[]);
340                        diagnostic.results_mut().push(result);
341                    }
342
343                    if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("TableIsDataCoring"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && !ca_packs.contains(table_info.pack_key) {
344                        match game_info.vanilla_db_table_name_logic() {
345                            VanillaDBTableNameLogic::FolderName => {
346                                if table.table_name_without_tables() == file.path_in_container_split()[2] {
347                                    let result = TableDiagnosticReport::new(TableDiagnosticReportType::TableIsDataCoring, &[], &[]);
348                                    diagnostic.results_mut().push(result);
349                                }
350                            }
351
352                            VanillaDBTableNameLogic::DefaultName(ref default_name) => {
353                                if name == default_name {
354                                    let result = TableDiagnosticReport::new(TableDiagnosticReportType::TableIsDataCoring, &[], &[]);
355                                    diagnostic.results_mut().push(result);
356                                }
357                            }
358                        }
359                    }
360                }
361
362                // Columns we can try to check for paths.
363                let mut ignore_path_columns = vec![];
364                for (column, field) in table_info.fields_processed.iter().enumerate() {
365                    if let Some(rel_paths) = field.filename_relative_path(table_info.patches) {
366                        if rel_paths.iter().any(|path| path.contains('*')) {
367                            ignore_path_columns.push(column);
368                        }
369                    }
370                }
371
372                let mut no_ref_table_nor_column_found_marked = HashSet::new();
373                let mut no_ref_table_found_marked = HashSet::new();
374
375                for (row, cells) in table_info.table_data.iter().enumerate() {
376                    let mut row_keys_are_empty = true;
377                    let mut row_keys: BTreeMap<i32, &DecodedData> = BTreeMap::new();
378                    for (column, field) in table_info.fields_processed.iter().enumerate() {
379
380                        // Skip unused field on diagnostics.
381                        //if field.unused(patches) {
382                        //    continue;
383                        //}
384
385                        let cell_data = cells[column].data_to_string();
386
387                        // Path checks.
388                        if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field.name()), Some("FieldWithPathNotFound"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) &&
389                            !cell_data.is_empty() &&
390                            cell_data != "." &&
391                            cell_data != "x" &&
392                            cell_data != "false" &&
393                            cell_data != "building_placeholder" &&
394                            cell_data != "placeholder" &&
395                            cell_data != "PLACEHOLDER" &&
396                            cell_data != "placeholder.png" &&
397                            cell_data != "placehoder.png" &&
398                            table_info.fields_processed[column].is_filename(table_info.patches) &&
399                            !ignore_path_columns.contains(&column) {
400
401                            let mut path_found = false;
402                            let relative_paths = table_info.fields_processed[column].filename_relative_path(table_info.patches);
403                            let paths = if let Some(relative_paths) = relative_paths {
404                                relative_paths.iter()
405                                    .flat_map(|x| {
406                                        let mut paths = vec![];
407                                        let cell_data = cell_data.replace('\\', "/");
408                                        for cell_data in cell_data.split(',') {
409
410                                            // When analysing paths, fix the ones in older games starting with / or data/.
411                                            let mut start_offset = 0;
412                                            if cell_data.starts_with("/") {
413                                                start_offset += 1;
414                                            }
415                                            if cell_data.starts_with("data/") {
416                                                start_offset += 5;
417                                            }
418
419                                            paths.push(x.replace('%', &cell_data[start_offset..]));
420                                        }
421
422                                        paths
423                                    })
424                                    .collect::<Vec<_>>()
425                            } else {
426                                let mut paths = vec![];
427                                let cell_data = cell_data.replace('\\', "/");
428                                for cell_data in cell_data.split(',') {
429
430                                    // When analysing paths, fix the ones in older games starting with / or data/.
431                                    let mut start_offset = 0;
432                                    if cell_data.starts_with("/") {
433                                        start_offset += 1;
434                                    }
435                                    if cell_data.starts_with("data/") {
436                                        start_offset += 5;
437                                    }
438
439                                    paths.push(cell_data[start_offset..].to_string());
440                                }
441
442                                paths
443                            };
444
445                            for path in &paths {
446                                if !path_found && local_path_list.get(&path.to_lowercase()).is_some() {
447                                    path_found = true;
448                                }
449
450                                if !path_found && dependencies.file_exists(path, true, true, true) {
451                                    path_found = true;
452                                }
453
454                                if path_found {
455                                    break;
456                                }
457                            }
458
459                            if !path_found {
460                                let result = TableDiagnosticReport::new(TableDiagnosticReportType::FieldWithPathNotFound(paths), &[(row as i32, column as i32)], &table_info.fields_processed);
461                                diagnostic.results_mut().push(result);
462                            }
463                        }
464
465                        // Dependency checks.
466                        if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field.name()), None, &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) &&
467                            (field.is_reference(table_info.patches).is_some() ||
468                                (
469                                    is_twad_key_deletes &&
470                                    field.name() == "table_name"
471                                )
472                            ) {
473
474                            match dependency_data.get(&(column as i32)) {
475                                Some(ref_data) => {
476                                    if *ref_data.referenced_column_is_localised() {
477                                        // TODO: report missing loc data here.
478                                    }
479                                    /*
480                                    else if ref_data.referenced_table_is_ak_only {
481                                        // If it's only in the AK, ignore it.
482                                    }*/
483
484                                    // Blue cell check. Only one for each column, so we don't fill the diagnostics with this.
485                                    //
486                                    // NOTE: This diagnostics should be one, per column. Once it's done, do not create a new one
487                                    // for the same column in subsequent rows.
488                                    else if ref_data.data().is_empty() && (no_ref_table_nor_column_found_marked.is_empty() || !no_ref_table_nor_column_found_marked.contains(&column)) {
489                                        if !dependencies.is_asskit_data_loaded() {
490                                            if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("NoReferenceTableNorColumnFoundNoPak"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) {
491                                                let field_name = table_info.fields_processed[column].name().to_string();
492                                                let result = TableDiagnosticReport::new(TableDiagnosticReportType::NoReferenceTableNorColumnFoundNoPak(field_name), &[(-1, column as i32)], &table_info.fields_processed);
493                                                diagnostic.results_mut().push(result);
494                                            }
495                                        }
496                                        else if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("NoReferenceTableNorColumnFoundPak"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) {
497                                            let field_name = table_info.fields_processed[column].name().to_string();
498                                            let result = TableDiagnosticReport::new(TableDiagnosticReportType::NoReferenceTableNorColumnFoundPak(field_name), &[(-1, column as i32)], &table_info.fields_processed);
499                                            diagnostic.results_mut().push(result);
500                                        }
501
502                                        no_ref_table_nor_column_found_marked.insert(column);
503                                    }
504
505                                    // Check for non-empty cells with reference data, but the data in the cell is not in the reference data list.
506                                    else if !ref_data.data().is_empty() && !cell_data.is_empty() && !ref_data.data().contains_key(&*cell_data) && (!*ref_data.referenced_table_is_ak_only() || check_ak_only) {
507
508                                        // Numeric cells with 0 are "empty" references and should not be checked.
509                                        let is_number = *field.field_type() == FieldType::I32 || *field.field_type() == FieldType::I64 || *field.field_type() == FieldType::OptionalI32 || *field.field_type() == FieldType::OptionalI64;
510                                        let is_valid_reference = if is_number { cell_data != "0" } else { true };
511                                        if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field.name()), Some("InvalidReference"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && is_valid_reference {
512                                            let result = TableDiagnosticReport::new(TableDiagnosticReportType::InvalidReference(cell_data.to_string(), field.name().to_string()), &[(row as i32, column as i32)], &table_info.fields_processed);
513                                            diagnostic.results_mut().push(result);
514                                        }
515                                    }
516                                }
517                                None => {
518
519                                    // This diagnostic also needs to be done once per column.
520                                    if no_ref_table_found_marked.is_empty() || !no_ref_table_found_marked.contains(&column) {
521                                        if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("NoReferenceTableFound"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) {
522                                            let field_name = table_info.fields_processed[column].name().to_string();
523                                            let result = TableDiagnosticReport::new(TableDiagnosticReportType::NoReferenceTableFound(field_name), &[(-1, column as i32)], &table_info.fields_processed);
524                                            diagnostic.results_mut().push(result);
525                                        }
526                                        no_ref_table_found_marked.insert(column);
527                                    }
528                                }
529                            }
530                        }
531
532                        if row_keys_are_empty && field.is_key(table_info.patches) && (!cell_data.is_empty() && cell_data != "false") {
533                            row_keys_are_empty = false;
534                        }
535
536                        if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field.name()), Some("EmptyKeyField"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && field.is_key(table_info.patches) && table_info.key_amount == 1 && *field.field_type() != FieldType::OptionalStringU8 && *field.field_type() != FieldType::Boolean && (cell_data.is_empty() || cell_data == "false") {
537                            let result = TableDiagnosticReport::new(TableDiagnosticReportType::EmptyKeyField(field.name().to_string()), &[(row as i32, column as i32)], &table_info.fields_processed);
538                            diagnostic.results_mut().push(result);
539                        }
540
541                        if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field.name()), Some("InvalidKey"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && field.is_key(table_info.patches) && !cell_data.is_empty() && (cell_data.ends_with(' ') || cell_data.contains('\n') || cell_data.contains('\r') || cell_data.contains('\t')) {
542                            let result = TableDiagnosticReport::new(TableDiagnosticReportType::InvalidKey, &[(row as i32, column as i32)], &table_info.fields_processed);
543                            diagnostic.results_mut().push(result);
544                        }
545
546                        if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field.name()), Some("ValueCannotBeEmpty"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && cell_data.is_empty() && field.cannot_be_empty(table_info.patches) {
547                            let result = TableDiagnosticReport::new(TableDiagnosticReportType::ValueCannotBeEmpty(field.name().to_string()), &[(row as i32, column as i32)], &table_info.fields_processed);
548                            diagnostic.results_mut().push(result);
549                        }
550
551                        if field.is_key(table_info.patches) {
552                            row_keys.insert(column as i32, &cells[column]);
553                        }
554                    }
555
556                    if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("EmptyRow"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && cells == &table_info.default_row {
557                        let result = TableDiagnosticReport::new(TableDiagnosticReportType::EmptyRow, &[(row as i32, -1)], &table_info.fields_processed);
558                        diagnostic.results_mut().push(result);
559                    }
560
561                    if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("EmptyKeyFields"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && row_keys_are_empty && table_info.key_amount > 1 {
562                        let cells_affected = row_keys.keys().map(|column| (row as i32, *column)).collect::<Vec<(i32, i32)>>();
563                        let result = TableDiagnosticReport::new(TableDiagnosticReportType::EmptyKeyFields, &cells_affected, &table_info.fields_processed);
564                        diagnostic.results_mut().push(result);
565                    }
566
567                    let keys = row_keys.values().copied().collect::<Vec<_>>();
568                    let values = (row_keys.keys().map(|column| (row as i32, *column)).collect::<Vec<(i32, i32)>>(), index);
569                    if !keys.is_empty() {
570                        match global_keys.get_mut(&keys) {
571                            Some(val) => val.push(values),
572                            None => { global_keys.insert(keys, vec![values]); },
573                        }
574                    }
575                }
576
577                if !diagnostic.results().is_empty() {
578                    diagnostics.push(DiagnosticType::DB(diagnostic));
579                }
580            }
581        }
582
583        // This diagnostics needs the row keys data for all tables to be generated. So we have to perform it outside the usual check loop.
584        //
585        // Also, unlike other diagnostics, we don't know what entries of this should be ignored (if any) until we find the duplicates.
586        global_keys.iter()
587            .filter(|(_, val)| val.len() > 1)
588            .for_each(|(key, val)| {
589                for (pos, index) in val {
590                    if let Some(table_info) = table_infos.get(*index) {
591                        if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics,
592                            None,
593                            Some("DuplicatedCombinedKeys"),
594                            &table_info.ignored_fields,
595                            &table_info.ignored_diagnostics,
596                            &table_info.ignored_diagnostics_for_fields
597                        ) {
598
599                            match diagnostics.iter_mut().find(|x| x.path() == table_info.path) {
600                                Some(diag) => if let DiagnosticType::DB(ref mut diag) = diag {
601                                    diag.results_mut().push(
602                                        TableDiagnosticReport::new(
603                                            TableDiagnosticReportType::DuplicatedCombinedKeys(
604                                                key.iter().map(|x| x.data_to_string()).join("| |")
605                                            ),
606                                            pos,
607                                            &table_info.fields_processed
608                                        )
609                                    )
610                                }
611                                None => {
612                                    let mut diag = TableDiagnostic::new(table_info.path, table_info.pack_key);
613                                        diag.results_mut().push(
614                                        TableDiagnosticReport::new(
615                                            TableDiagnosticReportType::DuplicatedCombinedKeys(
616                                                key.iter().map(|x| x.data_to_string()).join("| |")
617                                            ),
618                                            pos,
619                                            &table_info.fields_processed
620                                        )
621                                    );
622
623                                    // Add the new diagnostic and update the cached references so this one is also searched when processing other entries.
624                                    diagnostics.push(DiagnosticType::DB(diag));
625                                }
626                            }
627                        }
628                    }
629                }
630            });
631
632        diagnostics
633    }
634
635    /// This function takes care of checking the loc tables of your mod for errors.
636    pub fn check_loc(
637        files: &[(&str, &RFile)],
638        global_ignored_diagnostics: &[String],
639        files_to_ignore: &Option<Vec<DiagnosticIgnoreEntry>>
640    ) -> Vec<DiagnosticType> {
641        let mut diagnostics = vec![];
642
643        // So, the way we do this semi-optimized, is we do a first loop getting all the cached data we're going to need,
644        // then do the real loop, having the data of all files available for checking diagnostics.
645        let mut table_infos = vec![];
646        for (pack_key, file) in files {
647            let (ignored_fields, ignored_diagnostics, ignored_diagnostics_for_fields) = Diagnostics::ignore_data_for_file(file, files_to_ignore).unwrap_or_default();
648            if let Ok(RFileDecoded::Loc(table)) = file.decoded() {
649                let fields_processed = table.definition().fields_processed();
650                let patches = Some(table.definition().patches());
651                let table_data = table.data();
652
653                table_infos.push(TableInfo {
654                    path: file.path_in_container_raw(),
655                    pack_key,
656                    key_amount: fields_processed.iter().filter(|field| field.is_key(patches)).count(),
657                    fields_processed,
658                    patches,
659                    table_data,
660                    default_row: table.new_row(),
661                    ignored_fields,
662                    ignored_diagnostics,
663                    ignored_diagnostics_for_fields
664                });
665            }
666        }
667
668        let dec_files = files.iter()
669            .filter_map(|(_, x)| match x.decoded().ok() {
670                Some(RFileDecoded::Loc(ref table)) => Some((table, *x)),
671                _ => None,
672            })
673            .collect::<Vec<_>>();
674
675        let mut global_keys: LocKeyIndex = HashMap::with_capacity(table_infos.iter().map(|x| x.table_data.len()).sum());
676
677        for (index, (table, _file)) in dec_files.iter().enumerate() {
678            if let Some(table_info) = table_infos.get(index) {
679                let mut diagnostic = TableDiagnostic::new(table_info.path, table_info.pack_key);
680                let fields = table.definition().fields_processed();
681                let field_key_name = fields[0].name();
682                let field_text_name = fields[1].name();
683
684                for (row, cells) in table_info.table_data.iter().enumerate() {
685                    let key = cells[0].data_to_string();
686                    let data = cells[1].data_to_string();
687                    if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field_key_name), Some("InvalidKey"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && !key.is_empty() && (key.ends_with(' ') || key.contains('\n') || key.contains('\r') || key.contains('\t')) {
688                        let result = TableDiagnosticReport::new(TableDiagnosticReportType::InvalidKey, &[(row as i32, 0)], &fields);
689                        diagnostic.results_mut().push(result);
690                    }
691
692                    // Only in case none of the two columns are ignored, we perform these checks.
693                    if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field_key_name), Some("EmptyRow"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field_text_name), Some("EmptyRow"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && key.is_empty() && data.is_empty() {
694                        let result = TableDiagnosticReport::new(TableDiagnosticReportType::EmptyRow, &[(row as i32, -1)], &fields);
695                        diagnostic.results_mut().push(result);
696                    }
697
698                    if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field_key_name), Some("EmptyKeyField"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && key.is_empty() && !data.is_empty() {
699                        let result = TableDiagnosticReport::new(TableDiagnosticReportType::EmptyKeyField("Key".to_string()), &[(row as i32, 0)], &fields);
700                        diagnostic.results_mut().push(result);
701                    }
702
703                    // Magic Regex. It works. Don't ask why.
704                    if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field_text_name), Some("InvalidEscape"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) &&
705                        !data.is_empty() &&
706                        (
707
708                            // Magic Regex didn't work, so we have to check for lines either with formatted symbols, or with missing slashes.
709                            (data.contains("\r") || (data.match_indices("\\r").count() != data.match_indices("\\\\r").count())) ||
710                            (data.contains("\n") || (data.match_indices("\\n").count() != data.match_indices("\\\\n").count())) ||
711                            (data.contains("\t") || (data.match_indices("\\t").count() != data.match_indices("\\\\t").count()))
712                        ) {
713                        let result = TableDiagnosticReport::new(TableDiagnosticReportType::InvalidEscape, &[(row as i32, 1)], &fields);
714                        diagnostic.results_mut().push(result);
715                    }
716
717                    match global_keys.get_mut(&cells[0]) {
718                        Some(val) => val.push(((row as i32, 0i32), index)),
719                        None => { global_keys.insert(&cells[0], vec![((row as i32, 0i32), index)]); },
720                    }
721                }
722
723
724                if !diagnostic.results().is_empty() {
725                    diagnostics.push(DiagnosticType::Loc(diagnostic));
726                }
727            }
728        }
729
730        // This diagnostics needs the row keys data for all tables to be generated. So we have to perform it outside the usual check loop.
731        //
732        // Also, unlike other diagnostics, we don't know what entries of this should be ignored (if any) until we find the duplicates.
733        global_keys.iter()
734            .filter(|(_, val)| val.len() > 1)
735            .for_each(|(key, val)| {
736                for (pos, index) in val {
737                    if let Some(table_info) = table_infos.get(*index) {
738                        if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics,
739                            None,
740                            Some("DuplicatedCombinedKeys"),
741                            &table_info.ignored_fields,
742                            &table_info.ignored_diagnostics,
743                            &table_info.ignored_diagnostics_for_fields
744                        ) {
745
746                            match diagnostics.iter_mut().find(|x| x.path() == table_info.path) {
747                                Some(diag) => if let DiagnosticType::Loc(ref mut diag) = diag {
748                                    diag.results_mut().push(
749                                        TableDiagnosticReport::new(
750                                            TableDiagnosticReportType::DuplicatedCombinedKeys(
751                                                key.data_to_string().to_string()
752                                            ),
753                                            &[*pos],
754                                            &table_info.fields_processed
755                                        )
756                                    )
757                                }
758                                None => {
759                                    let mut diag = TableDiagnostic::new(table_info.path, table_info.pack_key);
760                                        diag.results_mut().push(
761                                        TableDiagnosticReport::new(
762                                            TableDiagnosticReportType::DuplicatedCombinedKeys(
763                                                key.data_to_string().to_string()
764                                            ),
765                                            &[*pos],
766                                            &table_info.fields_processed
767                                        )
768                                    );
769
770                                    // Add the new diagnostic and update the cached references so this one is also searched when processing other entries.
771                                    diagnostics.push(DiagnosticType::Loc(diag));
772                                }
773                            }
774                        }
775
776                    }
777                }
778
779                let mut values = Vec::with_capacity(val.len());
780                for (pos, index) in val {
781                    if let Some(table_info) = table_infos.get(*index) {
782                        values.push((&table_info.table_data[pos.0 as usize][1], pos.0, index));
783                    }
784                }
785
786                values.sort_unstable_by_key(|x| x.0.data_to_string());
787                let dups = values.iter().duplicates_by(|x| x.0);
788                let poss = values.iter().positions(|x| dups.clone().any(|y| y.0 == x.0));
789
790                for pos in poss {
791                    if let Some((data, row, index)) = values.get(pos) {
792                        if let Some(table_info) = table_infos.get(**index) {
793                            if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics,
794                                None,
795                                Some("DuplicatedRow"),
796                                &table_info.ignored_fields,
797                                &table_info.ignored_diagnostics,
798                                &table_info.ignored_diagnostics_for_fields
799                            ) {
800
801                                match diagnostics.iter_mut().find(|x| x.path() == table_info.path) {
802                                    Some(diag) => if let DiagnosticType::Loc(ref mut diag) = diag {
803                                        diag.results_mut().push(
804                                            TableDiagnosticReport::new(
805                                                TableDiagnosticReportType::DuplicatedRow(
806                                                    String::from(table_info.table_data[*row as usize][0].data_to_string()) + "| |" + &data.data_to_string()
807                                                ),
808                                                &[(*row, 0), (*row, 1)],
809                                                &table_info.fields_processed
810                                            )
811                                        )
812                                    }
813                                    None => {
814                                        let mut diag = TableDiagnostic::new(table_info.path, table_info.pack_key);
815                                            diag.results_mut().push(
816                                            TableDiagnosticReport::new(
817                                                TableDiagnosticReportType::DuplicatedRow(
818                                                    String::from(table_info.table_data[*row as usize][0].data_to_string()) + "| |" + &data.data_to_string()
819                                                ),
820                                                &[(*row, 0), (*row, 1)],
821                                                &table_info.fields_processed
822                                            )
823                                        );
824
825                                        // Add the new diagnostic and update the cached references so this one is also searched when processing other entries.
826                                        diagnostics.push(DiagnosticType::Loc(diag));
827                                    }
828                                }
829                            }
830                        }
831                    }
832                }
833            });
834
835        diagnostics
836    }
837}
rpfm_extensions/diagnostics/table.rs

rpfm_extensions/diagnostics/
table.rs
