rpfm_extensions/diagnostics/
table.rs

1//---------------------------------------------------------------------------//
2// Copyright (c) 2017-2026 Ismael Gutiérrez González. All rights reserved.
3//
4// This file is part of the Rusted PackFile Manager (RPFM) project,
5// which can be found here: https://github.com/Frodo45127/rpfm.
6//
7// This file is licensed under the MIT license, which can be found here:
8// https://github.com/Frodo45127/rpfm/blob/master/LICENSE.
9//---------------------------------------------------------------------------//
10
11//! Module with the structs and functions specific for `Table` diagnostics.
12
13use getset::{Getters, MutGetters};
14use itertools::Itertools;
15use serde_derive::{Serialize, Deserialize};
16
17use std::{fmt, fmt::Display};
18
19use rpfm_lib::files::pack::DiagnosticIgnoreEntry;
20use rpfm_lib::files::table::DecodedData;
21use rpfm_lib::schema::{DefinitionPatch, Field};
22
23use crate::diagnostics::*;
24
25/// Multi-column key index used by the DB duplicate-row check.
26///
27/// Maps each key-tuple to the list of `((cell_row, cell_column) list, file_index)` occurrences
28/// so duplicates across multiple files can be flagged.
29type DbKeyIndex<'a> = HashMap<Vec<&'a DecodedData>, Vec<(Vec<(i32, i32)>, usize)>>;
30
31/// Single-column key index used by the Loc duplicate-row check.
32///
33/// Maps each key cell to the list of `((row, column), file_index)` occurrences.
34type LocKeyIndex<'a> = HashMap<&'a DecodedData, Vec<((i32, i32), usize)>>;
35
36//-------------------------------------------------------------------------------//
37//                              Enums & Structs
38//-------------------------------------------------------------------------------//
39
40/// This struct contains the results of a Table diagnostic.
41#[derive(Debug, Clone, Default, Getters, MutGetters, Serialize, Deserialize)]
42#[getset(get = "pub", get_mut = "pub")]
43pub struct TableDiagnostic {
44    path: String,
45    pack: String,
46    results: Vec<TableDiagnosticReport>
47}
48
49/// This struct defines an individual table diagnostic result.
50#[derive(Debug, Clone, Getters, MutGetters, Serialize, Deserialize)]
51#[getset(get = "pub", get_mut = "pub")]
52pub struct TableDiagnosticReport {
53
54    /// List of cells, in "row, column" format.
55    ///
56    /// If the full row or full column are affected, use -1.
57    cells_affected: Vec<(i32, i32)>,
58
59    /// Name of the columns that corresponds to the affected cells.
60    column_names: Vec<String>,
61    report_type: TableDiagnosticReportType,
62}
63
64#[derive(Debug, Clone, Serialize, Deserialize)]
65pub enum TableDiagnosticReportType {
66    OutdatedTable,
67    InvalidReference(String, String),
68    EmptyRow,
69    EmptyKeyField(String),
70    EmptyKeyFields,
71    DuplicatedCombinedKeys(String),
72    NoReferenceTableFound(String),
73    NoReferenceTableNorColumnFoundPak(String),
74    NoReferenceTableNorColumnFoundNoPak(String),
75    InvalidEscape,
76    DuplicatedRow(String),
77    InvalidLocKey,
78    TableNameEndsInNumber,
79    TableNameHasSpace,
80    TableIsDataCoring,
81    FieldWithPathNotFound(Vec<String>),
82    BannedTable,
83    ValueCannotBeEmpty(String),
84    AlteredTable,
85}
86
87/// Internal struct with cached data of tables.
88///
89/// Used to cache multiple tables and having them being aware of each other on check.
90struct TableInfo<'a> {
91    path: &'a str,
92    pack_key: &'a str,
93    fields_processed: Vec<Field>,
94    patches: Option<&'a DefinitionPatch>,
95    key_amount: usize,
96    table_data: Cow<'a, [Vec<DecodedData>]>,
97    default_row: Vec<DecodedData>,
98    ignored_fields: Vec<String>,
99    ignored_diagnostics: HashSet<String>,
100    ignored_diagnostics_for_fields: HashMap<String, Vec<String>>,
101}
102
103//-------------------------------------------------------------------------------//
104//                             Implementations
105//-------------------------------------------------------------------------------//
106
107impl TableDiagnosticReport {
108    pub fn new(report_type: TableDiagnosticReportType, cells_affected: &[(i32, i32)], fields: &[Field]) -> Self {
109        let mut fields_affected = cells_affected.iter().map(|(_, column)| *column).collect::<Vec<_>>();
110        fields_affected.sort();
111        fields_affected.dedup();
112
113        if fields_affected.contains(&-1) {
114            fields_affected = vec![-1];
115        }
116
117        Self {
118            cells_affected: cells_affected.to_vec(),
119            column_names: fields_affected.iter().flat_map(|index| {
120                if index == &-1 {
121                    fields.iter().map(|field| field.name().to_owned()).collect()
122                } else {
123                    vec![fields[*index as usize].name().to_owned()]
124                }
125            }).collect(),
126            report_type
127        }
128    }
129}
130
131impl DiagnosticReport for TableDiagnosticReport {
132    fn message(&self) -> String {
133        match &self.report_type {
134            TableDiagnosticReportType::OutdatedTable => "Possibly outdated table".to_owned(),
135            TableDiagnosticReportType::InvalidReference(cell_data, field_name) => format!("Invalid reference \"{cell_data}\" in column \"{field_name}\"."),
136            TableDiagnosticReportType::EmptyRow => "Empty row.".to_owned(),
137            TableDiagnosticReportType::EmptyKeyField(field_name) => format!("Empty key for column \"{field_name}\"."),
138            TableDiagnosticReportType::EmptyKeyFields => "Empty key fields.".to_owned(),
139            TableDiagnosticReportType::DuplicatedCombinedKeys(combined_keys) => format!("Duplicated combined keys: {}.", &combined_keys),
140            TableDiagnosticReportType::NoReferenceTableFound(field_name) => format!("No reference table found for column \"{field_name}\"."),
141            TableDiagnosticReportType::NoReferenceTableNorColumnFoundPak(field_name) => format!("No reference column found in referenced table for column \"{field_name}\". Maybe a problem with the schema?"),
142            TableDiagnosticReportType::NoReferenceTableNorColumnFoundNoPak(field_name) => format!("No reference column found in referenced table for column \"{field_name}\". Did you forget to generate the Dependencies Cache, or did you generate it before installing the Assembly kit?"),
143            TableDiagnosticReportType::InvalidEscape => "Invalid line jump/tabulation detected in loc entry. Use \\\\n or \\\\t instead.".to_owned(),
144            TableDiagnosticReportType::DuplicatedRow(combined_keys) => format!("Duplicated row: {combined_keys}."),
145            TableDiagnosticReportType::InvalidLocKey => "Invalid localisation key.".to_owned(),
146            TableDiagnosticReportType::TableNameEndsInNumber => "Table name ends in number.".to_owned(),
147            TableDiagnosticReportType::TableNameHasSpace => "Table name contains spaces.".to_owned(),
148            TableDiagnosticReportType::TableIsDataCoring => "Table is datacoring.".to_owned(),
149            TableDiagnosticReportType::FieldWithPathNotFound(paths) => format!("Path not found: {}.", paths.iter().join(" || ")),
150            TableDiagnosticReportType::BannedTable => "Banned table.".to_owned(),
151            TableDiagnosticReportType::ValueCannotBeEmpty(field_name) => format!("Empty value for column \"{field_name}\"."),
152            TableDiagnosticReportType::AlteredTable => "Altered Table".to_owned(),
153        }
154    }
155
156    fn level(&self) -> DiagnosticLevel {
157        match self.report_type {
158            TableDiagnosticReportType::OutdatedTable => DiagnosticLevel::Error,
159            TableDiagnosticReportType::InvalidReference(_,_) => DiagnosticLevel::Error,
160            TableDiagnosticReportType::EmptyRow => DiagnosticLevel::Warning,
161            TableDiagnosticReportType::EmptyKeyField(_) => DiagnosticLevel::Error,
162            TableDiagnosticReportType::EmptyKeyFields => DiagnosticLevel::Warning,
163            TableDiagnosticReportType::DuplicatedCombinedKeys(_) => DiagnosticLevel::Warning,
164            TableDiagnosticReportType::NoReferenceTableFound(_) => DiagnosticLevel::Info,
165            TableDiagnosticReportType::NoReferenceTableNorColumnFoundPak(_) => DiagnosticLevel::Info,
166            TableDiagnosticReportType::NoReferenceTableNorColumnFoundNoPak(_) => DiagnosticLevel::Warning,
167            TableDiagnosticReportType::InvalidEscape => DiagnosticLevel::Warning,
168            TableDiagnosticReportType::DuplicatedRow(_) => DiagnosticLevel::Warning,
169            TableDiagnosticReportType::InvalidLocKey => DiagnosticLevel::Error,
170            TableDiagnosticReportType::TableNameEndsInNumber => DiagnosticLevel::Error,
171            TableDiagnosticReportType::TableNameHasSpace => DiagnosticLevel::Error,
172            TableDiagnosticReportType::TableIsDataCoring => DiagnosticLevel::Warning,
173            TableDiagnosticReportType::FieldWithPathNotFound(_) => DiagnosticLevel::Warning,
174            TableDiagnosticReportType::BannedTable => DiagnosticLevel::Error,
175            TableDiagnosticReportType::ValueCannotBeEmpty(_) => DiagnosticLevel::Error,
176            TableDiagnosticReportType::AlteredTable => DiagnosticLevel::Error,
177        }
178    }
179}
180
181impl Display for TableDiagnosticReportType {
182    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
183        Display::fmt(match self {
184            Self::OutdatedTable => "OutdatedTable",
185            Self::InvalidReference(_,_) => "InvalidReference",
186            Self::EmptyRow => "EmptyRow",
187            Self::EmptyKeyField(_) => "EmptyKeyField",
188            Self::EmptyKeyFields => "EmptyKeyFields",
189            Self::DuplicatedCombinedKeys(_) => "DuplicatedCombinedKeys",
190            Self::NoReferenceTableFound(_) => "NoReferenceTableFound",
191            Self::NoReferenceTableNorColumnFoundPak(_) => "NoReferenceTableNorColumnFoundPak",
192            Self::NoReferenceTableNorColumnFoundNoPak(_) => "NoReferenceTableNorColumnFoundNoPak",
193            Self::InvalidEscape => "InvalidEscape",
194            Self::DuplicatedRow(_) => "DuplicatedRow",
195            Self::InvalidLocKey => "InvalidLocKey",
196            Self::TableNameEndsInNumber => "TableNameEndsInNumber",
197            Self::TableNameHasSpace => "TableNameHasSpace",
198            Self::TableIsDataCoring => "TableIsDataCoring",
199            Self::FieldWithPathNotFound(_) => "FieldWithPathNotFound",
200            Self::BannedTable => "BannedTable",
201            Self::ValueCannotBeEmpty(_) => "ValueCannotBeEmpty",
202            Self::AlteredTable => "AlteredTable",
203        }, f)
204    }
205}
206
207
208impl TableDiagnostic {
209    pub fn new(path: &str, pack: &str) -> Self {
210        Self {
211            path: path.to_owned(),
212            pack: pack.to_owned(),
213            results: vec![],
214        }
215    }
216
217    /// This function is used to check if a table is outdated or not.
218    fn is_table_outdated(table_name: &str, table_version: i32, dependencies: &Dependencies) -> bool {
219        if let Ok(vanilla_dbs) = dependencies.db_data(table_name, true, false) {
220            if let Some(max_version) = vanilla_dbs.iter()
221                .filter_map(|x| {
222                    if let Ok(RFileDecoded::DB(table)) = x.decoded() {
223                        Some(table.definition().version())
224                    } else {
225                        None
226                    }
227                }).max_by(|x, y| x.cmp(y)) {
228                if *max_version != table_version {
229                    return true
230                }
231            }
232        }
233
234        false
235    }
236
237    /// This function takes care of checking the db tables of your mod for errors.
238    #[allow(clippy::too_many_arguments)]
239    pub fn check_db(
240        files: &[(&str, &RFile)],
241        dependencies: &Dependencies,
242        global_ignored_diagnostics: &[String],
243        game_info: &GameInfo,
244        local_path_list: &HashMap<String, Vec<String>>,
245        check_ak_only_refs: bool,
246        files_to_ignore: &Option<Vec<DiagnosticIgnoreEntry>>,
247        packs: &BTreeMap<String, Pack>,
248        schema: &Schema,
249        loc_data: &Option<HashMap<Cow<str>, Cow<str>>>
250    ) -> Vec<DiagnosticType> {
251        let mut diagnostics = vec![];
252
253        if files.is_empty() {
254            return diagnostics;
255        }
256
257        // Get the dependency data for tables once per batch. That way we can speed up this a lot.
258        let file = files.first().and_then(|(_, x)| x.decoded().ok());
259        let dependency_data = if let Some(RFileDecoded::DB(table)) = file {
260            dependencies.db_reference_data(schema, packs, table.table_name(), table.definition(), loc_data)
261        } else {
262            return diagnostics
263        };
264
265        // So, the way we do this semi-optimized, is we do a first loop getting all the cached data we're going to need,
266        // then do the real loop, having the data of all files available for checking diagnostics.
267        let mut table_infos = vec![];
268        for (pack_key, file) in files {
269            let (ignored_fields, ignored_diagnostics, ignored_diagnostics_for_fields) = Diagnostics::ignore_data_for_file(file, files_to_ignore).unwrap_or_default();
270            if let Ok(RFileDecoded::DB(table)) = file.decoded() {
271                let fields_processed = table.definition().fields_processed();
272                let patches = Some(table.definition().patches());
273                let table_data = table.data();
274
275                table_infos.push(TableInfo {
276                    path: file.path_in_container_raw(),
277                    pack_key,
278                    key_amount: fields_processed.iter().filter(|field| field.is_key(patches)).count(),
279                    fields_processed,
280                    patches,
281                    table_data,
282                    default_row: table.new_row(),
283                    ignored_fields,
284                    ignored_diagnostics,
285                    ignored_diagnostics_for_fields
286                });
287            }
288        }
289
290        let mut global_keys: DbKeyIndex = HashMap::with_capacity(table_infos.iter().map(|x| x.table_data.len()).sum());
291        let dec_files = files.iter()
292            .filter_map(|(_, x)| match x.decoded().ok() {
293                Some(RFileDecoded::DB(ref table)) => Some((table, *x)),
294                _ => None,
295            })
296            .collect::<Vec<_>>();
297
298        for (index, (table, file)) in dec_files.iter().enumerate() {
299            let is_twad_key_deletes = table.table_name().starts_with("twad_key_deletes");
300            let check_ak_only = check_ak_only_refs || table.table_name().starts_with("start_pos_");
301            if let Some(table_info) = table_infos.get(index) {
302                let mut diagnostic = TableDiagnostic::new(table_info.path, table_info.pack_key);
303
304                // Before anything else, check if the table is outdated.
305                if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("OutdatedTable"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && Self::is_table_outdated(table.table_name(), *table.definition().version(), dependencies) {
306                    let result = TableDiagnosticReport::new(TableDiagnosticReportType::OutdatedTable, &[], &[]);
307                    diagnostic.results_mut().push(result);
308                }
309
310                // Check if it's one of the banned tables for the game selected.
311                if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("BannedTable"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && game_info.is_file_banned(file.path_in_container_raw()) {
312                    let result = TableDiagnosticReport::new(TableDiagnosticReportType::BannedTable, &[], &[]);
313                    diagnostic.results_mut().push(result);
314                }
315
316                if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("AlteredTable"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && table.altered() {
317                    let result = TableDiagnosticReport::new(TableDiagnosticReportType::AlteredTable, &[], &[]);
318                    diagnostic.results_mut().push(result);
319                }
320
321                // Check if the table name has a number at the end, which causes very annoying bugs.
322                if let Some(name) = file.file_name() {
323                    if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("TableNameEndsInNumber"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && (name.ends_with('0') ||
324                        name.ends_with('1') ||
325                        name.ends_with('2') ||
326                        name.ends_with('3') ||
327                        name.ends_with('4') ||
328                        name.ends_with('5') ||
329                        name.ends_with('6') ||
330                        name.ends_with('7') ||
331                        name.ends_with('8') || name.ends_with('9')) {
332
333                        let result = TableDiagnosticReport::new(TableDiagnosticReportType::TableNameEndsInNumber, &[], &[]);
334                        diagnostic.results_mut().push(result);
335                    }
336
337                    if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("TableNameHasSpace"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && name.contains(' ') {
338                        let result = TableDiagnosticReport::new(TableDiagnosticReportType::TableNameHasSpace, &[], &[]);
339                        diagnostic.results_mut().push(result);
340                    }
341
342                    if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("TableIsDataCoring"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) {
343                        match game_info.vanilla_db_table_name_logic() {
344                            VanillaDBTableNameLogic::FolderName => {
345                                if table.table_name_without_tables() == file.path_in_container_split()[2] {
346                                    let result = TableDiagnosticReport::new(TableDiagnosticReportType::TableIsDataCoring, &[], &[]);
347                                    diagnostic.results_mut().push(result);
348                                }
349                            }
350
351                            VanillaDBTableNameLogic::DefaultName(ref default_name) => {
352                                if name == default_name {
353                                    let result = TableDiagnosticReport::new(TableDiagnosticReportType::TableIsDataCoring, &[], &[]);
354                                    diagnostic.results_mut().push(result);
355                                }
356                            }
357                        }
358                    }
359                }
360
361                // Columns we can try to check for paths.
362                let mut ignore_path_columns = vec![];
363                for (column, field) in table_info.fields_processed.iter().enumerate() {
364                    if let Some(rel_paths) = field.filename_relative_path(table_info.patches) {
365                        if rel_paths.iter().any(|path| path.contains('*')) {
366                            ignore_path_columns.push(column);
367                        }
368                    }
369                }
370
371                let mut no_ref_table_nor_column_found_marked = HashSet::new();
372                let mut no_ref_table_found_marked = HashSet::new();
373
374                for (row, cells) in table_info.table_data.iter().enumerate() {
375                    let mut row_keys_are_empty = true;
376                    let mut row_keys: BTreeMap<i32, &DecodedData> = BTreeMap::new();
377                    for (column, field) in table_info.fields_processed.iter().enumerate() {
378
379                        // Skip unused field on diagnostics.
380                        //if field.unused(patches) {
381                        //    continue;
382                        //}
383
384                        let cell_data = cells[column].data_to_string();
385
386                        // Path checks.
387                        if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field.name()), Some("FieldWithPathNotFound"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) &&
388                            !cell_data.is_empty() &&
389                            cell_data != "." &&
390                            cell_data != "x" &&
391                            cell_data != "false" &&
392                            cell_data != "building_placeholder" &&
393                            cell_data != "placeholder" &&
394                            cell_data != "PLACEHOLDER" &&
395                            cell_data != "placeholder.png" &&
396                            cell_data != "placehoder.png" &&
397                            table_info.fields_processed[column].is_filename(table_info.patches) &&
398                            !ignore_path_columns.contains(&column) {
399
400                            let mut path_found = false;
401                            let relative_paths = table_info.fields_processed[column].filename_relative_path(table_info.patches);
402                            let paths = if let Some(relative_paths) = relative_paths {
403                                relative_paths.iter()
404                                    .flat_map(|x| {
405                                        let mut paths = vec![];
406                                        let cell_data = cell_data.replace('\\', "/");
407                                        for cell_data in cell_data.split(',') {
408
409                                            // When analysing paths, fix the ones in older games starting with / or data/.
410                                            let mut start_offset = 0;
411                                            if cell_data.starts_with("/") {
412                                                start_offset += 1;
413                                            }
414                                            if cell_data.starts_with("data/") {
415                                                start_offset += 5;
416                                            }
417
418                                            paths.push(x.replace('%', &cell_data[start_offset..]));
419                                        }
420
421                                        paths
422                                    })
423                                    .collect::<Vec<_>>()
424                            } else {
425                                let mut paths = vec![];
426                                let cell_data = cell_data.replace('\\', "/");
427                                for cell_data in cell_data.split(',') {
428
429                                    // When analysing paths, fix the ones in older games starting with / or data/.
430                                    let mut start_offset = 0;
431                                    if cell_data.starts_with("/") {
432                                        start_offset += 1;
433                                    }
434                                    if cell_data.starts_with("data/") {
435                                        start_offset += 5;
436                                    }
437
438                                    paths.push(cell_data[start_offset..].to_string());
439                                }
440
441                                paths
442                            };
443
444                            for path in &paths {
445                                if !path_found && local_path_list.get(&path.to_lowercase()).is_some() {
446                                    path_found = true;
447                                }
448
449                                if !path_found && dependencies.file_exists(path, true, true, true) {
450                                    path_found = true;
451                                }
452
453                                if path_found {
454                                    break;
455                                }
456                            }
457
458                            if !path_found {
459                                let result = TableDiagnosticReport::new(TableDiagnosticReportType::FieldWithPathNotFound(paths), &[(row as i32, column as i32)], &table_info.fields_processed);
460                                diagnostic.results_mut().push(result);
461                            }
462                        }
463
464                        // Dependency checks.
465                        if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field.name()), None, &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) &&
466                            (field.is_reference(table_info.patches).is_some() ||
467                                (
468                                    is_twad_key_deletes &&
469                                    field.name() == "table_name"
470                                )
471                            ) {
472
473                            match dependency_data.get(&(column as i32)) {
474                                Some(ref_data) => {
475                                    if *ref_data.referenced_column_is_localised() {
476                                        // TODO: report missing loc data here.
477                                    }
478                                    /*
479                                    else if ref_data.referenced_table_is_ak_only {
480                                        // If it's only in the AK, ignore it.
481                                    }*/
482
483                                    // Blue cell check. Only one for each column, so we don't fill the diagnostics with this.
484                                    //
485                                    // NOTE: This diagnostics should be one, per column. Once it's done, do not create a new one
486                                    // for the same column in subsequent rows.
487                                    else if ref_data.data().is_empty() && (no_ref_table_nor_column_found_marked.is_empty() || !no_ref_table_nor_column_found_marked.contains(&column)) {
488                                        if !dependencies.is_asskit_data_loaded() {
489                                            if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("NoReferenceTableNorColumnFoundNoPak"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) {
490                                                let field_name = table_info.fields_processed[column].name().to_string();
491                                                let result = TableDiagnosticReport::new(TableDiagnosticReportType::NoReferenceTableNorColumnFoundNoPak(field_name), &[(-1, column as i32)], &table_info.fields_processed);
492                                                diagnostic.results_mut().push(result);
493                                            }
494                                        }
495                                        else if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("NoReferenceTableNorColumnFoundPak"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) {
496                                            let field_name = table_info.fields_processed[column].name().to_string();
497                                            let result = TableDiagnosticReport::new(TableDiagnosticReportType::NoReferenceTableNorColumnFoundPak(field_name), &[(-1, column as i32)], &table_info.fields_processed);
498                                            diagnostic.results_mut().push(result);
499                                        }
500
501                                        no_ref_table_nor_column_found_marked.insert(column);
502                                    }
503
504                                    // Check for non-empty cells with reference data, but the data in the cell is not in the reference data list.
505                                    else if !ref_data.data().is_empty() && !cell_data.is_empty() && !ref_data.data().contains_key(&*cell_data) && (!*ref_data.referenced_table_is_ak_only() || check_ak_only) {
506
507                                        // Numeric cells with 0 are "empty" references and should not be checked.
508                                        let is_number = *field.field_type() == FieldType::I32 || *field.field_type() == FieldType::I64 || *field.field_type() == FieldType::OptionalI32 || *field.field_type() == FieldType::OptionalI64;
509                                        let is_valid_reference = if is_number { cell_data != "0" } else { true };
510                                        if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field.name()), Some("InvalidReference"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && is_valid_reference {
511                                            let result = TableDiagnosticReport::new(TableDiagnosticReportType::InvalidReference(cell_data.to_string(), field.name().to_string()), &[(row as i32, column as i32)], &table_info.fields_processed);
512                                            diagnostic.results_mut().push(result);
513                                        }
514                                    }
515                                }
516                                None => {
517
518                                    // This diagnostic also needs to be done once per column.
519                                    if no_ref_table_found_marked.is_empty() || !no_ref_table_found_marked.contains(&column) {
520                                        if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("NoReferenceTableFound"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) {
521                                            let field_name = table_info.fields_processed[column].name().to_string();
522                                            let result = TableDiagnosticReport::new(TableDiagnosticReportType::NoReferenceTableFound(field_name), &[(-1, column as i32)], &table_info.fields_processed);
523                                            diagnostic.results_mut().push(result);
524                                        }
525                                        no_ref_table_found_marked.insert(column);
526                                    }
527                                }
528                            }
529                        }
530
531                        if row_keys_are_empty && field.is_key(table_info.patches) && (!cell_data.is_empty() && cell_data != "false") {
532                            row_keys_are_empty = false;
533                        }
534
535                        if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field.name()), Some("EmptyKeyField"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && field.is_key(table_info.patches) && table_info.key_amount == 1 && *field.field_type() != FieldType::OptionalStringU8 && *field.field_type() != FieldType::Boolean && (cell_data.is_empty() || cell_data == "false") {
536                            let result = TableDiagnosticReport::new(TableDiagnosticReportType::EmptyKeyField(field.name().to_string()), &[(row as i32, column as i32)], &table_info.fields_processed);
537                            diagnostic.results_mut().push(result);
538                        }
539
540                        if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field.name()), Some("ValueCannotBeEmpty"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && cell_data.is_empty() && field.cannot_be_empty(table_info.patches) {
541                            let result = TableDiagnosticReport::new(TableDiagnosticReportType::ValueCannotBeEmpty(field.name().to_string()), &[(row as i32, column as i32)], &table_info.fields_processed);
542                            diagnostic.results_mut().push(result);
543                        }
544
545                        if field.is_key(table_info.patches) {
546                            row_keys.insert(column as i32, &cells[column]);
547                        }
548                    }
549
550                    if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("EmptyRow"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && cells == &table_info.default_row {
551                        let result = TableDiagnosticReport::new(TableDiagnosticReportType::EmptyRow, &[(row as i32, -1)], &table_info.fields_processed);
552                        diagnostic.results_mut().push(result);
553                    }
554
555                    if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("EmptyKeyFields"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && row_keys_are_empty && table_info.key_amount > 1 {
556                        let cells_affected = row_keys.keys().map(|column| (row as i32, *column)).collect::<Vec<(i32, i32)>>();
557                        let result = TableDiagnosticReport::new(TableDiagnosticReportType::EmptyKeyFields, &cells_affected, &table_info.fields_processed);
558                        diagnostic.results_mut().push(result);
559                    }
560
561                    let keys = row_keys.values().copied().collect::<Vec<_>>();
562                    let values = (row_keys.keys().map(|column| (row as i32, *column)).collect::<Vec<(i32, i32)>>(), index);
563                    if !keys.is_empty() {
564                        match global_keys.get_mut(&keys) {
565                            Some(val) => val.push(values),
566                            None => { global_keys.insert(keys, vec![values]); },
567                        }
568                    }
569                }
570
571                if !diagnostic.results().is_empty() {
572                    diagnostics.push(DiagnosticType::DB(diagnostic));
573                }
574            }
575        }
576
577        // This diagnostics needs the row keys data for all tables to be generated. So we have to perform it outside the usual check loop.
578        //
579        // Also, unlike other diagnostics, we don't know what entries of this should be ignored (if any) until we find the duplicates.
580        global_keys.iter()
581            .filter(|(_, val)| val.len() > 1)
582            .for_each(|(key, val)| {
583                for (pos, index) in val {
584                    if let Some(table_info) = table_infos.get(*index) {
585                        if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics,
586                            None,
587                            Some("DuplicatedCombinedKeys"),
588                            &table_info.ignored_fields,
589                            &table_info.ignored_diagnostics,
590                            &table_info.ignored_diagnostics_for_fields
591                        ) {
592
593                            match diagnostics.iter_mut().find(|x| x.path() == table_info.path) {
594                                Some(diag) => if let DiagnosticType::DB(ref mut diag) = diag {
595                                    diag.results_mut().push(
596                                        TableDiagnosticReport::new(
597                                            TableDiagnosticReportType::DuplicatedCombinedKeys(
598                                                key.iter().map(|x| x.data_to_string()).join("| |")
599                                            ),
600                                            pos,
601                                            &table_info.fields_processed
602                                        )
603                                    )
604                                }
605                                None => {
606                                    let mut diag = TableDiagnostic::new(table_info.path, table_info.pack_key);
607                                        diag.results_mut().push(
608                                        TableDiagnosticReport::new(
609                                            TableDiagnosticReportType::DuplicatedCombinedKeys(
610                                                key.iter().map(|x| x.data_to_string()).join("| |")
611                                            ),
612                                            pos,
613                                            &table_info.fields_processed
614                                        )
615                                    );
616
617                                    // Add the new diagnostic and update the cached references so this one is also searched when processing other entries.
618                                    diagnostics.push(DiagnosticType::DB(diag));
619                                }
620                            }
621                        }
622                    }
623                }
624            });
625
626        diagnostics
627    }
628
629    /// This function takes care of checking the loc tables of your mod for errors.
630    pub fn check_loc(
631        files: &[(&str, &RFile)],
632        global_ignored_diagnostics: &[String],
633        files_to_ignore: &Option<Vec<DiagnosticIgnoreEntry>>
634    ) -> Vec<DiagnosticType> {
635        let mut diagnostics = vec![];
636
637        // So, the way we do this semi-optimized, is we do a first loop getting all the cached data we're going to need,
638        // then do the real loop, having the data of all files available for checking diagnostics.
639        let mut table_infos = vec![];
640        for (pack_key, file) in files {
641            let (ignored_fields, ignored_diagnostics, ignored_diagnostics_for_fields) = Diagnostics::ignore_data_for_file(file, files_to_ignore).unwrap_or_default();
642            if let Ok(RFileDecoded::Loc(table)) = file.decoded() {
643                let fields_processed = table.definition().fields_processed();
644                let patches = Some(table.definition().patches());
645                let table_data = table.data();
646
647                table_infos.push(TableInfo {
648                    path: file.path_in_container_raw(),
649                    pack_key,
650                    key_amount: fields_processed.iter().filter(|field| field.is_key(patches)).count(),
651                    fields_processed,
652                    patches,
653                    table_data,
654                    default_row: table.new_row(),
655                    ignored_fields,
656                    ignored_diagnostics,
657                    ignored_diagnostics_for_fields
658                });
659            }
660        }
661
662        let dec_files = files.iter()
663            .filter_map(|(_, x)| match x.decoded().ok() {
664                Some(RFileDecoded::Loc(ref table)) => Some((table, *x)),
665                _ => None,
666            })
667            .collect::<Vec<_>>();
668
669        let mut global_keys: LocKeyIndex = HashMap::with_capacity(table_infos.iter().map(|x| x.table_data.len()).sum());
670
671        for (index, (table, _file)) in dec_files.iter().enumerate() {
672            if let Some(table_info) = table_infos.get(index) {
673                let mut diagnostic = TableDiagnostic::new(table_info.path, table_info.pack_key);
674                let fields = table.definition().fields_processed();
675                let field_key_name = fields[0].name();
676                let field_text_name = fields[1].name();
677
678                for (row, cells) in table_info.table_data.iter().enumerate() {
679                    let key = cells[0].data_to_string();
680                    let data = cells[1].data_to_string();
681                    if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field_key_name), Some("InvalidLocKey"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && !key.is_empty() && (key.contains('\n') || key.contains('\r') || key.contains('\t')) {
682                        let result = TableDiagnosticReport::new(TableDiagnosticReportType::InvalidLocKey, &[(row as i32, 0)], &fields);
683                        diagnostic.results_mut().push(result);
684                    }
685
686                    // Only in case none of the two columns are ignored, we perform these checks.
687                    if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field_key_name), Some("EmptyRow"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field_text_name), Some("EmptyRow"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && key.is_empty() && data.is_empty() {
688                        let result = TableDiagnosticReport::new(TableDiagnosticReportType::EmptyRow, &[(row as i32, -1)], &fields);
689                        diagnostic.results_mut().push(result);
690                    }
691
692                    if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field_key_name), Some("EmptyKeyField"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && key.is_empty() && !data.is_empty() {
693                        let result = TableDiagnosticReport::new(TableDiagnosticReportType::EmptyKeyField("Key".to_string()), &[(row as i32, 0)], &fields);
694                        diagnostic.results_mut().push(result);
695                    }
696
697                    // Magic Regex. It works. Don't ask why.
698                    if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field_text_name), Some("InvalidEscape"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) &&
699                        !data.is_empty() &&
700                        (
701
702                            // Magic Regex didn't work, so we have to check for lines either with formatted symbols, or with missing slashes.
703                            (data.contains("\r") || (data.match_indices("\\r").count() != data.match_indices("\\\\r").count())) ||
704                            (data.contains("\n") || (data.match_indices("\\n").count() != data.match_indices("\\\\n").count())) ||
705                            (data.contains("\t") || (data.match_indices("\\t").count() != data.match_indices("\\\\t").count()))
706                        ) {
707                        let result = TableDiagnosticReport::new(TableDiagnosticReportType::InvalidEscape, &[(row as i32, 1)], &fields);
708                        diagnostic.results_mut().push(result);
709                    }
710
711                    match global_keys.get_mut(&cells[0]) {
712                        Some(val) => val.push(((row as i32, 0i32), index)),
713                        None => { global_keys.insert(&cells[0], vec![((row as i32, 0i32), index)]); },
714                    }
715                }
716
717
718                if !diagnostic.results().is_empty() {
719                    diagnostics.push(DiagnosticType::Loc(diagnostic));
720                }
721            }
722        }
723
724        // This diagnostics needs the row keys data for all tables to be generated. So we have to perform it outside the usual check loop.
725        //
726        // Also, unlike other diagnostics, we don't know what entries of this should be ignored (if any) until we find the duplicates.
727        global_keys.iter()
728            .filter(|(_, val)| val.len() > 1)
729            .for_each(|(key, val)| {
730                for (pos, index) in val {
731                    if let Some(table_info) = table_infos.get(*index) {
732                        if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics,
733                            None,
734                            Some("DuplicatedCombinedKeys"),
735                            &table_info.ignored_fields,
736                            &table_info.ignored_diagnostics,
737                            &table_info.ignored_diagnostics_for_fields
738                        ) {
739
740                            match diagnostics.iter_mut().find(|x| x.path() == table_info.path) {
741                                Some(diag) => if let DiagnosticType::Loc(ref mut diag) = diag {
742                                    diag.results_mut().push(
743                                        TableDiagnosticReport::new(
744                                            TableDiagnosticReportType::DuplicatedCombinedKeys(
745                                                key.data_to_string().to_string()
746                                            ),
747                                            &[*pos],
748                                            &table_info.fields_processed
749                                        )
750                                    )
751                                }
752                                None => {
753                                    let mut diag = TableDiagnostic::new(table_info.path, table_info.pack_key);
754                                        diag.results_mut().push(
755                                        TableDiagnosticReport::new(
756                                            TableDiagnosticReportType::DuplicatedCombinedKeys(
757                                                key.data_to_string().to_string()
758                                            ),
759                                            &[*pos],
760                                            &table_info.fields_processed
761                                        )
762                                    );
763
764                                    // Add the new diagnostic and update the cached references so this one is also searched when processing other entries.
765                                    diagnostics.push(DiagnosticType::Loc(diag));
766                                }
767                            }
768                        }
769
770                    }
771                }
772
773                let mut values = Vec::with_capacity(val.len());
774                for (pos, index) in val {
775                    if let Some(table_info) = table_infos.get(*index) {
776                        values.push((&table_info.table_data[pos.0 as usize][1], pos.0, index));
777                    }
778                }
779
780                values.sort_unstable_by_key(|x| x.0.data_to_string());
781                let dups = values.iter().duplicates_by(|x| x.0);
782                let poss = values.iter().positions(|x| dups.clone().any(|y| y.0 == x.0));
783
784                for pos in poss {
785                    if let Some((data, row, index)) = values.get(pos) {
786                        if let Some(table_info) = table_infos.get(**index) {
787                            if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics,
788                                None,
789                                Some("DuplicatedRow"),
790                                &table_info.ignored_fields,
791                                &table_info.ignored_diagnostics,
792                                &table_info.ignored_diagnostics_for_fields
793                            ) {
794
795                                match diagnostics.iter_mut().find(|x| x.path() == table_info.path) {
796                                    Some(diag) => if let DiagnosticType::Loc(ref mut diag) = diag {
797                                        diag.results_mut().push(
798                                            TableDiagnosticReport::new(
799                                                TableDiagnosticReportType::DuplicatedRow(
800                                                    String::from(table_info.table_data[*row as usize][0].data_to_string()) + "| |" + &data.data_to_string()
801                                                ),
802                                                &[(*row, 0), (*row, 1)],
803                                                &table_info.fields_processed
804                                            )
805                                        )
806                                    }
807                                    None => {
808                                        let mut diag = TableDiagnostic::new(table_info.path, table_info.pack_key);
809                                            diag.results_mut().push(
810                                            TableDiagnosticReport::new(
811                                                TableDiagnosticReportType::DuplicatedRow(
812                                                    String::from(table_info.table_data[*row as usize][0].data_to_string()) + "| |" + &data.data_to_string()
813                                                ),
814                                                &[(*row, 0), (*row, 1)],
815                                                &table_info.fields_processed
816                                            )
817                                        );
818
819                                        // Add the new diagnostic and update the cached references so this one is also searched when processing other entries.
820                                        diagnostics.push(DiagnosticType::Loc(diag));
821                                    }
822                                }
823                            }
824                        }
825                    }
826                }
827            });
828
829        diagnostics
830    }
831}
rpfm_extensions/diagnostics/table.rs

rpfm_extensions/diagnostics/
table.rs
