rpfm_extensions/diagnostics/
table.rs

1//---------------------------------------------------------------------------//
2// Copyright (c) 2017-2026 Ismael Gutiérrez González. All rights reserved.
3//
4// This file is part of the Rusted PackFile Manager (RPFM) project,
5// which can be found here: https://github.com/Frodo45127/rpfm.
6//
7// This file is licensed under the MIT license, which can be found here:
8// https://github.com/Frodo45127/rpfm/blob/master/LICENSE.
9//---------------------------------------------------------------------------//
10
11//! Module with the structs and functions specific for `Table` diagnostics.
12
13use getset::{Getters, MutGetters};
14use itertools::Itertools;
15use serde_derive::{Serialize, Deserialize};
16
17use std::{fmt, fmt::Display};
18
19use rpfm_lib::files::table::DecodedData;
20use rpfm_lib::schema::{DefinitionPatch, Field};
21
22use crate::diagnostics::*;
23
24//-------------------------------------------------------------------------------//
25//                              Enums & Structs
26//-------------------------------------------------------------------------------//
27
28/// This struct contains the results of a Table diagnostic.
29#[derive(Debug, Clone, Default, Getters, MutGetters, Serialize, Deserialize)]
30#[getset(get = "pub", get_mut = "pub")]
31pub struct TableDiagnostic {
32    path: String,
33    pack: String,
34    results: Vec<TableDiagnosticReport>
35}
36
37/// This struct defines an individual table diagnostic result.
38#[derive(Debug, Clone, Getters, MutGetters, Serialize, Deserialize)]
39#[getset(get = "pub", get_mut = "pub")]
40pub struct TableDiagnosticReport {
41
42    /// List of cells, in "row, column" format.
43    ///
44    /// If the full row or full column are affected, use -1.
45    cells_affected: Vec<(i32, i32)>,
46
47    /// Name of the columns that corresponds to the affected cells.
48    column_names: Vec<String>,
49    report_type: TableDiagnosticReportType,
50}
51
52#[derive(Debug, Clone, Serialize, Deserialize)]
53pub enum TableDiagnosticReportType {
54    OutdatedTable,
55    InvalidReference(String, String),
56    EmptyRow,
57    EmptyKeyField(String),
58    EmptyKeyFields,
59    DuplicatedCombinedKeys(String),
60    NoReferenceTableFound(String),
61    NoReferenceTableNorColumnFoundPak(String),
62    NoReferenceTableNorColumnFoundNoPak(String),
63    InvalidEscape,
64    DuplicatedRow(String),
65    InvalidLocKey,
66    TableNameEndsInNumber,
67    TableNameHasSpace,
68    TableIsDataCoring,
69    FieldWithPathNotFound(Vec<String>),
70    BannedTable,
71    ValueCannotBeEmpty(String),
72    AlteredTable,
73}
74
75/// Internal struct with cached data of tables.
76///
77/// Used to cache multiple tables and having them being aware of each other on check.
78struct TableInfo<'a> {
79    path: &'a str,
80    pack_key: &'a str,
81    fields_processed: Vec<Field>,
82    patches: Option<&'a DefinitionPatch>,
83    key_amount: usize,
84    table_data: Cow<'a, [Vec<DecodedData>]>,
85    default_row: Vec<DecodedData>,
86    ignored_fields: Vec<String>,
87    ignored_diagnostics: HashSet<String>,
88    ignored_diagnostics_for_fields: HashMap<String, Vec<String>>,
89}
90
91//-------------------------------------------------------------------------------//
92//                             Implementations
93//-------------------------------------------------------------------------------//
94
95impl TableDiagnosticReport {
96    pub fn new(report_type: TableDiagnosticReportType, cells_affected: &[(i32, i32)], fields: &[Field]) -> Self {
97        let mut fields_affected = cells_affected.iter().map(|(_, column)| *column).collect::<Vec<_>>();
98        fields_affected.sort();
99        fields_affected.dedup();
100
101        if fields_affected.contains(&-1) {
102            fields_affected = vec![-1];
103        }
104
105        Self {
106            cells_affected: cells_affected.to_vec(),
107            column_names: fields_affected.iter().flat_map(|index| {
108                if index == &-1 {
109                    fields.iter().map(|field| field.name().to_owned()).collect()
110                } else {
111                    vec![fields[*index as usize].name().to_owned()]
112                }
113            }).collect(),
114            report_type
115        }
116    }
117}
118
119impl DiagnosticReport for TableDiagnosticReport {
120    fn message(&self) -> String {
121        match &self.report_type {
122            TableDiagnosticReportType::OutdatedTable => "Possibly outdated table".to_owned(),
123            TableDiagnosticReportType::InvalidReference(cell_data, field_name) => format!("Invalid reference \"{cell_data}\" in column \"{field_name}\"."),
124            TableDiagnosticReportType::EmptyRow => "Empty row.".to_owned(),
125            TableDiagnosticReportType::EmptyKeyField(field_name) => format!("Empty key for column \"{field_name}\"."),
126            TableDiagnosticReportType::EmptyKeyFields => "Empty key fields.".to_owned(),
127            TableDiagnosticReportType::DuplicatedCombinedKeys(combined_keys) => format!("Duplicated combined keys: {}.", &combined_keys),
128            TableDiagnosticReportType::NoReferenceTableFound(field_name) => format!("No reference table found for column \"{field_name}\"."),
129            TableDiagnosticReportType::NoReferenceTableNorColumnFoundPak(field_name) => format!("No reference column found in referenced table for column \"{field_name}\". Maybe a problem with the schema?"),
130            TableDiagnosticReportType::NoReferenceTableNorColumnFoundNoPak(field_name) => format!("No reference column found in referenced table for column \"{field_name}\". Did you forget to generate the Dependencies Cache, or did you generate it before installing the Assembly kit?"),
131            TableDiagnosticReportType::InvalidEscape => "Invalid line jump/tabulation detected in loc entry. Use \\\\n or \\\\t instead.".to_owned(),
132            TableDiagnosticReportType::DuplicatedRow(combined_keys) => format!("Duplicated row: {combined_keys}."),
133            TableDiagnosticReportType::InvalidLocKey => "Invalid localisation key.".to_owned(),
134            TableDiagnosticReportType::TableNameEndsInNumber => "Table name ends in number.".to_owned(),
135            TableDiagnosticReportType::TableNameHasSpace => "Table name contains spaces.".to_owned(),
136            TableDiagnosticReportType::TableIsDataCoring => "Table is datacoring.".to_owned(),
137            TableDiagnosticReportType::FieldWithPathNotFound(paths) => format!("Path not found: {}.", paths.iter().join(" || ")),
138            TableDiagnosticReportType::BannedTable => "Banned table.".to_owned(),
139            TableDiagnosticReportType::ValueCannotBeEmpty(field_name) => format!("Empty value for column \"{field_name}\"."),
140            TableDiagnosticReportType::AlteredTable => "Altered Table".to_owned(),
141        }
142    }
143
144    fn level(&self) -> DiagnosticLevel {
145        match self.report_type {
146            TableDiagnosticReportType::OutdatedTable => DiagnosticLevel::Error,
147            TableDiagnosticReportType::InvalidReference(_,_) => DiagnosticLevel::Error,
148            TableDiagnosticReportType::EmptyRow => DiagnosticLevel::Warning,
149            TableDiagnosticReportType::EmptyKeyField(_) => DiagnosticLevel::Error,
150            TableDiagnosticReportType::EmptyKeyFields => DiagnosticLevel::Warning,
151            TableDiagnosticReportType::DuplicatedCombinedKeys(_) => DiagnosticLevel::Warning,
152            TableDiagnosticReportType::NoReferenceTableFound(_) => DiagnosticLevel::Info,
153            TableDiagnosticReportType::NoReferenceTableNorColumnFoundPak(_) => DiagnosticLevel::Info,
154            TableDiagnosticReportType::NoReferenceTableNorColumnFoundNoPak(_) => DiagnosticLevel::Warning,
155            TableDiagnosticReportType::InvalidEscape => DiagnosticLevel::Warning,
156            TableDiagnosticReportType::DuplicatedRow(_) => DiagnosticLevel::Warning,
157            TableDiagnosticReportType::InvalidLocKey => DiagnosticLevel::Error,
158            TableDiagnosticReportType::TableNameEndsInNumber => DiagnosticLevel::Error,
159            TableDiagnosticReportType::TableNameHasSpace => DiagnosticLevel::Error,
160            TableDiagnosticReportType::TableIsDataCoring => DiagnosticLevel::Warning,
161            TableDiagnosticReportType::FieldWithPathNotFound(_) => DiagnosticLevel::Warning,
162            TableDiagnosticReportType::BannedTable => DiagnosticLevel::Error,
163            TableDiagnosticReportType::ValueCannotBeEmpty(_) => DiagnosticLevel::Error,
164            TableDiagnosticReportType::AlteredTable => DiagnosticLevel::Error,
165        }
166    }
167}
168
169impl Display for TableDiagnosticReportType {
170    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
171        Display::fmt(match self {
172            Self::OutdatedTable => "OutdatedTable",
173            Self::InvalidReference(_,_) => "InvalidReference",
174            Self::EmptyRow => "EmptyRow",
175            Self::EmptyKeyField(_) => "EmptyKeyField",
176            Self::EmptyKeyFields => "EmptyKeyFields",
177            Self::DuplicatedCombinedKeys(_) => "DuplicatedCombinedKeys",
178            Self::NoReferenceTableFound(_) => "NoReferenceTableFound",
179            Self::NoReferenceTableNorColumnFoundPak(_) => "NoReferenceTableNorColumnFoundPak",
180            Self::NoReferenceTableNorColumnFoundNoPak(_) => "NoReferenceTableNorColumnFoundNoPak",
181            Self::InvalidEscape => "InvalidEscape",
182            Self::DuplicatedRow(_) => "DuplicatedRow",
183            Self::InvalidLocKey => "InvalidLocKey",
184            Self::TableNameEndsInNumber => "TableNameEndsInNumber",
185            Self::TableNameHasSpace => "TableNameHasSpace",
186            Self::TableIsDataCoring => "TableIsDataCoring",
187            Self::FieldWithPathNotFound(_) => "FieldWithPathNotFound",
188            Self::BannedTable => "BannedTable",
189            Self::ValueCannotBeEmpty(_) => "ValueCannotBeEmpty",
190            Self::AlteredTable => "AlteredTable",
191        }, f)
192    }
193}
194
195
196impl TableDiagnostic {
197    pub fn new(path: &str, pack: &str) -> Self {
198        Self {
199            path: path.to_owned(),
200            pack: pack.to_owned(),
201            results: vec![],
202        }
203    }
204
205    /// This function is used to check if a table is outdated or not.
206    fn is_table_outdated(table_name: &str, table_version: i32, dependencies: &Dependencies) -> bool {
207        if let Ok(vanilla_dbs) = dependencies.db_data(table_name, true, false) {
208            if let Some(max_version) = vanilla_dbs.iter()
209                .filter_map(|x| {
210                    if let Ok(RFileDecoded::DB(table)) = x.decoded() {
211                        Some(table.definition().version())
212                    } else {
213                        None
214                    }
215                }).max_by(|x, y| x.cmp(y)) {
216                if *max_version != table_version {
217                    return true
218                }
219            }
220        }
221
222        false
223    }
224
225    /// This function takes care of checking the db tables of your mod for errors.
226    pub fn check_db(
227        files: &[(&str, &RFile)],
228        dependencies: &Dependencies,
229        global_ignored_diagnostics: &[String],
230        game_info: &GameInfo,
231        local_path_list: &HashMap<String, Vec<String>>,
232        check_ak_only_refs: bool,
233        files_to_ignore: &Option<Vec<(String, Vec<String>, Vec<String>)>>,
234        packs: &BTreeMap<String, Pack>,
235        schema: &Schema,
236        loc_data: &Option<HashMap<Cow<str>, Cow<str>>>
237    ) -> Vec<DiagnosticType> {
238        let mut diagnostics = vec![];
239
240        if files.is_empty() {
241            return diagnostics;
242        }
243
244        // Get the dependency data for tables once per batch. That way we can speed up this a lot.
245        let file = files.first().and_then(|(_, x)| x.decoded().ok());
246        let dependency_data = if let Some(RFileDecoded::DB(table)) = file {
247            dependencies.db_reference_data(schema, packs, table.table_name(), table.definition(), loc_data)
248        } else {
249            return diagnostics
250        };
251
252        // So, the way we do this semi-optimized, is we do a first loop getting all the cached data we're going to need,
253        // then do the real loop, having the data of all files available for checking diagnostics.
254        let mut table_infos = vec![];
255        for (pack_key, file) in files {
256            let (ignored_fields, ignored_diagnostics, ignored_diagnostics_for_fields) = Diagnostics::ignore_data_for_file(file, files_to_ignore).unwrap_or_default();
257            if let Ok(RFileDecoded::DB(table)) = file.decoded() {
258                let fields_processed = table.definition().fields_processed();
259                let patches = Some(table.definition().patches());
260                let table_data = table.data();
261
262                table_infos.push(TableInfo {
263                    path: file.path_in_container_raw(),
264                    pack_key,
265                    key_amount: fields_processed.iter().filter(|field| field.is_key(patches)).count(),
266                    fields_processed,
267                    patches,
268                    table_data,
269                    default_row: table.new_row(),
270                    ignored_fields,
271                    ignored_diagnostics,
272                    ignored_diagnostics_for_fields
273                });
274            }
275        }
276
277        let mut global_keys: HashMap<Vec<&DecodedData>, Vec<(Vec<(i32, i32)>, usize)>> = HashMap::with_capacity(table_infos.iter().map(|x| x.table_data.len()).sum());
278        let dec_files = files.iter()
279            .filter_map(|(_, x)| match x.decoded().ok() {
280                Some(RFileDecoded::DB(ref table)) => Some((table, *x)),
281                _ => None,
282            })
283            .collect::<Vec<_>>();
284
285        for (index, (table, file)) in dec_files.iter().enumerate() {
286            let is_twad_key_deletes = table.table_name().starts_with("twad_key_deletes");
287            let check_ak_only = check_ak_only_refs || table.table_name().starts_with("start_pos_");
288            if let Some(table_info) = table_infos.get(index) {
289                let mut diagnostic = TableDiagnostic::new(table_info.path, table_info.pack_key);
290
291                // Before anything else, check if the table is outdated.
292                if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("OutdatedTable"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && Self::is_table_outdated(table.table_name(), *table.definition().version(), dependencies) {
293                    let result = TableDiagnosticReport::new(TableDiagnosticReportType::OutdatedTable, &[], &[]);
294                    diagnostic.results_mut().push(result);
295                }
296
297                // Check if it's one of the banned tables for the game selected.
298                if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("BannedTable"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && game_info.is_file_banned(file.path_in_container_raw()) {
299                    let result = TableDiagnosticReport::new(TableDiagnosticReportType::BannedTable, &[], &[]);
300                    diagnostic.results_mut().push(result);
301                }
302
303                if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("AlteredTable"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && table.altered() {
304                    let result = TableDiagnosticReport::new(TableDiagnosticReportType::AlteredTable, &[], &[]);
305                    diagnostic.results_mut().push(result);
306                }
307
308                // Check if the table name has a number at the end, which causes very annoying bugs.
309                if let Some(name) = file.file_name() {
310                    if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("TableNameEndsInNumber"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && (name.ends_with('0') ||
311                        name.ends_with('1') ||
312                        name.ends_with('2') ||
313                        name.ends_with('3') ||
314                        name.ends_with('4') ||
315                        name.ends_with('5') ||
316                        name.ends_with('6') ||
317                        name.ends_with('7') ||
318                        name.ends_with('8') || name.ends_with('9')) {
319
320                        let result = TableDiagnosticReport::new(TableDiagnosticReportType::TableNameEndsInNumber, &[], &[]);
321                        diagnostic.results_mut().push(result);
322                    }
323
324                    if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("TableNameHasSpace"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && name.contains(' ') {
325                        let result = TableDiagnosticReport::new(TableDiagnosticReportType::TableNameHasSpace, &[], &[]);
326                        diagnostic.results_mut().push(result);
327                    }
328
329                    if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("TableIsDataCoring"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) {
330                        match game_info.vanilla_db_table_name_logic() {
331                            VanillaDBTableNameLogic::FolderName => {
332                                if table.table_name_without_tables() == file.path_in_container_split()[2] {
333                                    let result = TableDiagnosticReport::new(TableDiagnosticReportType::TableIsDataCoring, &[], &[]);
334                                    diagnostic.results_mut().push(result);
335                                }
336                            }
337
338                            VanillaDBTableNameLogic::DefaultName(ref default_name) => {
339                                if name == default_name {
340                                    let result = TableDiagnosticReport::new(TableDiagnosticReportType::TableIsDataCoring, &[], &[]);
341                                    diagnostic.results_mut().push(result);
342                                }
343                            }
344                        }
345                    }
346                }
347
348                // Columns we can try to check for paths.
349                let mut ignore_path_columns = vec![];
350                for (column, field) in table_info.fields_processed.iter().enumerate() {
351                    if let Some(rel_paths) = field.filename_relative_path(table_info.patches) {
352                        if rel_paths.iter().any(|path| path.contains('*')) {
353                            ignore_path_columns.push(column);
354                        }
355                    }
356                }
357
358                let mut no_ref_table_nor_column_found_marked = HashSet::new();
359                let mut no_ref_table_found_marked = HashSet::new();
360
361                for (row, cells) in table_info.table_data.iter().enumerate() {
362                    let mut row_keys_are_empty = true;
363                    let mut row_keys: BTreeMap<i32, &DecodedData> = BTreeMap::new();
364                    for (column, field) in table_info.fields_processed.iter().enumerate() {
365
366                        // Skip unused field on diagnostics.
367                        //if field.unused(patches) {
368                        //    continue;
369                        //}
370
371                        let cell_data = cells[column].data_to_string();
372
373                        // Path checks.
374                        if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field.name()), Some("FieldWithPathNotFound"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) &&
375                            !cell_data.is_empty() &&
376                            cell_data != "." &&
377                            cell_data != "x" &&
378                            cell_data != "false" &&
379                            cell_data != "building_placeholder" &&
380                            cell_data != "placeholder" &&
381                            cell_data != "PLACEHOLDER" &&
382                            cell_data != "placeholder.png" &&
383                            cell_data != "placehoder.png" &&
384                            table_info.fields_processed[column].is_filename(table_info.patches) &&
385                            !ignore_path_columns.contains(&column) {
386
387                            let mut path_found = false;
388                            let relative_paths = table_info.fields_processed[column].filename_relative_path(table_info.patches);
389                            let paths = if let Some(relative_paths) = relative_paths {
390                                relative_paths.iter()
391                                    .flat_map(|x| {
392                                        let mut paths = vec![];
393                                        let cell_data = cell_data.replace('\\', "/");
394                                        for cell_data in cell_data.split(',') {
395
396                                            // When analysing paths, fix the ones in older games starting with / or data/.
397                                            let mut start_offset = 0;
398                                            if cell_data.starts_with("/") {
399                                                start_offset += 1;
400                                            }
401                                            if cell_data.starts_with("data/") {
402                                                start_offset += 5;
403                                            }
404
405                                            paths.push(x.replace('%', &cell_data[start_offset..]));
406                                        }
407
408                                        paths
409                                    })
410                                    .collect::<Vec<_>>()
411                            } else {
412                                let mut paths = vec![];
413                                let cell_data = cell_data.replace('\\', "/");
414                                for cell_data in cell_data.split(',') {
415
416                                    // When analysing paths, fix the ones in older games starting with / or data/.
417                                    let mut start_offset = 0;
418                                    if cell_data.starts_with("/") {
419                                        start_offset += 1;
420                                    }
421                                    if cell_data.starts_with("data/") {
422                                        start_offset += 5;
423                                    }
424
425                                    paths.push(cell_data[start_offset..].to_string());
426                                }
427
428                                paths
429                            };
430
431                            for path in &paths {
432                                if !path_found && local_path_list.get(&path.to_lowercase()).is_some() {
433                                    path_found = true;
434                                }
435
436                                if !path_found && dependencies.file_exists(path, true, true, true) {
437                                    path_found = true;
438                                }
439
440                                if path_found {
441                                    break;
442                                }
443                            }
444
445                            if !path_found {
446                                let result = TableDiagnosticReport::new(TableDiagnosticReportType::FieldWithPathNotFound(paths), &[(row as i32, column as i32)], &table_info.fields_processed);
447                                diagnostic.results_mut().push(result);
448                            }
449                        }
450
451                        // Dependency checks.
452                        if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field.name()), None, &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) &&
453                            (field.is_reference(table_info.patches).is_some() ||
454                                (
455                                    is_twad_key_deletes &&
456                                    field.name() == "table_name"
457                                )
458                            ) {
459
460                            match dependency_data.get(&(column as i32)) {
461                                Some(ref_data) => {
462                                    if *ref_data.referenced_column_is_localised() {
463                                        // TODO: report missing loc data here.
464                                    }
465                                    /*
466                                    else if ref_data.referenced_table_is_ak_only {
467                                        // If it's only in the AK, ignore it.
468                                    }*/
469
470                                    // Blue cell check. Only one for each column, so we don't fill the diagnostics with this.
471                                    //
472                                    // NOTE: This diagnostics should be one, per column. Once it's done, do not create a new one
473                                    // for the same column in subsequent rows.
474                                    else if ref_data.data().is_empty() && (no_ref_table_nor_column_found_marked.is_empty() || !no_ref_table_nor_column_found_marked.contains(&column)) {
475                                        if !dependencies.is_asskit_data_loaded() {
476                                            if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("NoReferenceTableNorColumnFoundNoPak"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) {
477                                                let field_name = table_info.fields_processed[column].name().to_string();
478                                                let result = TableDiagnosticReport::new(TableDiagnosticReportType::NoReferenceTableNorColumnFoundNoPak(field_name), &[(-1, column as i32)], &table_info.fields_processed);
479                                                diagnostic.results_mut().push(result);
480                                            }
481                                        }
482                                        else if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("NoReferenceTableNorColumnFoundPak"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) {
483                                            let field_name = table_info.fields_processed[column].name().to_string();
484                                            let result = TableDiagnosticReport::new(TableDiagnosticReportType::NoReferenceTableNorColumnFoundPak(field_name), &[(-1, column as i32)], &table_info.fields_processed);
485                                            diagnostic.results_mut().push(result);
486                                        }
487
488                                        no_ref_table_nor_column_found_marked.insert(column);
489                                    }
490
491                                    // Check for non-empty cells with reference data, but the data in the cell is not in the reference data list.
492                                    else if !ref_data.data().is_empty() && !cell_data.is_empty() && !ref_data.data().contains_key(&*cell_data) && (!*ref_data.referenced_table_is_ak_only() || check_ak_only) {
493
494                                        // Numeric cells with 0 are "empty" references and should not be checked.
495                                        let is_number = *field.field_type() == FieldType::I32 || *field.field_type() == FieldType::I64 || *field.field_type() == FieldType::OptionalI32 || *field.field_type() == FieldType::OptionalI64;
496                                        let is_valid_reference = if is_number { cell_data != "0" } else { true };
497                                        if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field.name()), Some("InvalidReference"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && is_valid_reference {
498                                            let result = TableDiagnosticReport::new(TableDiagnosticReportType::InvalidReference(cell_data.to_string(), field.name().to_string()), &[(row as i32, column as i32)], &table_info.fields_processed);
499                                            diagnostic.results_mut().push(result);
500                                        }
501                                    }
502                                }
503                                None => {
504
505                                    // This diagnostic also needs to be done once per column.
506                                    if no_ref_table_found_marked.is_empty() || !no_ref_table_found_marked.contains(&column) {
507                                        if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("NoReferenceTableFound"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) {
508                                            let field_name = table_info.fields_processed[column].name().to_string();
509                                            let result = TableDiagnosticReport::new(TableDiagnosticReportType::NoReferenceTableFound(field_name), &[(-1, column as i32)], &table_info.fields_processed);
510                                            diagnostic.results_mut().push(result);
511                                        }
512                                        no_ref_table_found_marked.insert(column);
513                                    }
514                                }
515                            }
516                        }
517
518                        if row_keys_are_empty && field.is_key(table_info.patches) && (!cell_data.is_empty() && cell_data != "false") {
519                            row_keys_are_empty = false;
520                        }
521
522                        if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field.name()), Some("EmptyKeyField"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && field.is_key(table_info.patches) && table_info.key_amount == 1 && *field.field_type() != FieldType::OptionalStringU8 && *field.field_type() != FieldType::Boolean && (cell_data.is_empty() || cell_data == "false") {
523                            let result = TableDiagnosticReport::new(TableDiagnosticReportType::EmptyKeyField(field.name().to_string()), &[(row as i32, column as i32)], &table_info.fields_processed);
524                            diagnostic.results_mut().push(result);
525                        }
526
527                        if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field.name()), Some("ValueCannotBeEmpty"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && cell_data.is_empty() && field.cannot_be_empty(table_info.patches) {
528                            let result = TableDiagnosticReport::new(TableDiagnosticReportType::ValueCannotBeEmpty(field.name().to_string()), &[(row as i32, column as i32)], &table_info.fields_processed);
529                            diagnostic.results_mut().push(result);
530                        }
531
532                        if field.is_key(table_info.patches) {
533                            row_keys.insert(column as i32, &cells[column]);
534                        }
535                    }
536
537                    if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("EmptyRow"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && cells == &table_info.default_row {
538                        let result = TableDiagnosticReport::new(TableDiagnosticReportType::EmptyRow, &[(row as i32, -1)], &table_info.fields_processed);
539                        diagnostic.results_mut().push(result);
540                    }
541
542                    if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, None, Some("EmptyKeyFields"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && row_keys_are_empty && table_info.key_amount > 1 {
543                        let cells_affected = row_keys.keys().map(|column| (row as i32, *column)).collect::<Vec<(i32, i32)>>();
544                        let result = TableDiagnosticReport::new(TableDiagnosticReportType::EmptyKeyFields, &cells_affected, &table_info.fields_processed);
545                        diagnostic.results_mut().push(result);
546                    }
547
548                    let keys = row_keys.values().copied().collect::<Vec<_>>();
549                    let values = (row_keys.keys().map(|column| (row as i32, *column)).collect::<Vec<(i32, i32)>>(), index);
550                    if !keys.is_empty() {
551                        match global_keys.get_mut(&keys) {
552                            Some(val) => val.push(values),
553                            None => { global_keys.insert(keys, vec![values]); },
554                        }
555                    }
556                }
557
558                if !diagnostic.results().is_empty() {
559                    diagnostics.push(DiagnosticType::DB(diagnostic));
560                }
561            }
562        }
563
564        // This diagnostics needs the row keys data for all tables to be generated. So we have to perform it outside the usual check loop.
565        //
566        // Also, unlike other diagnostics, we don't know what entries of this should be ignored (if any) until we find the duplicates.
567        global_keys.iter()
568            .filter(|(_, val)| val.len() > 1)
569            .for_each(|(key, val)| {
570                for (pos, index) in val {
571                    if let Some(table_info) = table_infos.get(*index) {
572                        if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics,
573                            None,
574                            Some("DuplicatedCombinedKeys"),
575                            &table_info.ignored_fields,
576                            &table_info.ignored_diagnostics,
577                            &table_info.ignored_diagnostics_for_fields
578                        ) {
579
580                            match diagnostics.iter_mut().find(|x| x.path() == table_info.path) {
581                                Some(diag) => if let DiagnosticType::DB(ref mut diag) = diag {
582                                    diag.results_mut().push(
583                                        TableDiagnosticReport::new(
584                                            TableDiagnosticReportType::DuplicatedCombinedKeys(
585                                                key.iter().map(|x| x.data_to_string()).join("| |")
586                                            ),
587                                            pos,
588                                            &table_info.fields_processed
589                                        )
590                                    )
591                                }
592                                None => {
593                                    let mut diag = TableDiagnostic::new(table_info.path, table_info.pack_key);
594                                        diag.results_mut().push(
595                                        TableDiagnosticReport::new(
596                                            TableDiagnosticReportType::DuplicatedCombinedKeys(
597                                                key.iter().map(|x| x.data_to_string()).join("| |")
598                                            ),
599                                            pos,
600                                            &table_info.fields_processed
601                                        )
602                                    );
603
604                                    // Add the new diagnostic and update the cached references so this one is also searched when processing other entries.
605                                    diagnostics.push(DiagnosticType::DB(diag));
606                                }
607                            }
608                        }
609                    }
610                }
611            });
612
613        diagnostics
614    }
615
616    /// This function takes care of checking the loc tables of your mod for errors.
617    pub fn check_loc(
618        files: &[(&str, &RFile)],
619        global_ignored_diagnostics: &[String],
620        files_to_ignore: &Option<Vec<(String, Vec<String>, Vec<String>)>>
621    ) -> Vec<DiagnosticType> {
622        let mut diagnostics = vec![];
623
624        // So, the way we do this semi-optimized, is we do a first loop getting all the cached data we're going to need,
625        // then do the real loop, having the data of all files available for checking diagnostics.
626        let mut table_infos = vec![];
627        for (pack_key, file) in files {
628            let (ignored_fields, ignored_diagnostics, ignored_diagnostics_for_fields) = Diagnostics::ignore_data_for_file(file, files_to_ignore).unwrap_or_default();
629            if let Ok(RFileDecoded::Loc(table)) = file.decoded() {
630                let fields_processed = table.definition().fields_processed();
631                let patches = Some(table.definition().patches());
632                let table_data = table.data();
633
634                table_infos.push(TableInfo {
635                    path: file.path_in_container_raw(),
636                    pack_key,
637                    key_amount: fields_processed.iter().filter(|field| field.is_key(patches)).count(),
638                    fields_processed,
639                    patches,
640                    table_data,
641                    default_row: table.new_row(),
642                    ignored_fields,
643                    ignored_diagnostics,
644                    ignored_diagnostics_for_fields
645                });
646            }
647        }
648
649        let dec_files = files.iter()
650            .filter_map(|(_, x)| match x.decoded().ok() {
651                Some(RFileDecoded::Loc(ref table)) => Some((table, *x)),
652                _ => None,
653            })
654            .collect::<Vec<_>>();
655
656        let mut global_keys: HashMap<&DecodedData, Vec<((i32, i32), usize)>> = HashMap::with_capacity(table_infos.iter().map(|x| x.table_data.len()).sum());
657
658        for (index, (table, _file)) in dec_files.iter().enumerate() {
659            if let Some(table_info) = table_infos.get(index) {
660                let mut diagnostic = TableDiagnostic::new(table_info.path, table_info.pack_key);
661                let fields = table.definition().fields_processed();
662                let field_key_name = fields[0].name();
663                let field_text_name = fields[1].name();
664
665                for (row, cells) in table_info.table_data.iter().enumerate() {
666                    let key = cells[0].data_to_string();
667                    let data = cells[1].data_to_string();
668                    if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field_key_name), Some("InvalidLocKey"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && !key.is_empty() && (key.contains('\n') || key.contains('\r') || key.contains('\t')) {
669                        let result = TableDiagnosticReport::new(TableDiagnosticReportType::InvalidLocKey, &[(row as i32, 0)], &fields);
670                        diagnostic.results_mut().push(result);
671                    }
672
673                    // Only in case none of the two columns are ignored, we perform these checks.
674                    if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field_key_name), Some("EmptyRow"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field_text_name), Some("EmptyRow"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && key.is_empty() && data.is_empty() {
675                        let result = TableDiagnosticReport::new(TableDiagnosticReportType::EmptyRow, &[(row as i32, -1)], &fields);
676                        diagnostic.results_mut().push(result);
677                    }
678
679                    if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field_key_name), Some("EmptyKeyField"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) && key.is_empty() && !data.is_empty() {
680                        let result = TableDiagnosticReport::new(TableDiagnosticReportType::EmptyKeyField("Key".to_string()), &[(row as i32, 0)], &fields);
681                        diagnostic.results_mut().push(result);
682                    }
683
684                    // Magic Regex. It works. Don't ask why.
685                    if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics, Some(field_text_name), Some("InvalidEscape"), &table_info.ignored_fields, &table_info.ignored_diagnostics, &table_info.ignored_diagnostics_for_fields) &&
686                        !data.is_empty() &&
687                        (
688
689                            // Magic Regex didn't work, so we have to check for lines either with formatted symbols, or with missing slashes.
690                            (data.contains("\r") || (data.match_indices("\\r").count() != data.match_indices("\\\\r").count())) ||
691                            (data.contains("\n") || (data.match_indices("\\n").count() != data.match_indices("\\\\n").count())) ||
692                            (data.contains("\t") || (data.match_indices("\\t").count() != data.match_indices("\\\\t").count()))
693                        ) {
694                        let result = TableDiagnosticReport::new(TableDiagnosticReportType::InvalidEscape, &[(row as i32, 1)], &fields);
695                        diagnostic.results_mut().push(result);
696                    }
697
698                    match global_keys.get_mut(&cells[0]) {
699                        Some(val) => val.push(((row as i32, 0i32), index)),
700                        None => { global_keys.insert(&cells[0], vec![((row as i32, 0i32), index)]); },
701                    }
702                }
703
704
705                if !diagnostic.results().is_empty() {
706                    diagnostics.push(DiagnosticType::Loc(diagnostic));
707                }
708            }
709        }
710
711        // This diagnostics needs the row keys data for all tables to be generated. So we have to perform it outside the usual check loop.
712        //
713        // Also, unlike other diagnostics, we don't know what entries of this should be ignored (if any) until we find the duplicates.
714        global_keys.iter()
715            .filter(|(_, val)| val.len() > 1)
716            .for_each(|(key, val)| {
717                for (pos, index) in val {
718                    if let Some(table_info) = table_infos.get(*index) {
719                        if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics,
720                            None,
721                            Some("DuplicatedCombinedKeys"),
722                            &table_info.ignored_fields,
723                            &table_info.ignored_diagnostics,
724                            &table_info.ignored_diagnostics_for_fields
725                        ) {
726
727                            match diagnostics.iter_mut().find(|x| x.path() == table_info.path) {
728                                Some(diag) => if let DiagnosticType::Loc(ref mut diag) = diag {
729                                    diag.results_mut().push(
730                                        TableDiagnosticReport::new(
731                                            TableDiagnosticReportType::DuplicatedCombinedKeys(
732                                                key.data_to_string().to_string()
733                                            ),
734                                            &[*pos],
735                                            &table_info.fields_processed
736                                        )
737                                    )
738                                }
739                                None => {
740                                    let mut diag = TableDiagnostic::new(table_info.path, table_info.pack_key);
741                                        diag.results_mut().push(
742                                        TableDiagnosticReport::new(
743                                            TableDiagnosticReportType::DuplicatedCombinedKeys(
744                                                key.data_to_string().to_string()
745                                            ),
746                                            &[*pos],
747                                            &table_info.fields_processed
748                                        )
749                                    );
750
751                                    // Add the new diagnostic and update the cached references so this one is also searched when processing other entries.
752                                    diagnostics.push(DiagnosticType::Loc(diag));
753                                }
754                            }
755                        }
756
757                    }
758                }
759
760                let mut values = Vec::with_capacity(val.len());
761                for (pos, index) in val {
762                    if let Some(table_info) = table_infos.get(*index) {
763                        values.push((&table_info.table_data[pos.0 as usize][1], pos.0, index));
764                    }
765                }
766
767                values.sort_unstable_by_key(|x| x.0.data_to_string());
768                let dups = values.iter().duplicates_by(|x| x.0);
769                let poss = values.iter().positions(|x| dups.clone().any(|y| y.0 == x.0));
770
771                for pos in poss {
772                    if let Some((data, row, index)) = values.get(pos) {
773                        if let Some(table_info) = table_infos.get(**index) {
774                            if !Diagnostics::ignore_diagnostic(global_ignored_diagnostics,
775                                None,
776                                Some("DuplicatedRow"),
777                                &table_info.ignored_fields,
778                                &table_info.ignored_diagnostics,
779                                &table_info.ignored_diagnostics_for_fields
780                            ) {
781
782                                match diagnostics.iter_mut().find(|x| x.path() == table_info.path) {
783                                    Some(diag) => if let DiagnosticType::Loc(ref mut diag) = diag {
784                                        diag.results_mut().push(
785                                            TableDiagnosticReport::new(
786                                                TableDiagnosticReportType::DuplicatedRow(
787                                                    String::from(table_info.table_data[*row as usize][0].data_to_string()) + "| |" + &data.data_to_string()
788                                                ),
789                                                &[(*row, 0), (*row, 1)],
790                                                &table_info.fields_processed
791                                            )
792                                        )
793                                    }
794                                    None => {
795                                        let mut diag = TableDiagnostic::new(table_info.path, table_info.pack_key);
796                                            diag.results_mut().push(
797                                            TableDiagnosticReport::new(
798                                                TableDiagnosticReportType::DuplicatedRow(
799                                                    String::from(table_info.table_data[*row as usize][0].data_to_string()) + "| |" + &data.data_to_string()
800                                                ),
801                                                &[(*row, 0), (*row, 1)],
802                                                &table_info.fields_processed
803                                            )
804                                        );
805
806                                        // Add the new diagnostic and update the cached references so this one is also searched when processing other entries.
807                                        diagnostics.push(DiagnosticType::Loc(diag));
808                                    }
809                                }
810                            }
811                        }
812                    }
813                }
814            });
815
816        diagnostics
817    }
818}
rpfm_extensions/diagnostics/table.rs

rpfm_extensions/diagnostics/
table.rs
