rpfm_extensions/startpos/extract/
compare.rs

1//---------------------------------------------------------------------------//
2// Copyright (c) 2017-2026 Ismael Gutiérrez González. All rights reserved.
3//
4// This file is part of the Rusted PackFile Manager (RPFM) project,
5// which can be found here: https://github.com/Frodo45127/rpfm.
6//
7// This file is licensed under the MIT license, which can be found here:
8// https://github.com/Frodo45127/rpfm/blob/master/LICENSE.
9//---------------------------------------------------------------------------//
10
11//! Comparison of the tables extracted from a startpos with the tables it was built from.
12//!
13//! Every value the extraction gets back differently is something the build lost or changed on its way into the
14//! startpos. Rows are matched by what they describe, not by their ids (which the extraction renumbers): ids that
15//! reference factions, regions, settlements and characters are replaced by their keys first. Characters are told
16//! apart by faction, names and agent type.
17//!
18//! Only the columns the extraction fills are compared.
19
20use getset::Getters;
21
22use std::collections::{BTreeMap, HashMap, HashSet};
23use std::fmt;
24
25use rpfm_lib::error::Result;
26use rpfm_lib::schema::Schema;
27
28use super::ExtractedTables;
29use crate::startpos::tables::{Row, StartposTables, Table};
30use crate::startpos::world::writer::HORDE_BUILDING_COLUMNS;
31
32#[cfg(test)] #[path = "compare_test.rs"] mod compare_test;
33
34/// Most differences kept as examples per column.
35const MAX_EXAMPLES: usize = 5;
36
37/// Relative difference under which two numbers count as equal.
38const NUMBER_TOLERANCE: f64 = 1e-3;
39
40/// Stances of `start_pos_diplomacy` that have a direction (the subject first).
41const DIRECTED_STANCES: [&str; 2] = ["vassal", "client_state"];
42
43/// Relationship of `start_pos_family_relationships` without direction.
44const SPOUSE_RELATIONSHIP: &str = "spouse";
45
46/// How each table is compared.
47const TABLE_SPECS: &[TableSpec] = &[
48    TableSpec { name: "start_pos_calendars", keys: &["campaign"], refs: &[], renumbered: &[] },
49    TableSpec { name: "start_pos_factions", keys: &["faction"], refs: &[], renumbered: &[] },
50    TableSpec { name: "start_pos_technologies", keys: &["faction", "technology"], refs: &[("faction", Reference::Faction)], renumbered: &[] },
51    TableSpec { name: "start_pos_victory_conditions", keys: &["start_pos_faction", "region", "victory_type"], refs: &[("start_pos_faction", Reference::Faction)], renumbered: &[] },
52    TableSpec { name: "start_pos_diplomacy", keys: &["faction1", "faction2"], refs: &[("faction1", Reference::Faction), ("faction2", Reference::Faction)], renumbered: &["key"] },
53    TableSpec { name: "start_pos_regions", keys: &["region"], refs: &[("owning_faction", Reference::OptionalFaction)], renumbered: &[] },
54    TableSpec { name: "start_pos_settlements", keys: &["region"], refs: &[("region", Reference::Region)], renumbered: &["id"] },
55    TableSpec { name: "start_pos_region_religions", keys: &["region", "religion"], refs: &[("region", Reference::Region)], renumbered: &[] },
56    TableSpec { name: "start_pos_region_slot_templates", keys: &["region", "slot_type"], refs: &[("region", Reference::RegionKey)], renumbered: &["id"] },
57    TableSpec { name: "start_pos_regions_to_unit_resources", keys: &["key", "resource"], refs: &[("key", Reference::Region)], renumbered: &[] },
58    TableSpec { name: "start_pos_characters", keys: &["ID"], refs: &[("ID", Reference::Character), ("faction", Reference::Faction)], renumbered: &[] },
59    TableSpec { name: "start_pos_character_traits", keys: &["character_id", "trait_level"], refs: &[("character_id", Reference::Character)], renumbered: &["id"] },
60    TableSpec { name: "start_pos_character_to_settlements", keys: &["character", "settlement"], refs: &[("character", Reference::Character), ("settlement", Reference::Settlement)], renumbered: &[] },
61    TableSpec { name: "start_pos_family_relationships", keys: &["character", "related_to", "relationship"], refs: &[("character", Reference::Character), ("related_to", Reference::Character)], renumbered: &[] },
62    TableSpec { name: "start_pos_land_units", keys: &["general", "unit_type"], refs: &[("general", Reference::Character)], renumbered: &["id"] },
63    TableSpec { name: "start_pos_naval_units", keys: &["admiral", "unit_type"], refs: &[("admiral", Reference::Character)], renumbered: &["ID"] },
64    TableSpec { name: "start_pos_horde_details", keys: &["general"], refs: &[("general", Reference::Character)], renumbered: &[] },
65];
66
67/// Columns of `start_pos_regions` the build sums over a province (fertility) or a faction's part of a province.
68const PROVINCE_SUMS: [&str; 1] = ["base_fertility"];
69const PROVINCE_PART_SUMS: [&str; 2] = ["development_points", "initial_public_order"];
70
71/// Highest public order of a faction's part of a province: the build clamps the sums to it, both ways.
72const MAX_PUBLIC_ORDER: i64 = 100;
73
74//---------------------------------------------------------------------------//
75//                              Types
76//---------------------------------------------------------------------------//
77
78/// How a table is compared.
79struct TableSpec {
80
81    /// Table name, without the `_tables` suffix.
82    name: &'static str,
83
84    /// Columns that identify a row, after resolving references. Rows with the same values are told apart by order.
85    keys: &'static [&'static str],
86
87    /// Columns with ids of other tables' rows, replaced by keys before comparing.
88    refs: &'static [(&'static str, Reference)],
89
90    /// Row id columns the extraction numbers anew, left out of the comparison.
91    renumbered: &'static [&'static str],
92}
93
94/// The kind of row an id column references.
95#[derive(Debug, Clone, Copy, PartialEq, Eq)]
96enum Reference {
97
98    /// A `start_pos_factions` row of the campaign. Rows referencing other factions aren't the campaign's.
99    Faction,
100
101    /// A `start_pos_factions` row of the campaign, or anything else for none (like rebel-owned regions).
102    OptionalFaction,
103
104    /// A `start_pos_regions` row of the campaign.
105    Region,
106
107    /// A region key of the campaign (not an id).
108    RegionKey,
109
110    /// A `start_pos_settlements` row of the campaign's regions.
111    Settlement,
112
113    /// A `start_pos_characters` row of the campaign's factions.
114    Character,
115}
116
117/// The keys of the rows of one table set that other rows reference, by row id.
118#[derive(Debug, Default)]
119struct References {
120    factions: HashMap<String, String>,
121    regions: HashMap<String, String>,
122    region_keys: HashSet<String>,
123    settlements: HashMap<String, String>,
124    characters: HashMap<String, String>,
125}
126
127/// How the rows of one table of the extraction compare with the original ones.
128#[derive(Debug, Clone, Default, Getters)]
129#[getset(get = "pub")]
130pub struct TableComparison {
131
132    /// Table name, without the `_tables` suffix.
133    table: String,
134
135    /// Rows found in both.
136    matched_rows: usize,
137
138    /// Keys of the original rows the extraction lacks.
139    missing_rows: Vec<String>,
140
141    /// Keys of the extracted rows the original tables lack.
142    extra_rows: Vec<String>,
143
144    /// How each compared column matches, in table order.
145    columns: Vec<ColumnComparison>,
146}
147
148/// How the values of one column compare in the matched rows.
149#[derive(Debug, Clone, Default, Getters)]
150#[getset(get = "pub")]
151pub struct ColumnComparison {
152
153    /// Column name.
154    column: String,
155
156    /// Matched rows with the same value.
157    equal: usize,
158
159    /// Matched rows with a different value.
160    different: usize,
161
162    /// Some of the differences: the row's key, the original value and the extracted one.
163    examples: Vec<(String, String, String)>,
164}
165
166//---------------------------------------------------------------------------//
167//                              Implementations
168//---------------------------------------------------------------------------//
169
170/// Compares the tables extracted from a startpos with the tables it was built from.
171///
172/// # Arguments
173///
174/// * `original` - The tables the startpos was built from. Rows of other campaigns are left out.
175/// * `extracted` - The tables extracted from the startpos.
176/// * `schema` - Schema of the game, to type the extracted values.
177///
178/// # Returns
179///
180/// The comparison of each extracted table.
181///
182/// # Errors
183///
184/// Returns an error if the extracted values don't fit the schema, or a table lacks a column.
185pub fn compare_tables(original: &StartposTables, extracted: &ExtractedTables, schema: &Schema) -> Result<Vec<TableComparison>> {
186    let campaign = extracted.campaign();
187    let extracted_tables = extracted.to_tables(schema)?;
188    let original_references = References::new(original, campaign)?;
189    let extracted_references = References::new(&extracted_tables, campaign)?;
190
191    let mut comparisons = vec![];
192    for spec in TABLE_SPECS {
193        let Some(rows) = extracted.table(spec.name) else { continue };
194        let columns = rows.columns().iter().copied().filter(|column| !spec.keys.contains(column) && !spec.renumbered.contains(column)).collect::<Vec<_>>();
195        let mut ours = canonical_rows(original.table(spec.name).ok(), spec, &original_references, campaign)?;
196        let mut theirs = canonical_rows(extracted_tables.table(spec.name).ok(), spec, &extracted_references, campaign)?;
197        if spec.name == "start_pos_regions" {
198            let provinces = region_provinces(original)?;
199            sum_by_province(&mut ours, &provinces);
200            sum_by_province(&mut theirs, &provinces);
201        }
202
203        let mut comparison = TableComparison {
204            table: spec.name.to_owned(),
205            columns: columns.iter().map(|column| ColumnComparison { column: column.to_string(), ..Default::default() }).collect(),
206            ..Default::default()
207        };
208        for (key, original_row) in &ours {
209            let Some(extracted_row) = theirs.get(key) else {
210                comparison.missing_rows.push(key.to_owned());
211                continue;
212            };
213            comparison.matched_rows += 1;
214            for column in &mut comparison.columns {
215                let original_value = original_row.get(&column.column).cloned().unwrap_or_default();
216                let extracted_value = extracted_row.get(&column.column).cloned().unwrap_or_default();
217                if same_value(&original_value, &extracted_value) {
218                    column.equal += 1;
219                } else {
220                    column.different += 1;
221                    if column.examples.len() < MAX_EXAMPLES {
222                        column.examples.push((key.to_owned(), original_value, extracted_value));
223                    }
224                }
225            }
226        }
227        comparison.extra_rows = theirs.keys().filter(|key| !ours.contains_key(*key)).cloned().collect();
228        comparisons.push(comparison);
229    }
230    Ok(comparisons)
231}
232
233impl TableComparison {
234
235    /// Returns the share of the original values the extraction got back: values of matched rows that are equal, over
236    /// the values of every original row.
237    pub fn match_ratio(&self) -> f64 {
238        let equal = self.columns.iter().map(|column| column.equal).sum::<usize>() + self.matched_rows;
239        let total = (self.matched_rows + self.missing_rows.len()) * (self.columns.len() + 1);
240        if total == 0 { 1.0 } else { equal as f64 / total as f64 }
241    }
242}
243
244impl fmt::Display for TableComparison {
245    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
246        writeln!(f, "{}: {} rows matched, {} missing, {} extra ({:.4} of the values)", self.table, self.matched_rows, self.missing_rows.len(), self.extra_rows.len(), self.match_ratio())?;
247        for key in self.missing_rows.iter().take(MAX_EXAMPLES) {
248            writeln!(f, "    missing row {key}")?;
249        }
250        for key in self.extra_rows.iter().take(MAX_EXAMPLES) {
251            writeln!(f, "    extra row {key}")?;
252        }
253        for column in self.columns.iter().filter(|column| column.different > 0) {
254            writeln!(f, "    {}: {} equal, {} different", column.column, column.equal, column.different)?;
255            for (key, original, extracted) in &column.examples {
256                writeln!(f, "        {key}: {original:?} -> {extracted:?}")?;
257            }
258        }
259        Ok(())
260    }
261}
262
263impl References {
264
265    /// Collects the keys of the campaign's factions, regions, settlements and characters in a table set.
266    fn new(tables: &StartposTables, campaign: &str) -> Result<Self> {
267        let mut references = Self::default();
268        if let Ok(factions) = tables.table("start_pos_factions") {
269            for row in factions.rows().filter(|row| in_campaign(row, campaign)) {
270                references.factions.insert(row.str("ID")?.into_owned(), row.str("faction")?.into_owned());
271            }
272        }
273        if let Ok(regions) = tables.table("start_pos_regions") {
274            for row in regions.rows().filter(|row| in_campaign(row, campaign)) {
275                references.regions.insert(row.str("id")?.into_owned(), row.str("region")?.into_owned());
276                references.region_keys.insert(row.str("region")?.into_owned());
277            }
278        }
279        if let Ok(settlements) = tables.table("start_pos_settlements") {
280            for row in settlements.rows() {
281                if let Some(region) = references.regions.get(row.str("region")?.as_ref()) {
282                    references.settlements.insert(row.str("id")?.into_owned(), region.to_owned());
283                }
284            }
285        }
286
287        // Characters with the same faction, first name, age and agent type are told apart by their order. Other
288        // names aren't part of it, as the game drops the ones that don't fit the faction.
289        if let Ok(characters) = tables.table("start_pos_characters") {
290            let mut seen = HashMap::<String, usize>::new();
291            for row in characters.rows() {
292                let Some(faction) = references.factions.get(row.str("faction")?.as_ref()) else { continue };
293                let signature = format!("{faction}/{}/{}/{}", row.str("Name")?, row.str("Age")?, row.str("Type")?);
294                let occurrence = seen.entry(signature.to_owned()).or_default();
295                *occurrence += 1;
296                references.characters.insert(row.str("ID")?.into_owned(), format!("{signature}#{occurrence}"));
297            }
298        }
299        Ok(references)
300    }
301
302    /// Returns the key of a referenced row, if it's one of the campaign's.
303    fn resolve(&self, reference: Reference, id: &str) -> Option<String> {
304        let map = match reference {
305            Reference::Faction | Reference::OptionalFaction => &self.factions,
306            Reference::Region => &self.regions,
307            Reference::RegionKey => return self.region_keys.get(id).map(|_| id.to_owned()),
308            Reference::Settlement => &self.settlements,
309            Reference::Character => &self.characters,
310        };
311        map.get(id).cloned()
312    }
313}
314
315/// Returns the campaign's rows of a table, with their references resolved, by key.
316fn canonical_rows(table: Option<&Table>, spec: &TableSpec, references: &References, campaign: &str) -> Result<BTreeMap<String, HashMap<String, String>>> {
317    let mut rows = BTreeMap::new();
318    let Some(table) = table else { return Ok(rows) };
319    let has_campaign = table.column("campaign").is_ok();
320    let mut seen = HashMap::<String, usize>::new();
321
322    'rows: for row in table.rows() {
323        if has_campaign && !in_campaign(&row, campaign) {
324            continue;
325        }
326
327        let mut values = HashMap::new();
328        for column in table.column_names() {
329            values.insert(column.to_owned(), row.str(column)?.into_owned());
330        }
331        for (column, reference) in spec.refs {
332            let id = values.get(*column).cloned().unwrap_or_default();
333            match (references.resolve(*reference, &id), reference) {
334                (Some(key), _) => { values.insert(column.to_string(), key); },
335                (None, Reference::OptionalFaction) => { values.insert(column.to_string(), String::new()); },
336                (None, _) => continue 'rows,
337            }
338        }
339        normalise(spec.name, &mut values);
340
341        let key = spec.keys.iter().map(|column| values.get(*column).map(String::as_str).unwrap_or_default()).collect::<Vec<_>>().join("|");
342        let occurrence = seen.entry(key.to_owned()).or_default();
343        *occurrence += 1;
344        let key = if *occurrence == 1 { key } else { format!("{key}#{occurrence}") };
345        rows.insert(key, values);
346    }
347    Ok(rows)
348}
349
350/// Puts the pairs without direction (diplomatic relationships between equals, spouses) in a fixed order, and packs
351/// horde buildings in the first columns, as the build does.
352fn normalise(table: &str, values: &mut HashMap<String, String>) {
353    if table == "start_pos_horde_details" {
354        let buildings = HORDE_BUILDING_COLUMNS.iter().filter_map(|column| values.get(*column).filter(|building| !building.is_empty()).cloned()).collect::<Vec<_>>();
355        for (index, column) in HORDE_BUILDING_COLUMNS.iter().enumerate() {
356            values.insert(column.to_string(), buildings.get(index).cloned().unwrap_or_default());
357        }
358        return;
359    }
360
361    let (first, second) = match table {
362        "start_pos_diplomacy" if !values.get("stance").is_some_and(|stance| DIRECTED_STANCES.contains(&stance.as_str())) => ("faction1", "faction2"),
363        "start_pos_family_relationships" if values.get("relationship").is_some_and(|relationship| relationship == SPOUSE_RELATIONSHIP) => ("character", "related_to"),
364        _ => return,
365    };
366    if values.get(first) > values.get(second) {
367        let first_value = values.remove(first).unwrap_or_default();
368        let second_value = values.remove(second).unwrap_or_default();
369        values.insert(first.to_owned(), second_value);
370        values.insert(second.to_owned(), first_value);
371    }
372}
373
374/// Returns the province of each region, by region key.
375fn region_provinces(tables: &StartposTables) -> Result<HashMap<String, String>> {
376    tables.table("region_to_province_junctions")?.rows()
377        .map(|row| Ok((row.str("region")?.into_owned(), row.str("province")?.into_owned())))
378        .collect()
379}
380
381/// Replaces the values the build sums over a province, or over a faction's part of it, with their sum, in every row
382/// of the group: the build only keeps the sums.
383fn sum_by_province(rows: &mut BTreeMap<String, HashMap<String, String>>, provinces: &HashMap<String, String>) {
384    let groups = |values: &HashMap<String, String>, column: &str| {
385        let province = values.get("region").and_then(|region| provinces.get(region)).cloned().unwrap_or_default();
386        if PROVINCE_SUMS.contains(&column) { province } else { format!("{province}|{}", values.get("owning_faction").cloned().unwrap_or_default()) }
387    };
388
389    for column in PROVINCE_SUMS.iter().chain(&PROVINCE_PART_SUMS) {
390        let mut sums = HashMap::<String, i64>::new();
391        for values in rows.values() {
392            let value = values.get(*column).and_then(|value| value.parse::<i64>().ok()).unwrap_or_default();
393            *sums.entry(groups(values, column)).or_default() += value;
394        }
395        for values in rows.values_mut() {
396            let mut sum = sums.get(&groups(values, column)).copied().unwrap_or_default();
397            if *column == "initial_public_order" {
398                sum = sum.clamp(-MAX_PUBLIC_ORDER, MAX_PUBLIC_ORDER);
399            }
400            values.insert(column.to_string(), sum.to_string());
401        }
402    }
403}
404
405/// Returns if a row with a `campaign` column belongs to the campaign. Rows without the column belong to every one.
406fn in_campaign(row: &Row, campaign: &str) -> bool {
407    row.str("campaign").map(|value| value == campaign).unwrap_or(true)
408}
409
410/// Returns if two values are the same: equal as text, or as numbers within [`NUMBER_TOLERANCE`].
411fn same_value(original: &str, extracted: &str) -> bool {
412    if original == extracted {
413        return true;
414    }
415    match (original.parse::<f64>(), extracted.parse::<f64>()) {
416        (Ok(original), Ok(extracted)) => (original - extracted).abs() <= NUMBER_TOLERANCE * original.abs().max(1.0),
417        _ => false,
418    }
419}
rpfm_extensions/startpos/extract/compare.rs

rpfm_extensions/startpos/extract/
compare.rs
