rpfm_extensions/startpos/extract/
mod.rs

1//---------------------------------------------------------------------------//
2// Copyright (c) 2017-2026 Ismael Gutiérrez González. All rights reserved.
3//
4// This file is part of the Rusted PackFile Manager (RPFM) project,
5// which can be found here: https://github.com/Frodo45127/rpfm.
6//
7// This file is licensed under the MIT license, which can be found here:
8// https://github.com/Frodo45127/rpfm/blob/master/LICENSE.
9//---------------------------------------------------------------------------//
10
11//! Extraction of a campaign's `start_pos_*` tables from its startpos: the inverse of the build.
12//!
13//! The extractor reads a startpos through the record layouts and rebuilds the rows it was built from, so a startpos
14//! without its tables can be edited again, and a generated startpos can be checked against the tables it came from
15//! (see [`compare`]).
16//!
17//! The build doesn't keep everything, so the extracted rows are the ones that build the same startpos:
18//!
19//! - Columns the build ignores (descriptions, multiplayer counts...) keep the table's default value.
20//! - A province's fertility, public order and development are sums over its regions: the sum goes to one region.
21//! - Horde buildings fill the first building columns, and garrison and bodyguard units aren't rows.
22//! - Faction and region ids are kept (the startpos has them in localisation keys). Other row ids are numbered from 1.
23//! - Lists the game stores in hash order (units, traits, technologies...) come out in startpos order.
24//!
25//! Besides the tables, the extraction gets back the victory objectives (`victory_objectives.txt`), which the build
26//! also reads. The settlements listed by loot and raze objectives aren't kept.
27//!
28//! # Submodules
29//!
30//! - [`compare`]: comparison of extracted tables with the tables a startpos was built from.
31
32use getset::Getters;
33
34use std::collections::{BTreeMap, HashMap};
35
36use rpfm_lib::error::{RLibError, Result};
37use rpfm_lib::files::db::DB;
38use rpfm_lib::files::esf::{NodeType, RecordNode, ESF};
39use rpfm_lib::files::table::DecodedData;
40use rpfm_lib::schema::Schema;
41
42use super::cai::CAI_WORLD_PATH;
43use super::layout::{child_path, GroupView, StartposLayout};
44use super::missions::{write_victory_objectives, VictoryObjectives};
45use super::tables::StartposTables;
46use super::world::writer::WORLD_PATH;
47
48pub mod compare;
49mod characters;
50mod factions;
51mod missions;
52mod regions;
53
54#[cfg(test)] pub(crate) mod extract_test;
55
56/// Path of the root record.
57const ROOT_PATH: &str = "CAMPAIGN_STARTPOS";
58
59/// Path of the pre-campaign map info.
60const PREOPEN_PATH: &str = "CAMPAIGN_STARTPOS/CAMPAIGN_PREOPEN_MAP_INFO";
61
62/// Path of the campaign environment.
63const ENV_PATH: &str = "CAMPAIGN_STARTPOS/CAMPAIGN_ENV";
64
65/// Path of the campaign model.
66const MODEL_PATH: &str = "CAMPAIGN_STARTPOS/CAMPAIGN_ENV/CAMPAIGN_MODEL";
67
68/// Path of a faction.
69const FACTION_PATH: &str = "CAMPAIGN_STARTPOS/CAMPAIGN_ENV/CAMPAIGN_MODEL/WORLD/FACTION_ARRAY/FACTION";
70
71/// Game tables the extraction reads, besides the startpos, without the `_tables` suffix.
72pub const EXTRACTION_TABLES: [&str; 2] = ["character_trait_levels", "seasons"];
73
74/// Localisation key prefix of the faction descriptions, followed by the faction's `start_pos_factions` id.
75const DESCRIPTION_KEY_PREFIX: &str = "start_pos_factions_description_";
76
77//---------------------------------------------------------------------------//
78//                              Types
79//---------------------------------------------------------------------------//
80
81/// The `start_pos_*` rows of a campaign, extracted from its startpos.
82#[derive(Debug, Clone, Default, Getters)]
83pub struct ExtractedTables {
84
85    /// Campaign key.
86    #[getset(get = "pub")]
87    campaign: String,
88
89    /// Rows of each table, by table name (without the `_tables` suffix).
90    tables: BTreeMap<String, TableRows>,
91
92    /// Victory objectives of each faction and kind of campaign.
93    #[getset(get = "pub")]
94    victory_objectives: Vec<VictoryObjectives>,
95}
96
97/// Rows of a table, with their values as text.
98#[derive(Debug, Clone, Default, PartialEq, Getters)]
99#[getset(get = "pub")]
100pub struct TableRows {
101
102    /// Names of the columns the rows have. Other columns keep their default value.
103    columns: Vec<&'static str>,
104
105    /// Rows, with their values in `columns` order.
106    rows: Vec<Vec<String>>,
107}
108
109/// What the extraction reads from a startpos, read once.
110struct Source<'a> {
111
112    /// Record layouts of the startpos.
113    layout: &'a StartposLayout,
114
115    /// Campaign key.
116    campaign: String,
117
118    /// The pre-campaign map info (`CAMPAIGN_PREOPEN_MAP_INFO`).
119    preopen: GroupView<'a>,
120
121    /// The calendar (`CAMPAIGN_CALENDAR`).
122    calendar: GroupView<'a>,
123
124    /// The calendar's start date.
125    start_date: GroupView<'a>,
126
127    /// Factions, in `FACTION_ARRAY` order.
128    factions: Vec<FactionSource<'a>>,
129
130    /// Regions (`REGION` groups), in `REGIONS_ARRAY` order.
131    regions: Vec<GroupView<'a>>,
132
133    /// Provinces (`PROVINCE` groups), in `PROVINCE_ARRAY` order.
134    provinces: Vec<GroupView<'a>>,
135
136    /// The world, for its singletons (family tree, slaves).
137    world: GroupView<'a>,
138
139    /// The AI's factions (`CAI_FACTION` groups).
140    cai_factions: Vec<GroupView<'a>>,
141}
142
143/// A faction of the startpos.
144struct FactionSource<'a> {
145
146    /// The `FACTION` group.
147    group: GroupView<'a>,
148
149    /// The faction's pre-campaign info (`FACTION_INFOS` group), if it has one.
150    info: Option<GroupView<'a>>,
151
152    /// Runtime id.
153    id: u32,
154
155    /// Faction key.
156    key: String,
157
158    /// Row id in `start_pos_factions`.
159    table_id: String,
160}
161
162/// Numbers the rows whose ids the startpos doesn't keep.
163#[derive(Debug, Default)]
164struct Ids {
165    last: u64,
166}
167
168//---------------------------------------------------------------------------//
169//                              Implementations
170//---------------------------------------------------------------------------//
171
172/// Extracts the `start_pos_*` tables of a campaign from its startpos.
173///
174/// # Arguments
175///
176/// * `startpos` - The startpos.
177/// * `layout` - Record layouts of the game (the ones the startpos is written with).
178/// * `tables` - Game tables with the [`EXTRACTION_TABLES`].
179///
180/// # Returns
181///
182/// The rows of every table the startpos keeps something of, and the victory objectives.
183///
184/// # Errors
185///
186/// Returns an error if the startpos doesn't fit the layouts, or a table is missing.
187pub fn extract_tables(startpos: &ESF, layout: &StartposLayout, tables: &StartposTables) -> Result<ExtractedTables> {
188    let source = Source::read(startpos, layout)?;
189    let mut ids = Ids::default();
190    let mut extracted = ExtractedTables { campaign: source.campaign.to_owned(), tables: BTreeMap::new(), victory_objectives: missions::extract(&source)? };
191
192    extracted.insert("start_pos_calendars", factions::calendar(&source, tables)?);
193    extracted.insert("start_pos_factions", factions::factions(&source)?);
194    extracted.insert("start_pos_technologies", factions::technologies(&source)?);
195    extracted.insert("start_pos_victory_conditions", factions::victory_conditions(&source));
196    extracted.insert("start_pos_diplomacy", factions::diplomacy(&source, &mut ids)?);
197
198    let regions = regions::extract(&source, &mut ids)?;
199    let characters = characters::extract(&source, tables, &regions, &mut ids)?;
200    for (name, rows) in regions.tables.into_iter().chain(characters) {
201        extracted.insert(name, rows);
202    }
203    Ok(extracted)
204}
205
206impl ExtractedTables {
207
208    /// Returns the rows of a table, by name (without the `_tables` suffix).
209    pub fn table(&self, name: &str) -> Option<&TableRows> {
210        self.tables.get(name)
211    }
212
213    /// Returns the tables, by name (without the `_tables` suffix).
214    pub fn tables(&self) -> impl Iterator<Item = (&str, &TableRows)> {
215        self.tables.iter().map(|(name, rows)| (name.as_str(), rows))
216    }
217
218    /// Returns the victory objectives as a `victory_objectives.txt` script.
219    pub fn victory_objectives_script(&self) -> String {
220        write_victory_objectives(&self.victory_objectives)
221    }
222
223    /// Builds a DB file per table, with the newest definition of the schema. Columns the extraction doesn't fill keep
224    /// their default value, and columns the definition lacks are left out.
225    ///
226    /// # Arguments
227    ///
228    /// * `schema` - Schema of the game.
229    ///
230    /// # Errors
231    ///
232    /// Returns an error if the schema has no definition for a table, or a value doesn't fit its column's type.
233    pub fn to_dbs(&self, schema: &Schema) -> Result<Vec<DB>> {
234        self.tables.iter().map(|(name, rows)| {
235            let table_name = format!("{name}_tables");
236            let definition = schema.definitions_by_table_name(&table_name)
237                .and_then(|definitions| definitions.iter().max_by_key(|definition| *definition.version()))
238                .ok_or_else(|| extraction_error(&format!("the schema has no definition for {table_name}")))?;
239
240            let mut db = DB::new(definition, None, &table_name);
241            let fields = definition.fields_processed();
242            let positions = rows.columns.iter()
243                .map(|column| fields.iter().position(|field| field.name() == *column))
244                .collect::<Vec<_>>();
245
246            let mut data = Vec::with_capacity(rows.rows.len());
247            for row in &rows.rows {
248                let mut new_row = db.new_row();
249                for (position, value) in positions.iter().zip(row) {
250                    if let Some(position) = *position {
251                        new_row[position] = DecodedData::new_from_type_and_string(fields[position].field_type(), value)?;
252                    }
253                }
254                data.push(new_row);
255            }
256            db.set_data(&data)?;
257            Ok(db)
258        }).collect()
259    }
260
261    /// Builds a table set from the extracted rows, typed as the schema's newest definitions (see [`Self::to_dbs`]).
262    ///
263    /// # Errors
264    ///
265    /// Returns an error if the schema has no definition for a table, or a value doesn't fit its column's type.
266    pub fn to_tables(&self, schema: &Schema) -> Result<StartposTables> {
267        let mut tables = StartposTables::new();
268        for db in self.to_dbs(schema)? {
269            tables.add_db(&db)?;
270        }
271        Ok(tables)
272    }
273
274    /// Adds the rows of a table.
275    fn insert(&mut self, name: &str, rows: TableRows) {
276        self.tables.insert(name.to_owned(), rows);
277    }
278}
279
280impl TableRows {
281
282    /// Creates an empty table with the given columns.
283    fn new(columns: &[&'static str]) -> Self {
284        Self { columns: columns.to_vec(), rows: vec![] }
285    }
286
287    /// Adds a row, with its values in column order.
288    fn push(&mut self, row: Vec<String>) {
289        debug_assert_eq!(row.len(), self.columns.len(), "a row of the extracted tables doesn't fit its columns");
290        self.rows.push(row);
291    }
292}
293
294impl<'a> Source<'a> {
295
296    /// Reads the parts of a startpos the extraction needs.
297    fn read(startpos: &'a ESF, layout: &'a StartposLayout) -> Result<Self> {
298        let NodeType::Record(root) = startpos.root_node() else { return Err(extraction_error("the startpos has no root record")) };
299        let root = layout.read_single(ROOT_PATH, root)?;
300        let preopen = layout.read_single(PREOPEN_PATH, required(&root, "CAMPAIGN_PREOPEN_MAP_INFO")?)?;
301        let env = layout.read_single(ENV_PATH, required(&root, "CAMPAIGN_ENV")?)?;
302        let model = layout.read_single(MODEL_PATH, required(&env, "CAMPAIGN_MODEL")?)?;
303        let world = layout.read_single(WORLD_PATH, required(&model, "WORLD")?)?;
304
305        let calendar_path = child_path(MODEL_PATH, "CAMPAIGN_CALENDAR");
306        let calendar = layout.read_single(&calendar_path, required(&model, "CAMPAIGN_CALENDAR")?)?;
307        let start_date = layout.read_single(&child_path(&calendar_path, "DATE"), required(&calendar, "DATE")?)?;
308
309        let infos_path = child_path(PREOPEN_PATH, "FACTION_INFOS");
310        let infos = match preopen.record("FACTION_INFOS") {
311            Some(record) => layout.read(&infos_path, record)?,
312            None => vec![],
313        };
314        let mut infos = infos.into_iter()
315            .filter_map(|info| Some((info.str("^+1")?.to_owned(), info)))
316            .collect::<HashMap<_, _>>();
317
318        let factions = layout.read_items(&child_path(WORLD_PATH, "FACTION_ARRAY"), &world, "FACTION_ARRAY", "FACTION")?.into_iter()
319            .map(|(_, group)| {
320                let key = group.str("^+2").unwrap_or_default().to_owned();
321                let info = infos.remove(&key);
322                let table_id = info.as_ref()
323                    .and_then(|info| info.str("^+5"))
324                    .and_then(|description| description.strip_prefix(DESCRIPTION_KEY_PREFIX))
325                    .unwrap_or_default()
326                    .to_owned();
327                FactionSource { id: group.u32("^+1").unwrap_or_default(), group, info, key, table_id }
328            })
329            .collect();
330
331        let region_manager_path = child_path(WORLD_PATH, "REGION_MANAGER");
332        let region_manager = layout.read_single(&region_manager_path, required(&world, "REGION_MANAGER")?)?;
333        let regions = layout.read_items(&child_path(&region_manager_path, "REGIONS_ARRAY"), &region_manager, "REGIONS_ARRAY", "REGION")?.into_iter()
334            .map(|(_, region)| region)
335            .collect();
336
337        let province_manager_path = child_path(WORLD_PATH, "PROVINCE_MANAGER");
338        let province_manager = layout.read_single(&province_manager_path, required(&world, "PROVINCE_MANAGER")?)?;
339        let provinces = layout.read_items(&child_path(&province_manager_path, "PROVINCE_ARRAY"), &province_manager, "PROVINCE_ARRAY", "PROVINCE")?.into_iter()
340            .map(|(_, province)| province)
341            .collect();
342
343        let cai_interface_path = child_path(MODEL_PATH, "CAI_INTERFACE");
344        let mut cai_factions = vec![];
345        if let Some(interface) = model.record("CAI_INTERFACE") {
346            let interface = layout.read_single(&cai_interface_path, interface)?;
347            if let Some(cai_world) = interface.record("CAI_WORLD") {
348                let cai_world = layout.read_single(CAI_WORLD_PATH, cai_world)?;
349                let factions_path = child_path(CAI_WORLD_PATH, "CAI_WORLD_FACTIONS");
350                cai_factions = layout.read_items(&factions_path, &cai_world, "CAI_WORLD_FACTIONS", "CAI_FACTION")?.into_iter()
351                    .map(|(_, faction)| faction)
352                    .collect();
353            }
354        }
355
356        Ok(Self {
357            layout,
358            campaign: preopen.str("DATE+1").unwrap_or_default().to_owned(),
359            preopen,
360            calendar,
361            start_date,
362            factions,
363            regions,
364            provinces,
365            world,
366            cai_factions,
367        })
368    }
369
370    /// Returns the start year.
371    fn start_year(&self) -> u32 {
372        self.start_date.u32("^+1").unwrap_or_default()
373    }
374
375    /// Returns a faction by runtime id.
376    fn faction(&self, id: u32) -> Option<&FactionSource<'a>> {
377        self.factions.iter().find(|faction| faction.id == id)
378    }
379
380    /// Returns a faction by key.
381    fn faction_by_key(&self, key: &str) -> Option<&FactionSource<'a>> {
382        self.factions.iter().find(|faction| faction.key == key)
383    }
384
385    /// Reads the only group of a child record of a faction.
386    fn faction_record(&self, faction: &FactionSource<'a>, key: &str) -> Result<Option<GroupView<'a>>> {
387        faction.group.record(key)
388            .map(|record| self.layout.read_single(&child_path(FACTION_PATH, key), record))
389            .transpose()
390    }
391}
392
393impl Ids {
394
395    /// Returns a new id.
396    fn next(&mut self) -> String {
397        self.last += 1;
398        self.last.to_string()
399    }
400}
401
402/// Returns a child record of a group, or an error naming it if the group lacks it.
403fn required<'a>(group: &GroupView<'a>, key: &str) -> Result<&'a RecordNode> {
404    group.record(key).ok_or_else(|| extraction_error(&format!("missing {key}")))
405}
406
407/// Builds the error for a startpos that lacks something.
408fn extraction_error(message: &str) -> RLibError {
409    RLibError::StartposExtractionError(message.to_owned())
410}
rpfm_extensions/startpos/extract/mod.rs

rpfm_extensions/startpos/extract/
mod.rs
