rpfm_lib/files/db/

mod.rs

//---------------------------------------------------------------------------//
// Copyright (c) 2017-2026 Ismael Gutiérrez González. All rights reserved.
//
// This file is part of the Rusted PackFile Manager (RPFM) project,
// which can be found here: https://github.com/Frodo45127/rpfm.
//
// This file is licensed under the MIT license, which can be found here:
// https://github.com/Frodo45127/rpfm/blob/master/LICENSE.
//---------------------------------------------------------------------------//

//! Database table files for Total War game data.
//!
//! DB files are the primary data storage format in Total War games, containing game data
//! organized into tables similar to relational database tables. Each table stores structured
//! data like units, buildings, technologies, and campaign settings.
//!
//! # Overview
//!
//! DB tables are binary files with a sequential format requiring a schema definition to decode.
//! The definition specifies the column types and order, and tables can be versioned to handle
//! format changes across game updates.
//!
//! Key characteristics:
//! - **Schema-dependent**: Requires a definition from the schema to decode
//! - **Versioned**: Optional version number in header for format evolution
//! - **Fragmented**: Tables can be split across multiple files
//! - **Binary**: Tightly packed binary format for efficient storage
//!
//! # Table Structure
//!
//! Each DB file consists of a header followed by row data. The header contains metadata
//! like version, GUID (for some games), and row count. The data section is a sequence
//! of rows matching the table definition.
//!
//! # Schema Definitions
//!
//! To decode a DB table, you need:
//! 1. **Table name**: Identifies which definition to use (e.g., `"units_tables"`)
//! 2. **Schema**: Contains definitions for all known table versions
//! 3. **Version**: Optional version number from the header (defaults to 0)
//!
//! See the [`Schema`](crate::schema) module for details on definitions and the
//! [`Table`](crate::files::table) module for the table data structure.
//!
//! # DB Structure
//!
//! ## Header
//!
//! | Bytes  | Type            | Data                                                         |
//! | ------ | --------------- | ------------------------------------------------------------ |
//! | 4      | &\[[u8]\]       | GUID Marker. Optional.                                       |
//! | 2 + 72 | Sized StringU16 | GUID. Only present if GUID Marker is present too.            |
//! | 4      | &\[[u8]\]       | Version Marker. Optional.                                    |
//! | 4      | [u32]           | Version of the table. Only present if Version Marker is too. |
//! | 1      | [bool]          | Unknown. Probably a bool because it's always either 0 or 1.  |
//! | 4      | [u32]           | Amount of entries on the table.                              |
//!
//! ## Data
//!
//! The data structure depends on the definition of the table.

use csv::{StringRecordsIter, Writer};
use getset::Getters;
use itertools::Itertools;
#[cfg(feature = "integration_sqlite")]use r2d2::Pool;
#[cfg(feature = "integration_sqlite")]use r2d2_sqlite::SqliteConnectionManager;
use rayon::prelude::*;
use serde_derive::{Serialize, Deserialize};
use uuid::Uuid;

use std::borrow::Cow;
use std::collections::{HashMap, HashSet};
use std::fs::File;
use std::io::SeekFrom;

use crate::binary::{ReadBytes, WriteBytes};
use crate::error::{RLibError, Result};
use crate::files::{Container, ContainerPath, DecodeableExtraData, Decodeable, EncodeableExtraData, Encodeable, FileType, table::{decode_table, DecodedData, local::TableInMemory, Table}, pack::Pack, RFileDecoded};
use crate::schema::{Definition, DefinitionPatch, Field, Schema};
use crate::utils::check_size_mismatch;

/// If this sequence is found, the DB Table has a GUID after it.
const GUID_MARKER: &[u8] = &[253, 254, 252, 255];

/// If this sequence is found, the DB Table has a version number after it.
const VERSION_MARKER: &[u8] = &[252, 253, 254, 255];

#[cfg(test)] mod db_test;

//---------------------------------------------------------------------------//
//                              Enum & Structs
//---------------------------------------------------------------------------//

/// In-memory representation of a decoded DB table.
///
/// Holds a complete DB table including header metadata and row data. The table data
/// is stored as a [`TableInMemory`] which provides access to rows and columns.
///
/// # Fields
///
/// * `mysterious_byte` - Boolean flag of unknown purpose (observed as `0` or `1`)
/// * `guid` - Globally Unique Identifier for this table instance
/// * `table` - The actual table data with definition and rows
///
/// # Getters
///
/// Fields have public getters via the `getset` crate:
/// - `mysterious_byte()` - Get the mysterious byte value
/// - `guid()` - Get the table's GUID
/// - `table()` - Get reference to the table data
///
/// # The Mysterious Byte
///
/// The purpose of this byte is unknown, but it appears in all DB tables. Observed values
/// are `0` or `1` (interpreted as boolean). In Warhammer 2, a value of `0` can cause
/// crashes when tables are loaded by the game.
///
/// # GUID Handling
///
/// GUIDs are only present in some games (e.g., Warhammer series). Older games like
/// Napoleon and Empire don't use GUIDs, and adding them can crash those games.
/// The encoding process respects the game's GUID requirements.
///
/// # Example
///
/// ```ignore
/// use rpfm_lib::files::{Decodeable, db::DB, DecodeableExtraData, table::Table};
/// use rpfm_lib::schema::Schema;
/// use std::io::Cursor;
///
/// # let schema = Schema::default();
/// # let table_data = vec![];
/// let mut extra = DecodeableExtraData::default();
/// extra.set_schema(Some(&schema));
/// extra.set_table_name(Some("units_tables"));
///
/// let mut reader = Cursor::new(table_data);
/// let db = DB::decode(&mut reader, &Some(extra)).unwrap();
///
/// // Access table data
/// let row_count = db.table().len();
/// ```
#[derive(PartialEq, Clone, Debug, Getters, Serialize, Deserialize)]
#[getset(get = "pub")]
pub struct DB {

    /// Boolean flag of unknown purpose (always `0` or `1`).
    ///
    /// In Warhammer 2, a value of `0` can crash the game when loading tables.
    mysterious_byte: bool,

    /// Globally Unique Identifier for this table instance.
    ///
    /// Only present in newer games. Empty string for games without GUID support.
    guid: String,

    /// The table data including definition and rows.
    table: TableInMemory,
}

//---------------------------------------------------------------------------//
//                           Implementation of DB
//---------------------------------------------------------------------------//

impl Decodeable for DB {

    fn decode<R: ReadBytes>(data: &mut R, extra_data: &Option<DecodeableExtraData>) -> Result<Self> {
        let extra_data = extra_data.as_ref().ok_or(RLibError::DecodingMissingExtraData)?;
        let schema = extra_data.schema.ok_or_else(|| RLibError::DecodingMissingExtraDataField("schema".to_owned()))?;
        let table_name = extra_data.table_name.ok_or_else(|| RLibError::DecodingMissingExtraDataField("table_name".to_owned()))?;
        let return_incomplete = extra_data.return_incomplete;

        let (version, mysterious_byte, guid, entry_count) = Self::read_header(data)?;

        // Try to get the table_definition for this table, if exists.
        let definitions = schema.definitions_by_table_name(table_name).ok_or({
            if entry_count == 0 {
                RLibError::DecodingDBNoDefinitionsFoundAndEmptyFile
            } else {
                RLibError::DecodingDBNoDefinitionsFound
            }
        })?;

        // Try to decode the table.
        let len = data.len()?;
        let table = if version == 0 {
            let mut altered = false;
            let index_reset = data.stream_position()?;

            // For version 0 tables, get all definitions between 0 and -99, and get the first one that works.
            let mut working_definition = Err(RLibError::DecodingDBNoDefinitionsFound);
            for definition in definitions.iter().filter(|definition| *definition.version() < 1) {

                // First, reset the index in case it was changed in a previous iteration.
                // Then, check if the definition works.
                data.seek(SeekFrom::Start(index_reset))?;
                let db = decode_table(data, definition, Some(entry_count), return_incomplete, &mut altered);
                if db.is_ok() && data.stream_position()? == len {
                    working_definition = Ok(definition);
                    break;
                }
            }

            let definition = working_definition?;
            let definition_patch = schema.patches_for_table(table_name).cloned().unwrap_or_default();

            // Reset the index before the table, and now decode the table with proper backend support.
            data.seek(SeekFrom::Start(index_reset))?;
            TableInMemory::decode(data, definition, &definition_patch, Some(entry_count), return_incomplete, table_name)?
        }

        // For +0 versions, we expect unique definitions.
        else {

            let definition = definitions.iter()
                .find(|definition| *definition.version() == version)
                .ok_or(RLibError::DecodingDBNoDefinitionsFound)?;

            let definition_patch = schema.patches_for_table(table_name).cloned().unwrap_or_default();
            TableInMemory::decode(data, definition, &definition_patch, Some(entry_count), return_incomplete, table_name)?
        };

        // If we are not in the last byte, it means we didn't parse the entire file, which means this file is corrupt, or the decoding failed and we bailed early.
        //
        // If we have return_incomplete enabled, we pass whatever we got decoded into this error.
        check_size_mismatch(data.stream_position()? as usize, len as usize).map_err(|error| {
            RLibError::DecodingTableIncomplete(error.to_string(), Box::new(table.clone()))
        })?;

        // If we've reached this, we've successfully decoded the table.
        Ok(Self {
            mysterious_byte,
            guid,
            table,
        })
    }
}

impl Encodeable for DB {

    fn encode<W: WriteBytes>(&mut self, buffer: &mut W, extra_data: &Option<EncodeableExtraData>) -> Result<()> {
        let table_has_guid = if let Some (ref extra_data) = extra_data { extra_data.table_has_guid } else { false };
        let regenerate_table_guid = if let Some (ref extra_data) = extra_data { extra_data.regenerate_table_guid } else { false };

        // Napoleon and Empire do not have GUID, and adding it to their tables crash both games.
        // So for those two games, remember that you have to ignore the GUID_MARKER and the GUID itself.
        if table_has_guid {
            buffer.write_all(GUID_MARKER)?;
            if regenerate_table_guid || self.guid.is_empty() {
                buffer.write_sized_string_u16(&Uuid::new_v4().to_string())?;
            } else {
                buffer.write_sized_string_u16(&self.guid)?;
            }
        }

        // Only put version numbers on tables with an actual version.
        if *self.table.definition().version() > 0 {
            buffer.write_all(VERSION_MARKER)?;
            buffer.write_i32(*self.table.definition().version())?;
        }

        buffer.write_bool(self.mysterious_byte)?;
        buffer.write_u32(self.table.len() as u32)?;

        self.table.encode(buffer)
    }
}

impl DB {

    /// Creates a new empty DB table with the specified definition.
    ///
    /// Initializes a DB table with no rows but with the structure defined by the provided
    /// definition. The mysterious byte is set to `true` (safe default) and the GUID is empty.
    ///
    /// # Arguments
    ///
    /// * `definition` - Schema definition specifying column types and structure
    /// * `definition_patch` - Optional patches to modify the definition
    /// * `table_name` - Name of the table (for internal tracking)
    ///
    /// # Returns
    ///
    /// A new empty DB table ready to have rows added.
    ///
    /// # Example
    ///
    /// ```ignore
    /// use rpfm_lib::files::{db::DB, table::Table};
    /// use rpfm_lib::schema::Definition;
    ///
    /// # let definition = Definition::default();
    /// let db = DB::new(&definition, None, "units_tables");
    /// assert_eq!(db.table().len(), 0);
    /// ```
    pub fn new(definition: &Definition, definition_patch: Option<&DefinitionPatch>, table_name: &str) -> Self {
        let table = TableInMemory::new(definition, definition_patch, table_name);

        Self {
            mysterious_byte: true,
            guid: String::new(),
            table,
        }
    }

    /// Decodes the header section of a DB table.
    ///
    /// Reads the header bytes to extract metadata without decoding the full table data.
    /// This is useful for inspecting table properties before committing to a full decode.
    ///
    /// # Header Format
    ///
    /// The header contains optional and required fields:
    /// 1. **Optional GUID Marker** (`0xFD 0xFE 0xFC 0xFF`) + 2-byte sized UTF-16 string
    /// 2. **Optional Version Marker** (`0xFC 0xFD 0xFE 0xFF`) + 4-byte signed integer
    /// 3. **Mysterious Byte** (1 byte boolean)
    /// 4. **Entry Count** (4-byte unsigned integer)
    ///
    /// # Arguments
    ///
    /// * `data` - Reader positioned at the start of the DB table
    ///
    /// # Returns
    ///
    /// A tuple containing:
    /// - `version` - Table version number (0 if no version marker present)
    /// - `mysterious_byte` - Unknown boolean flag
    /// - `guid` - Table GUID (empty string if no GUID marker present)
    /// - `entry_count` - Number of rows in the table
    ///
    /// # Errors
    ///
    /// Returns [`RLibError::DecodingDBNotADBTable`] if:
    /// - The data is less than 5 bytes (minimum valid header size)
    /// - The data doesn't conform to the DB header format
    ///
    /// # Side Effects
    ///
    /// After reading, the reader is positioned at the start of the table data section.
    pub fn read_header<R: ReadBytes>(data: &mut R) -> Result<(i32, bool, String, u32)> {

        // 5 is the minimum amount of bytes a valid DB Table can have. If there is less, either the table is broken,
        // or the data is not from a DB Table.
        if data.len()? < 5 {
            return Err(RLibError::DecodingDBNotADBTable);
        }

        // If there is a GUID_MARKER, get the GUID and store it. If not, just store an empty string.
        let guid = if data.read_slice(4, false)? == GUID_MARKER {
            data.read_sized_string_u16()?
        } else {
            data.seek(SeekFrom::Current(-4))?;
            String::new()
        };

        // If there is a VERSION_MARKER, we get the version (4 bytes for the marker, 4 for the version).
        // Otherwise, we default to version 0.
        let version = if data.read_slice(4, false)? == VERSION_MARKER {
            data.read_i32()?
        } else {
            data.seek(SeekFrom::Current(-4))?;
            0
        };

        // We get the rest of the data from the header.
        let mysterious_byte = data.read_bool()?;
        let entry_count = data.read_u32()?;
        Ok((version, mysterious_byte, guid, entry_count))
    }

    /// Returns the schema definition for this DB table.
    pub fn definition(&self) -> &Definition {
        self.table.definition()
    }

    /// Returns the definition patches applied to this DB table.
    pub fn patches(&self) -> &DefinitionPatch {
        self.table.patches()
    }

    /// Returns the table name (e.g., `"units_tables"`).
    pub fn table_name(&self) -> &str {
        self.table.table_name()
    }

    /// Returns the table name without the `"_tables"` suffix.
    ///
    /// # Panics
    ///
    /// Panics if the table name doesn't end with `"_tables"`.
    pub fn table_name_without_tables(&self) -> String {

        // Note: it needs this check because this explodes if instead of "_tables" we have non-ascii characters.
        if self.table_name().ends_with("_tables") {
            self.table_name().to_owned().drain(..self.table_name().len() - 7).collect()
        } else {
            panic!("Either the code is broken, or someone with a few loose screws has renamed the fucking table folder. Crash for now, may return an error in the future.")
        }
    }

    /// Returns the table rows as a slice of decoded data.
    pub fn data(&'_ self) -> Cow<'_, [Vec<DecodedData>]> {
        self.table.data()
    }

    /// Loads table data from a SQLite database, replacing current contents.
    #[cfg(feature = "integration_sqlite")]
    pub fn sql_to_db(&mut self, pool: &Pool<SqliteConnectionManager>, pack_name: &str, file_name: &str) -> Result<()> {
        self.table.sql_to_db(pool, pack_name, file_name)
    }

    /// Returns a mutable reference to the table rows.
    ///
    /// Ensure modifications maintain valid structure matching the definition.
    pub fn data_mut(&mut self) -> &mut Vec<Vec<DecodedData>> {
        self.table.data_mut()
    }

    /// Replaces all table data with the provided rows.
    ///
    /// # Errors
    ///
    /// Returns an error if rows don't match the table definition structure.
    pub fn set_data(&mut self, data: &[Vec<DecodedData>]) -> Result<()> {
        self.table.set_data(data)
    }

    /// Overwrites this table's GUID with the provided one.
    pub fn set_guid(&mut self, guid: String) {
        self.guid = guid;
    }

    /// Creates a new row with default values from the table definition.
    pub fn new_row(&self) -> Vec<DecodedData> {
        self.table().new_row()
    }

    /// Returns the column index for a given column name, or `None` if not found.
    pub fn column_position_by_name(&self, column_name: &str) -> Option<usize> {
        self.table.column_position_by_name(column_name)
    }

    /// Returns the number of rows in the table.
    pub fn len(&self) -> usize {
        self.table.len()
    }

    /// Returns `true` if the table has no rows.
    pub fn is_empty(&self) -> bool {
        self.table.is_empty()
    }

    /// Replaces the table definition and migrates existing data to match.
    ///
    /// Use this to update tables to a newer schema version. Data is converted
    /// between compatible types where possible.
    pub fn set_definition(&mut self, new_definition: &Definition) {
        self.table.set_definition(new_definition);
    }

    /// Alias for [`set_definition`](Self::set_definition).
    pub fn update(&mut self, new_definition: &Definition) {
        self.set_definition(new_definition)
    }

    /// Performs a cascade update of a value across all referencing tables in a Pack.
    ///
    /// When a key field value is changed, this function finds all tables that reference
    /// that field and updates them accordingly. It also updates corresponding Loc entries
    /// if the edited field affects localisation keys.
    ///
    /// # Arguments
    ///
    /// * `pack` - The Pack to search and update.
    /// * `schema` - Schema containing table definitions and reference information.
    /// * `table_name` - Name of the source table (e.g., `"units_tables"`).
    /// * `field` - The field being edited.
    /// * `definition` - Definition of the source table.
    /// * `value_before` - Original value being replaced.
    /// * `value_after` - New value to set.
    ///
    /// # Returns
    ///
    /// List of paths where references were found and updated.
    pub fn cascade_edition(pack: &mut Pack, schema: &Schema, table_name: &str, field: &Field, definition: &Definition, value_before: &str, value_after: &str) -> Vec<ContainerPath> {

        // So, how does this work:
        // - First, we need to calculate all related tables/columns. This includes the parent columns if this is a reference, and all references to this field.
        // - Second, we need to calculate all related loc fields corresponding to the edited fiels.
        // - Third, we edit the table entries.
        // - Fourth, we edit the loc entries.
        let mut edited_paths = vec![];

        // If we're not changing anything, don't bother performing an edition.
        if value_before == value_after {
            return vec![];
        }

        // Just in case we're in a reference field, find the source, and trigger the edition from there.
        let mut definition = definition.clone();
        let patches = Some(definition.patches().clone());
        let mut field = field.clone();
        let mut table_name = table_name.to_owned();
        while let Some((ref_table, ref_column)) = field.is_reference(patches.as_ref()) {
            let ref_table_name = format!("{ref_table}_tables");
            let parent_files = pack.files_by_type_and_paths_mut(&[FileType::DB], &ContainerPath::db_table_folders(&ref_table_name), true);
            if !parent_files.is_empty() {
                if let Ok(RFileDecoded::DB(table)) = parent_files[0].decoded() {
                    if let Some(index) = table.definition().column_position_by_name(&ref_column) {
                        definition = table.definition().clone();
                        field = definition.fields_processed()[index].clone();
                        table_name = table.table_name().to_owned();
                        continue;
                    }
                }
            }

            break;
        }

        // Get the tables/rows that need to be edited.
        let fields_processed = definition.fields_processed();
        let fields_localised = definition.localised_fields();
        let (mut ref_table_data, _) = schema.tables_and_columns_referencing_our_own(&table_name, field.name(), fields_processed, fields_localised);

        // Add the source table and column to the list to edit.
        ref_table_data.insert(table_name, vec![field.name().to_owned()]);

        let container_paths = ref_table_data.keys().flat_map(|ref_table_name| ContainerPath::db_table_folders(ref_table_name)).collect::<Vec<_>>();
        let mut files = pack.files_by_paths_mut(&container_paths, true);
        let mut loc_keys: Vec<(String, String)> = vec![];

        for file in files.iter_mut() {
            let path = file.path_in_container();
            if let Ok(RFileDecoded::DB(table)) = file.decoded_mut() {
                let fields_processed = table.definition().fields_processed().to_vec();
                let fields_localised = table.definition().localised_fields().to_vec();
                let localised_order = table.definition().localised_key_order().to_vec();
                let patches = table.definition().patches().clone();
                let table_name = table.table_name().to_owned();
                let table_name_no_tables = table.table_name_without_tables();
                let table_data = table.data_mut();

                let mut keys_edited = vec![];

                // Find the column to edit within the table.
                let column_indexes = fields_processed.iter()
                    .enumerate()
                    .filter_map(|(index, field)| if ref_table_data[&table_name].iter().any(|name| name == field.name()) { Some(index) } else { None })
                    .collect::<Vec<usize>>();

                // Then, go through all the rows and perform the edits.
                for row in table_data.iter_mut() {
                    for column in &column_indexes {

                        // TODO: FIX THIS SHIT. It duplicates ALL DATA IN ALL TABLES CHECK.
                        let row_copy = row.to_vec();

                        if let Some(field_data) = row.get_mut(*column) {
                            match field_data {
                                DecodedData::StringU8(field_data) |
                                DecodedData::StringU16(field_data) |
                                DecodedData::OptionalStringU8(field_data) |
                                DecodedData::OptionalStringU16(field_data) => {

                                    // Only edit exact matches.
                                    if field_data == value_before {
                                        let mut locs_edited = vec![];

                                        // If it's a key, calculate the relevant before and after loc keys.
                                        let is_key = fields_processed[*column].is_key(Some(&patches));
                                        if is_key {
                                            for loc_field in &fields_localised {
                                                let loc_key = localised_order.iter().map(|pos| row_copy[*pos as usize].data_to_string()).collect::<Vec<_>>().join("");
                                                locs_edited.push(format!("{}_{}_{}", table_name_no_tables, loc_field.name(), loc_key));
                                            }
                                        }

                                        *field_data = value_after.to_owned();

                                        if !locs_edited.is_empty() {
                                            for (index, loc_field) in fields_localised.iter().enumerate() {
                                                if let Some(key_old) = locs_edited.get(index) {
                                                    let loc_key = localised_order.iter().map(|pos| row[*pos as usize].data_to_string()).collect::<Vec<_>>().join("");
                                                    let key_new = format!("{}_{}_{}", table_name_no_tables, loc_field.name(), loc_key);
                                                    keys_edited.push((key_old.to_owned(), key_new.to_owned()))
                                                }
                                            }
                                        }

                                        if !edited_paths.contains(&path) {
                                            edited_paths.push(path.clone());
                                        }
                                    }
                                }
                                _ => continue
                            }
                        }
                    }
                }

                // If we edited a key field in the table, check if we need to edit any relevant loc field.
                if !keys_edited.is_empty() {
                    loc_keys.append(&mut keys_edited);
                }
            }
        }

        // Now, we find and replace all the loc keys we have to change.
        let mut loc_files = pack.files_by_type_mut(&[FileType::Loc]);
        for file in &mut loc_files {
            let path = file.path_in_container();
            if let Ok(RFileDecoded::Loc(data)) = file.decoded_mut() {
                let data = data.data_mut();
                for row in data.iter_mut() {
                    if let Some(
                        DecodedData::StringU8(field_data) |
                        DecodedData::StringU16(field_data) |
                        DecodedData::OptionalStringU8(field_data) |
                        DecodedData::OptionalStringU16(field_data)
                    ) = row.get_mut(0) {
                        for (key_old, key_new) in &loc_keys {
                            if field_data == key_old {
                                *field_data = key_new.to_owned();

                                if !edited_paths.contains(&path) {
                                    edited_paths.push(path.clone());
                                }
                            }
                        }
                    }
                }
            }
        }

        edited_paths
    }

    /// Merges multiple DB tables into a single new table.
    ///
    /// Combines all rows from the source tables. The first table's definition and
    /// patches are used for the merged result. All source tables are converted to
    /// match this definition before merging.
    ///
    /// # Errors
    ///
    /// Returns an error if:
    /// - Tables have different names (can't merge `units_tables` with `buildings_tables`)
    /// - Fewer than 2 tables are provided
    pub fn merge(sources: &[&Self]) -> Result<Self> {

        let table_names = sources.iter().map(|file| file.table_name()).collect::<HashSet<_>>();
        if table_names.len() > 1 {
            return Err(RLibError::RFileMergeTablesDifferentNames);
        }

        if sources.len() < 2 {
            return Err(RLibError::RFileMergeTablesNotEnoughTablesProvided);
        }

        let mut new_table = Self::new(sources[0].definition(), Some(sources[0].patches()), sources[0].table_name());
        let sources = sources.par_iter()
            .map(|table| {
                let mut table = table.table().clone();
                table.set_definition(new_table.definition());
                table
            })
            .collect::<Vec<_>>();

        let new_data = sources.par_iter()
            .map(|table| table.data().to_vec())
            .flatten()
            .collect::<Vec<_>>();
        new_table.set_data(&new_data)?;

        Ok(new_table)
    }

    /// Imports a DB table from TSV (tab-separated values) format.
    ///
    /// # Arguments
    ///
    /// * `records` - CSV reader iterator over TSV records.
    /// * `field_order` - Mapping of column positions to field names.
    /// * `schema` - Schema containing the table definition.
    /// * `table_name` - Name of the table (e.g., `"units_tables"`).
    /// * `table_version` - Version of the table definition to use.
    ///
    /// # Errors
    ///
    /// Returns an error if no matching definition is found in the schema.
    pub fn tsv_import(records: StringRecordsIter<File>, field_order: &HashMap<u32, String>, schema: &Schema, table_name: &str, table_version: i32) -> Result<Self> {
        let definition = schema.definition_by_name_and_version(table_name, table_version).ok_or(RLibError::DecodingDBNoDefinitionsFound)?;
        let definition_patch = schema.patches_for_table(table_name);
        let table = TableInMemory::tsv_import(records, definition, field_order, table_name, definition_patch)?;
        let db = DB::from(table);
        Ok(db)
    }

    /// Exports the DB table to TSV (tab-separated values) format.
    ///
    /// # Arguments
    ///
    /// * `writer` - CSV writer for the output file.
    /// * `table_path` - Path used in the TSV metadata header.
    /// * `keys_first` - If `true`, key columns are written before non-key columns.
    pub fn tsv_export(&self, writer: &mut Writer<File>, table_path: &str, keys_first: bool) -> Result<()> {
        self.table.tsv_export(writer, table_path, keys_first)
    }

    /// Returns `true` if data was modified during decoding (e.g., invalid values corrected).
    pub fn altered(&self) -> bool {
        *self.table.altered()
    }

    /// Generates combined primary keys to populate the `twad_key_deletes` table.
    ///
    /// Different tables use different key concatenation rules. This function handles
    /// the table-specific key format for each known table type.
    pub fn generate_twad_key_deletes_keys(&self, keys: &mut HashSet<String>) {
        let definition = self.definition();
        match self.table_name() {

            // Order reversed vs dave schemas: agent + attribute.
            "agent_to_agent_attributes_tables" => {
                let key_pos_0 = definition.column_position_by_name("agent").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("attribute").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + &x[key_pos_1].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Uses ; for concatenating keys: attacker + ";" + defender.
            "animation_set_prebattle_group_view_configurations_tables" => {
                let key_pos_0 = definition.column_position_by_name("attacker").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("defender").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + ";" + &x[key_pos_1].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Uses " | ", with whitespace, for concatenating keys: armory_item + " | "  + variant.
            "armory_item_variants_tables" => {
                let key_pos_0 = definition.column_position_by_name("armory_item").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("variant").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + " | " + &x[key_pos_1].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // It includes the item_type string two times: item_type + currency_type + item_type.
            "battle_currency_deployables_cost_values_tables" => {
                let key_pos_0 = definition.column_position_by_name("item_type").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("currency_type").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + &x[key_pos_1].data_to_string() + &x[key_pos_0].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // It includes the item_type string two times: item_type + currency_type + item_type.
            "battle_currency_units_cost_values_tables" => {
                let key_pos_0 = definition.column_position_by_name("item_type").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("currency_type").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + &x[key_pos_1].data_to_string() + &x[key_pos_0].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // It includes the army_name string two times: army_name + siege_item + army_name.
            "battle_set_pieces_siege_items_tables" => {
                let key_pos_0 = definition.column_position_by_name("army_name").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("siege_item").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + &x[key_pos_1].data_to_string() + &x[key_pos_0].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Uses ; for concatenating keys: distribution_profile_key + ";" + agent_type_key.
            "cai_agent_type_distribution_profile_junctions_tables" => {
                let key_pos_0 = definition.column_position_by_name("distribution_profile_key").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("agent_type_key").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + ";" + &x[key_pos_1].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Uses ; for concatenating keys: recruitment_profile_key + ";" + agent_type_key.
            "cai_agent_type_recruitment_profile_junctions_tables" => {
                let key_pos_0 = definition.column_position_by_name("recruitment_profile_key").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("agent_type_key").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + ";" + &x[key_pos_1].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Uses ; for concatenating keys: budget_allocation_key + ";" + budget_context_key + "; + budget_policy_key.
            "cai_personalities_budget_allocation_policy_junctions_tables" => {
                let key_pos_0 = definition.column_position_by_name("budget_allocation_key").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("budget_context_key").unwrap_or_default();
                let key_pos_2 = definition.column_position_by_name("budget_policy_key").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + ";" + &x[key_pos_1].data_to_string() + ";" + &x[key_pos_2].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Uses ; for concatenating keys: building_key + ";" + policy_key.
            "cai_personalities_construction_preference_policy_building_junctions_tables" => {
                let key_pos_0 = definition.column_position_by_name("building_key").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("policy_key").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + ";" + &x[key_pos_1].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // It includes the variable_group_key string two times: variable_group_key + variable_key + variable_group_key.
            "cai_task_management_system_task_generator_variable_group_junctions_tables" => {
                let key_pos_0 = definition.column_position_by_name("variable_group_key").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("variable_key").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + &x[key_pos_1].data_to_string() + &x[key_pos_0].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Uses ; for concatenating keys: manager + ";" + behaviour.
            "campaign_ai_manager_behaviour_junctions_tables" => {
                let key_pos_0 = definition.column_position_by_name("manager").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("behaviour").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + ";" + &x[key_pos_1].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Order reversed vs dave schemas: bmd_export_types + campaign_bmd_layer_group.
            "campaign_bmd_layer_group_bmd_export_types_junctions_tables" => {
                let key_pos_0 = definition.column_position_by_name("bmd_export_types").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("campaign_bmd_layer_group").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + &x[key_pos_1].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Uses custom formatting, and 1 and 0 for bools: campaign_difficulty_handicap + "-" + human ? 1 : 0 + "-" + effect + optional_campaign_key.
            "campaign_difficulty_handicap_effects_tables" => {
                let key_pos_0 = definition.column_position_by_name("campaign_difficulty_handicap").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("human").unwrap_or_default();
                let key_pos_2 = definition.column_position_by_name("effect").unwrap_or_default();
                let key_pos_3 = definition.column_position_by_name("optional_campaign_key").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| {
                        let human = if x[key_pos_1].data_to_string() == "true" { "1" } else { "0" };
                        x[key_pos_0].data_to_string().to_string() + "-" + human + "-" + &x[key_pos_2].data_to_string() + &x[key_pos_3].data_to_string()
                    })
                    .collect::<Vec<_>>()
                );
            }

            // Order reversed vs dave schemas: agent_key + campaign_effect_scope_key.
            "campaign_effect_scope_agent_junctions_tables" => {
                let key_pos_0 = definition.column_position_by_name("agent_key").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("campaign_effect_scope_key").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + &x[key_pos_1].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Uses ; for concatenating keys: feature + ";" + group.
            "campaign_features_tables" => {
                let key_pos_0 = definition.column_position_by_name("feature").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("group").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + ";" + &x[key_pos_1].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Uses ; for concatenating keys: marker_type + ";" + group.
            "campaign_markers_tables" => {
                let key_pos_0 = definition.column_position_by_name("marker_type").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("group").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + ";" + &x[key_pos_1].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Uses custom formatting: faction_override.empty() ? unit : unit + "_" + faction_override.
            "campaign_mercenary_unit_character_level_restrictions_tables" => {
                let key_pos_0 = definition.column_position_by_name("unit").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("faction_override").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| {
                        let fover = if x[key_pos_1].data_to_string().is_empty() {
                            "".to_owned()
                        } else {
                            "_".to_string() + &x[key_pos_1].data_to_string()
                        };
                        x[key_pos_0].data_to_string().to_string() + &fover
                    })
                    .collect::<Vec<_>>()
                );
            }

            // It includes the id string two times: id + id.
            "campaign_movement_spline_materials_tables" => {
                let key_pos_0 = definition.column_position_by_name("id").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + &x[key_pos_0].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Campaign doesn't exist in newer definitions: campaign + faction.
            "campaign_rogue_army_leaders_tables" => {
                let key_pos_0 = definition.column_position_by_name("campaign");
                let key_pos_1 = definition.column_position_by_name("faction").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| {
                        let camp = if let Some(y) = key_pos_0 { x[y].data_to_string().to_string() } else { "".to_owned() };
                        camp + &x[key_pos_1].data_to_string()
                    })
                    .collect::<Vec<_>>()
                );
            }

            // Uses ; for concatenating keys: faction + ";" + difficulty_level.
            "campaign_rogue_army_setups_tables" => {
                let key_pos_0 = definition.column_position_by_name("faction").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("difficulty_level").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + ";" + &x[key_pos_1].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Order custom vs dave schemas: variable_key + campaign_name + difficulty + campaign_type.
            "campaigns_campaign_variables_junctions_tables" => {
                let key_pos_0 = definition.column_position_by_name("variable_key").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("campaign_name").unwrap_or_default();
                let key_pos_2 = definition.column_position_by_name("difficulty").unwrap_or_default();
                let key_pos_3 = definition.column_position_by_name("campaign_type").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + &x[key_pos_1].data_to_string() + &x[key_pos_2].data_to_string() + &x[key_pos_3].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // It includes the tree_type string two times: tree_type + variable_key + tree_type.
            "campaign_tree_type_cultures_tables" => {
                let key_pos_0 = definition.column_position_by_name("tree_type").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("culture").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + &x[key_pos_1].data_to_string() + &x[key_pos_0].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Order reversed vs dave schemas: issuer_key + mission_key.
            "cdir_events_mission_issuer_junctions_tables" => {
                let key_pos_0 = definition.column_position_by_name("issuer_key").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("mission_key").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + &x[key_pos_1].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Uses custom formatting: (for_army ? (for_navy ? "navy" : "army") : agent_key) + skill_rank + optional_campaign_key.
            "character_experience_skill_tiers_tables" => {
                let key_pos_0 = definition.column_position_by_name("for_army").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("for_navy").unwrap_or_default();
                let key_pos_2 = definition.column_position_by_name("agent_key").unwrap_or_default();
                let key_pos_3 = definition.column_position_by_name("skill_rank").unwrap_or_default();
                let key_pos_4 = definition.column_position_by_name("optional_campaign_key").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| {
                        let mut ckey = String::new();
                        if x[key_pos_0].data_to_string() == "true" {
                            if x[key_pos_1].data_to_string() == "true" {
                                ckey.push_str("navy");
                            } else {
                                ckey.push_str("army");
                            }
                        } else {
                            ckey.push_str(&x[key_pos_2].data_to_string());
                        }

                        ckey + &x[key_pos_3].data_to_string() + &x[key_pos_4].data_to_string()
                    })
                    .collect::<Vec<_>>()
                );
            }

            // Order reversed vs dave schemas: battle_animations_table + culture_pack.
            "culture_to_battle_animation_tables_tables" => {
                let key_pos_0 = definition.column_position_by_name("battle_animations_table").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("culture_pack").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + &x[key_pos_1].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Order reversed vs dave schemas: effect + action_results_additional_outcome_record + bonus_value_id.
            "effect_bonus_value_id_action_results_additional_outcomes_junctions_tables" => {
                let key_pos_0 = definition.column_position_by_name("effect").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("action_results_additional_outcome_record").unwrap_or_default();
                let key_pos_2 = definition.column_position_by_name("bonus_value_id").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + &x[key_pos_1].data_to_string() + &x[key_pos_2].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Order reversed vs dave schemas: effect + bonus_value + projectile.
            "effect_bonus_value_projectile_junctions_tables" => {
                let key_pos_0 = definition.column_position_by_name("effect").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("bonus_value").unwrap_or_default();
                let key_pos_2 = definition.column_position_by_name("projectile").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + &x[key_pos_1].data_to_string() + &x[key_pos_2].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Uses ; for concatenating keys: effect + ";" + bonus_value_id + ";" + unit_attribute.
            "effect_bonus_value_unit_attribute_junctions_tables" => {
                let key_pos_0 = definition.column_position_by_name("effect").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("bonus_value_id").unwrap_or_default();
                let key_pos_2 = definition.column_position_by_name("unit_attribute").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + ";" + &x[key_pos_1].data_to_string() + ";" + &x[key_pos_2].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Order reversed vs dave schemas: bonus_value_id + effect + unit_record_key.
            "effect_bonus_value_unit_record_junctions_tables" => {
                let key_pos_0 = definition.column_position_by_name("bonus_value_id").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("effect").unwrap_or_default();
                let key_pos_2 = definition.column_position_by_name("unit_record_key").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + &x[key_pos_1].data_to_string() + &x[key_pos_2].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // It includes the affected_stat string two times: ground_type + affected_stat + affected_group + affected_stat.
            "ground_type_to_stat_effects_tables" => {
                let key_pos_0 = definition.column_position_by_name("ground_type").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("affected_stat").unwrap_or_default();
                let key_pos_2 = definition.column_position_by_name("affected_group").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + &x[key_pos_1].data_to_string() + &x[key_pos_2].data_to_string() + &x[key_pos_1].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Order reversed vs dave schemas: ability + land_unit.
            "land_units_to_unit_abilites_junctions_tables" => {
                let key_pos_0 = definition.column_position_by_name("land_unit").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("ability").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + &x[key_pos_1].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // It includes the affected_stat string three times: loading_quote + campaign + campaign + campaign.
            "loading_screen_quotes_to_campaigns_tables" => {
                let key_pos_0 = definition.column_position_by_name("loading_quote").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("campaign").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + &x[key_pos_1].data_to_string() + &x[key_pos_1].data_to_string() + &x[key_pos_1].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Order reversed vs dave schemas: event + optional_campaign_key + culture + optional_subculture.
            "message_event_strings_tables" => {
                let key_pos_0 = definition.column_position_by_name("event").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("optional_campaign_key").unwrap_or_default();
                let key_pos_2 = definition.column_position_by_name("culture").unwrap_or_default();
                let key_pos_3 = definition.column_position_by_name("optional_subculture").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + &x[key_pos_1].data_to_string() + &x[key_pos_2].data_to_string() + &x[key_pos_3].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Uses 1 and 0 for bools: template_key + battle_type + is_defender ? 1 : 0.
            "mp_force_gen_template_junctions_tables" => {
                let key_pos_0 = definition.column_position_by_name("template_key").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("battle_type").unwrap_or_default();
                let key_pos_2 = definition.column_position_by_name("is_defender").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| {
                        let is_defender = if x[key_pos_2].data_to_string() == "true" { "1" } else { "0" };
                        x[key_pos_0].data_to_string().to_string() + &x[key_pos_1].data_to_string() + is_defender
                    })
                    .collect::<Vec<_>>()
                );
            }

            // Uses ; for concatenating keys: initiative_record + ";" + strength.
            "provincial_initiative_strength_levels_tables" => {
                let key_pos_0 = definition.column_position_by_name("initiative_record").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("strength").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + ";" + &x[key_pos_1].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Order reversed vs dave schemas: province + region.
            "region_to_province_junctions_tables" => {
                let key_pos_0 = definition.column_position_by_name("province").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("region").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + &x[key_pos_1].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Uses | for concatenating keys: payload + "|" + agent_record.
            "ritual_payload_change_agent_capacities_tables" => {
                let key_pos_0 = definition.column_position_by_name("payload").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("agent_record").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + "|" + &x[key_pos_1].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Uses | for concatenating keys: payload + "|" + unit_record.
            "ritual_payload_change_unit_capacities_tables" => {
                let key_pos_0 = definition.column_position_by_name("payload").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("unit_record").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + "|" + &x[key_pos_1].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Uses | for concatenating keys: slot_set_type + "|" + feature.
            "slot_set_type_feature_junctions_tables" => {
                let key_pos_0 = definition.column_position_by_name("slot_set_type").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("feature").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + "|" + &x[key_pos_1].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Uses : for concatenating keys: special_ability + ":" + phase + ":" + order.
            "special_ability_to_special_ability_phase_junctions_tables" => {
                let key_pos_0 = definition.column_position_by_name("special_ability").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("phase").unwrap_or_default();
                let key_pos_2 = definition.column_position_by_name("order").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + ":" + &x[key_pos_1].data_to_string() + ":" + &x[key_pos_2].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Order reversed vs dave schemas: campaign + id + region + slot_template + slot_type.
            "start_pos_region_slot_templates_tables" => {
                let key_pos_0 = definition.column_position_by_name("campaign").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("id").unwrap_or_default();
                let key_pos_2 = definition.column_position_by_name("region").unwrap_or_default();
                let key_pos_3 = definition.column_position_by_name("slot_template").unwrap_or_default();
                let key_pos_4 = definition.column_position_by_name("slot_type").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x|
                        x[key_pos_0].data_to_string().to_string() +
                        &x[key_pos_1].data_to_string() +
                        &x[key_pos_2].data_to_string() +
                        &x[key_pos_3].data_to_string() +
                        &x[key_pos_4].data_to_string()
                    )
                    .collect::<Vec<_>>()
                );
            }

            // Order reversed vs dave schemas: faction + technology.
            "start_pos_technologies_tables" => {
                let key_pos_0 = definition.column_position_by_name("faction").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("technology").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + &x[key_pos_1].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Uses custom format and ; for concatenating some of the keys: tax_name + ";" + effect + ";" + optional_campaign_key + ";" + optional_difficulty_level + ai_only ? 1 : 0.
            "taxes_effects_jct_tables" => {
                let key_pos_0 = definition.column_position_by_name("tax_name").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("effect").unwrap_or_default();
                let key_pos_2 = definition.column_position_by_name("optional_campaign_key").unwrap_or_default();
                let key_pos_3 = definition.column_position_by_name("optional_difficulty_level").unwrap_or_default();
                let key_pos_4 = definition.column_position_by_name("ai_only").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| {
                        let ai_only = if x[key_pos_4].data_to_string() == "true" { "1" } else { "0" };
                        x[key_pos_0].data_to_string().to_string() + ";" +
                        &x[key_pos_1].data_to_string() + ";" +
                        &x[key_pos_2].data_to_string() + ";" +
                        &x[key_pos_3].data_to_string() + ai_only
                    })
                    .collect::<Vec<_>>()
                );
            }

            // Uses _ for concatenating keys: hex_id + "_" + category.
            "ui_purchasable_effects_to_hex_ids_tables" => {
                let key_pos_0 = definition.column_position_by_name("hex_id").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("category").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + "_" + &x[key_pos_1].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // Uses custom formatting and : for concatenating keys: unit + ":" + faction + ":" + (general_unit ? "1" : "0").
            "units_custom_battle_permissions_tables" => {
                let key_pos_0 = definition.column_position_by_name("unit").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("faction").unwrap_or_default();
                let key_pos_2 = definition.column_position_by_name("general_unit").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| {
                        let general_unit = if x[key_pos_2].data_to_string() == "true" { "1" } else { "0" };
                        x[key_pos_0].data_to_string().to_string() + ":" + &x[key_pos_1].data_to_string() + ":" + general_unit
                    })
                    .collect::<Vec<_>>()
                );
            }

            // It includes the level_to_unlock string two times: category + level_to_unlock + level_to_unlock.
            "workshop_categories_progress_levels_tables" => {
                let key_pos_0 = definition.column_position_by_name("category").unwrap_or_default();
                let key_pos_1 = definition.column_position_by_name("level_to_unlock").unwrap_or_default();

                keys.extend(self.data()
                    .iter()
                    .map(|x| x[key_pos_0].data_to_string().to_string() + &x[key_pos_1].data_to_string() + &x[key_pos_1].data_to_string())
                    .collect::<Vec<_>>()
                );
            }

            // For anything else (single-keys and keys that follow the schemas without weird behavior), use the twad order.
            _ => {
                let key_cols = definition.key_column_positions_by_ca_order();
                keys.extend(self.data()
                    .iter()
                    .map(|x| key_cols.iter()
                        .map(|y| x[*y].data_to_string())
                        .join("")
                    )
                    .collect::<Vec<_>>()
                );
            }
        }
    }
}

/// Implementation to create a `DB` from a `Table`.
impl From<TableInMemory> for DB {
    fn from(table: TableInMemory) -> Self {
        Self {
            mysterious_byte: true,
            guid: Uuid::new_v4().to_string(),
            table,
        }
    }
}