rpfm_lib/files/

mod.rs

//---------------------------------------------------------------------------//
// Copyright (c) 2017-2026 Ismael Gutiérrez González. All rights reserved.
//
// This file is part of the Rusted PackFile Manager (RPFM) project,
// which can be found here: https://github.com/Frodo45127/rpfm.
//
// This file is licensed under the MIT license, which can be found here:
// https://github.com/Frodo45127/rpfm/blob/master/LICENSE.
//---------------------------------------------------------------------------//

//! This module contains the definition of RFile, the file abstraction used by this lib to decode/encode files.
//!
//! There is an additional type: [`Unknown`]. This type is used as a wildcard,
//! so you can get the raw data of any non-supported file type and manipulate it yourself in a safe way.
//!
//! For more information about specific file types, including their binary format spec, please
//! **check their respective documentation**.
//!
//! # Core Abstractions
//!
//! ## RFile
//!
//! The [`RFile`] struct is the central abstraction for all files in rpfm_lib. It can represent:
//! - Files within PackFiles (or any [`Container`])
//! - Files on the filesystem
//! - Files in memory only
//!
//! Key features:
//! - **Lazy Loading**: Files can be loaded on-demand to reduce memory usage
//! - **Type Detection**: Automatically identifies file types based on path and content
//! - **Caching**: Supports caching decoded data or raw bytes
//! - **Metadata**: Tracks path, timestamp, container name, and file type
//!
//! ## File States
//!
//! Files can be in one of three internal states:
//! - **OnDisk**: Data not yet loaded (lazy loading)
//! - **Cached**: Raw bytes loaded but not decoded
//! - **Decoded**: Fully parsed into a type-specific structure
//!
//! ## Decoding/Encoding
//!
//! All file types implement the [`Decodeable`] and [`Encodeable`] traits:
//! - **Decodeable**: Parse binary data into structured format
//! - **Encodeable**: Serialize structured format back to binary
//!
//! Extra data can be passed via [`DecodeableExtraData`] and [`EncodeableExtraData`]
//! to provide context like schemas, game info, or file names.
//!
//! # Known file types
//!
//! | File Type              | Decoding Supported | Encoding Supported |
//! | ---------------------- | ------------------ | ------------------ |
//! | [`Anim`]               | Limited            | Limited            |
//! | [`AssetVariationDb`]   | Yes                | Yes                |
//! | [`AnimFragmentBattle`] | Limited            | Limited            |
//! | [`AnimPack`]           | Yes                | Yes                |
//! | [`AnimsTable`]         | Yes                | Yes                |
//! | [`Atlas`]              | Yes                | Yes                |
//! | [`Audio`]              | No                 | No                 |
//! | [`BMD`]                | Limited            | Limited            |
//! | [`BMD_Vegetation`]     | Limited            | Limited            |
//! | [`CS2_Collision`]      | Limited            | Limited            |
//! | [`CS2_Parsed`]         | Limited            | Limited            |
//! | [`Dat`]                | Limited            | Limited            |
//! | [`DB`]                 | Yes                | Yes                |
//! | [`ESF`]                | Limited            | Limited            |
//! | [`FastBinLists`]       | Yes                | Yes                |
//! | [`Font`]               | Limited            | Limited            |
//! | [`Fxc`]                | Yes                | Yes                |
//! | [`GroupFormations`]    | Limited            | Limited            |
//! | [`HLSL_Compiled`]      | Limited            | Limited            |
//! | [`Image`]              | Limited            | Limited            |
//! | [`Loc`]                | Yes                | Yes                |
//! | [`MatchedCombat`]      | Yes                | Yes                |
//! | [`Pack`]               | Yes                | Yes                |
//! | [`PortraitSettings`]   | Yes                | Yes                |
//! | [`RigidModel`]         | No                 | No                 |
//! | [`SoundBank`]          | No                 | No                 |
//! | SoundBankDatabase      | Limited            | Limited            |
//! | SoundEvents            | Limited            | Limited            |
//! | [`Text`]               | Yes                | Yes                |
//! | [`TileDatabase`]       | Yes                | Yes                |
//! | [`TradeRoutes`]        | Yes                | Yes                |
//! | [`UIC`]                | No                 | No                 |
//! | [`UnitVariant`]        | Yes                | Yes                |
//! | [`Video`]              | Yes                | Yes                |
//! | VMD                    | Yes                | Yes                |
//! | WSModel                | Yes                | Yes                |
//!
//! # Example Usage
//!
//! ```ignore
//! use rpfm_lib::files::{RFile, Decodeable, db::DB, DecodeableExtraData, table::Table};
//! use rpfm_lib::schema::Schema;
//! use std::path::Path;
//!
//! // Load a DB table from disk
//! let schema = Schema::load(Path::new("path/to/schema.ron"), None).unwrap();
//! let mut extra = DecodeableExtraData::default();
//! extra.set_schema(Some(&schema));
//! extra.set_table_name(Some("units_tables"));
//!
//! let rfile = RFile::from_disk(Path::new("units"), &extra).unwrap();
//!
//! // Access the decoded data
//! if let Some(db) = rfile.decoded().and_then(|d| d.db()) {
//!     println!("Table has {} rows", db.table().len());
//! }
//! ```
//!
//! [`Anim`]: crate::files::anim::Anim
//! [`AnimFragmentBattle`]: crate::files::anim_fragment_battle::AnimFragmentBattle
//! [`AnimPack`]: crate::files::animpack::AnimPack
//! [`AnimsTable`]: crate::files::anims_table::AnimsTable
//! [`Atlas`]: crate::files::atlas::Atlas
//! [`Audio`]: crate::files::audio::Audio
//! [`BMD`]: crate::files::bmd::Bmd
//! [`BMD_Vegetation`]: crate::files::bmd_vegetation::BmdVegetation
//! [`CS2_Collision`]: crate::files::cs2_collision::Cs2Collision
//! [`CS2_Parsed`]: crate::files::cs2_parsed::Cs2Parsed
//! [`Dat`]: crate::files::dat::Dat
//! [`DB`]: crate::files::db::DB
//! [`ESF`]: crate::files::esf::ESF
//! [`AssetVariationDb`]: crate::files::asset_variation_db::AssetVariationDb
//! [`FastBinLists`]: crate::files::fastbin_lists
//! [`Font`]: crate::files::font::Font
//! [`Fxc`]: crate::files::fxc::Fxc
//! [`GroupFormations`]: crate::files::group_formations::GroupFormations
//! [`HLSL_Compiled`]: crate::files::hlsl_compiled::HlslCompiled
//! [`Image`]: crate::files::image::Image
//! [`Loc`]: crate::files::loc::Loc
//! [`MatchedCombat`]: crate::files::matched_combat::MatchedCombat
//! [`Pack`]: crate::files::pack::Pack
//! [`PortraitSettings`]: crate::files::portrait_settings::PortraitSettings
//! [`RigidModel`]: crate::files::rigidmodel::RigidModel
//! [`SoundBank`]: crate::files::sound_bank::SoundBank
//! [`SoundEvents`]: crate::files::sound_events::SoundEvents
//! [`Text`]: crate::files::text::Text
//! [`TileDatabase`]: crate::files::tile_database::TileDatabase
//! [`TradeRoutes`]: crate::files::trade_routes::TradeRoutes
//! [`UIC`]: crate::files::uic::UIC
//! [`UnitVariant`]: crate::files::unit_variant::UnitVariant
//! [`Unknown`]: crate::files::unknown::Unknown
//! [`Video`]: crate::files::video::Video

// File decoders intentionally use the `let mut x = T::default(); x.field = …;` pattern
// to make debugging them easier.
#![allow(clippy::field_reassign_with_default)]

use crc_fast::{checksum, CrcAlgorithm};
use csv::{QuoteStyle, ReaderBuilder, WriterBuilder};
use getset::*;
use log::warn;
use rayon::prelude::*;
use serde_derive::{Serialize, Deserialize};

use std::cmp::Ordering;
use std::collections::{HashMap, HashSet};
use std::{fmt, fmt::{Debug, Display}};
use std::fs::{DirBuilder, File};
use std::io::{BufReader, Cursor, Read, Seek, SeekFrom, BufWriter, Write};
use std::path::{Path, PathBuf};

use crate::binary::{ReadBytes, WriteBytes};
use crate::compression::{CompressionFormat, Decompressible};
use crate::encryption::Decryptable;
use crate::error::{Result, RLibError};
use crate::games::{GameInfo, pfh_version::PFHVersion, supported_games::*};
use crate::{REGEX_CEO_DB, REGEX_DB, REGEX_PORTRAIT_SETTINGS};
use crate::schema::{Schema, Definition};
use crate::utils::*;

use self::anim::Anim;
use self::anim_fragment_battle::AnimFragmentBattle;
use self::animpack::AnimPack;
use self::anims_table::AnimsTable;
use self::atlas::Atlas;
use self::audio::Audio;
use self::bmd::Bmd;
use self::bmd_vegetation::BmdVegetation;
use self::dat::Dat;
use self::db::DB;
use self::esf::ESF;
use self::font::Font;
use self::group_formations::GroupFormations;
use self::hlsl_compiled::HlslCompiled;
use self::image::Image;
use self::loc::Loc;
use self::matched_combat::MatchedCombat;
use self::pack::{Pack, RESERVED_NAME_SETTINGS, RESERVED_NAME_NOTES, RESERVED_NAME_DEPENDENCIES_MANAGER, RESERVED_NAME_DEPENDENCIES_MANAGER_V2};
use self::portrait_settings::PortraitSettings;
use self::rigidmodel::RigidModel;
use self::sound_bank::SoundBank;
use self::text::Text;
use self::trade_routes::TradeRoutes;
use self::uic::UIC;
use self::unit_variant::UnitVariant;
use self::unknown::Unknown;
use self::video::Video;

pub mod anim;
pub mod anim_fragment_battle;
pub mod animpack;
pub mod anims_table;
pub mod asset_variation_db;
pub mod atlas;
pub mod audio;
#[allow(dead_code)]pub mod bmd;
#[allow(dead_code)]pub mod bmd_vegetation;
pub mod cs2_collision;
pub mod cs2_parsed;
pub mod dat;
pub mod db;
#[allow(dead_code)]pub mod esf;
pub mod fastbin_lists;
pub mod font;
pub mod fxc;
pub mod group_formations;
pub mod hlsl_compiled;
pub mod image;
pub mod loc;
pub mod matched_combat;
pub mod pack;
pub mod portrait_settings;
pub mod rigidmodel;
#[allow(dead_code)]pub mod sound_bank;
pub mod sound_bank_database;
pub mod sound_events;
pub mod table;
pub mod text;
pub mod tile_database;
pub mod trade_routes;
pub mod uic;
pub mod unit_variant;
pub mod unknown;
pub mod video;

#[cfg(test)] mod rfile_test;

//---------------------------------------------------------------------------//
//                              Enum & Structs
//---------------------------------------------------------------------------//

/// Central file abstraction for rpfm_lib.
///
/// Represents a file that can exist in multiple locations and states:
/// - Inside a PackFile or other container
/// - On the filesystem
/// - In memory only
///
/// # Lazy Loading
///
/// RFile supports lazy loading to minimize memory usage. Files can remain on disk
/// until their data is actually needed, at which point they're loaded into memory
/// automatically.
///
/// # File States
///
/// Internally, RFile can be in three states:
/// - **OnDisk**: Metadata loaded, data still on disk (minimal memory)
/// - **Cached**: Raw bytes loaded, not yet decoded
/// - **Decoded**: Fully parsed into type-specific structure
///
/// # Type Detection
///
/// File types are detected based on:
/// - File extension and path patterns
/// - Magic numbers and header bytes
/// - Container metadata
///
/// # Metadata
///
/// Each RFile tracks:
/// - **path**: Location within container or filesystem (may be empty for memory-only files)
/// - **timestamp**: Last modified time (optional)
/// - **file_type**: Detected or specified file type
/// - **container_name**: Source container name if from a container
///
/// # Example
///
/// ```ignore
/// use rpfm_lib::files::{RFile, FileType};
/// use std::path::Path;
///
/// // Load a file from disk
/// let rfile = RFile::from_disk(Path::new("units.loc"), &None).unwrap();
///
/// // Check file type
/// assert_eq!(*rfile.file_type(), FileType::Loc);
///
/// // Decode the file
/// let decoded = rfile.decode(&None, false, false).unwrap();
/// ```
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct RFile {

    /// Path of the file within a container or filesystem.
    ///
    /// Empty string for memory-only files.
    path: String,

    /// Last modified timestamp (Unix epoch).
    timestamp: Option<u64>,

    /// Detected or specified file type.
    file_type: FileType,

    /// Name of the source container, if applicable.
    container_name: Option<String>,

    /// Internal data storage (OnDisk, Cached, or Decoded).
    ///
    /// Use RFile methods instead of accessing directly.
    data: RFileInnerData,
}

/// Internal data storage states for RFile.
///
/// Represents the three possible states of file data in memory:
/// - **Decoded**: Fully parsed into structured format (highest memory, fastest access)
/// - **Cached**: Raw bytes in memory (medium memory, requires decoding)
/// - **OnDisk**: Metadata only, data on disk (lowest memory, requires loading + decoding)
///
/// This enum is internal and should not be used directly. Use RFile's public methods instead.
///
/// # State Transitions
///
/// ```text
/// OnDisk → load() → Cached → decode() → Decoded
///                     ↑                     ↓
///                     └───── encode() ──────┘
/// ```
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
enum RFileInnerData {

    /// File data loaded and decoded into type-specific structure.
    ///
    /// Ready for immediate use. Highest memory usage.
    Decoded(Box<RFileDecoded>),

    /// Raw bytes loaded into memory but not decoded.
    ///
    /// Requires decoding before use. Medium memory usage.
    Cached(Vec<u8>),

    /// File data still on disk, only metadata in memory.
    ///
    /// Requires loading and decoding before use. Lowest memory usage.
    OnDisk(OnDisk)
}

/// This struct represents a file on disk, which data has not been loaded to memory yet.
///
/// This may be a file directly on disk, or one inside another file (like inside a [Container]).
///
/// This is internal only. Users should not use it directly.
#[derive(Clone, Debug, PartialEq, Getters, Serialize, Deserialize)]
struct OnDisk {

    /// Path of the file on disk where the data is.
    ///
    /// This may be a singular file or a file containing it
    path: String,

    /// Last modified date of the file that contains the data.
    ///
    /// This is used to both, get the last modified data into the file's metadata
    /// and to check if the file has been manipulated since we created the OnDisk of it.
    timestamp: u64,

    /// Offset of the start of the file's data.
    ///
    /// `0` if the whole file is the data we want.
    start: u64,

    /// Size in bytes of the file's data.
    size: u64,

    /// Is the data compressed?.
    is_compressed: bool,

    /// If the data is compressed with [`CompressionFormat::Lz4Block`], its uncompressed size.
    ///
    /// That format cannot be auto-detected, so this is how we know the data uses it.
    #[serde(default)]
    lz4_block_uncompressed_size: Option<u64>,

    /// Is the data encrypted? And if so, with which format?.
    is_encrypted: Option<PFHVersion>,
}

/// This enum allow us to store any kind of decoded file type on a common place.
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub enum RFileDecoded {
    Anim(Anim),
    AnimFragmentBattle(AnimFragmentBattle),
    AnimPack(AnimPack),
    AnimsTable(AnimsTable),
    Atlas(Atlas),
    Audio(Audio),
    BMD(Box<Bmd>),
    BMDVegetation(BmdVegetation),
    Dat(Dat),
    DB(DB),
    ESF(ESF),
    Font(Font),
    GroupFormations(GroupFormations),
    HlslCompiled(HlslCompiled),
    Image(Image),
    Loc(Loc),
    MatchedCombat(MatchedCombat),
    Pack(Pack),
    PortraitSettings(PortraitSettings),
    RigidModel(RigidModel),
    SoundBank(SoundBank),
    Text(Text),
    TradeRoutes(TradeRoutes),
    UIC(UIC),
    UnitVariant(UnitVariant),
    Unknown(Unknown),
    Video(Video),
    VMD(Text),
    WSModel(Text),
}

/// Known file types in Total War games.
///
/// Categorizes files by their format and purpose. Each variant corresponds to a dedicated
/// submodule that implements parsing and encoding for that file type.
///
/// # Type Detection
///
/// File types are determined by:
/// - **Extension matching**: Primary method for most file types
/// - **Path patterns**: For files with special naming (e.g., DB tables)
/// - **Magic numbers**: For format disambiguation when needed
///
/// # Support Levels
///
/// - **Full**: Complete decoding and encoding support
/// - **Partial**: Read support with limitations
/// - **Passthrough**: Raw data only (use [`Unknown`] for custom handling)
///
/// See the module-level documentation for a complete support matrix.
///
/// # Unknown Type
///
/// [`FileType::Unknown`] is the default fallback for unrecognized files. These files
/// can still be read and written using the [`Unknown`] file type, which provides
/// access to raw bytes.
#[derive(Clone, Copy, Debug, Default, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)]
pub enum FileType {
    Anim,
    AnimFragmentBattle,
    AnimPack,
    AnimsTable,
    Atlas,
    Audio,
    BMD,
    BMDVegetation,
    Dat,
    DB,
    ESF,
    Font,
    GroupFormations,
    HlslCompiled,
    Image,
    Loc,
    MatchedCombat,
    Pack,
    PortraitSettings,
    RigidModel,
    SoundBank,
    Text,
    TradeRoutes,
    UIC,
    UnitVariant,
    Video,
    VMD,
    WSModel,

    #[default]
    Unknown,
}

/// This enum represents a ***Path*** inside a [Container].
#[derive(Clone, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)]
pub enum ContainerPath {

    /// This variant represents the path of a single file.
    File(String),

    /// This variant represents the path of a single folder.
    ///
    /// If this is empty, it represents the root of the container.
    Folder(String),
}

/// Additional context data for [`Decodeable::decode()`] operations.
///
/// This structure provides optional configuration and metadata that decoders may need
/// to properly interpret binary data. Different file types use different subsets of
/// these fields.
///
/// # Field Categories
///
/// - **Configuration toggles**: Control decoder behavior (lazy loading, encryption status, etc.)
/// - **OnDisk-related data**: File location and timestamp information
/// - **Table-related data**: Database table-specific context
/// - **Image-related data**: Image format detection flags
/// - **General-purpose data**: Game info, file names, sizes
///
/// # Usage
///
/// ```ignore
/// use rpfm_lib::files::DecodeableExtraData;
///
/// // For decoding a DB table
/// let extra_data = DecodeableExtraData::default()
///     .set_schema(Some(&schema))
///     .set_game_info(Some(&game_info))
///     .set_table_name(Some("units_tables"))
///     .set_return_incomplete(true);
/// ```
///
/// # Required Fields by Type
///
/// - **DB Tables**: Require `schema` and optionally `table_name` for fragments
/// - **Containers (PackFiles)**: Use `lazy_load`, `disk_file_path`, `disk_file_offset`
/// - **Images**: Use `is_dds` to enable DDS-specific decoding
/// - **Generic files**: May use `game_info`, `file_name`, `data_size`
///
/// For specific requirements, consult each file type's documentation.
#[derive(Clone, Debug, Default, Getters, Setters)]
#[getset(get = "pub", set = "pub")]
pub struct DecodeableExtraData<'a> {

    //-----------------------//
    // Configuration toggles //
    //-----------------------//

    /// Enable lazy loading for container files.
    ///
    /// When `true`, [`Container`] implementors will defer loading file data until accessed,
    /// storing only metadata initially. This reduces memory usage for large containers.
    lazy_load: bool,

    /// Indicates whether the source data was encrypted.
    ///
    /// This is informational only - data reaching decode functions should already be decrypted.
    /// Used for metadata tracking and logging.
    is_encrypted: bool,

    /// Allow returning partial data on decode errors (DB tables only).
    ///
    /// When `true`, table decoders will return successfully decoded rows even if later
    /// rows fail to decode. When `false`, any decode error fails the entire operation.
    return_incomplete: bool,

    /// Schema definition for decoding DB tables and Loc files.
    ///
    /// Required for decoding database tables, as the schema defines the table structure,
    /// column types, and versioning information.
    schema: Option<&'a Schema>,

    //----------------------------//
    // OnDisk-related config data //
    //----------------------------//

    /// Absolute path to the file on disk.
    ///
    /// Used by lazy-loading containers to know where to read data from when needed.
    /// Should be `None` for in-memory data.
    disk_file_path: Option<&'a str>,

    /// Byte offset within the disk file where this file's data begins.
    ///
    /// Used for files embedded within larger containers (e.g., individual files in a PackFile).
    /// Set to `0` for standalone files.
    disk_file_offset: u64,

    /// File modification timestamp (Unix epoch seconds).
    ///
    /// Preserved from the original file for metadata tracking.
    timestamp: u64,

    //----------------------------//
    // Table-related config data  //
    //----------------------------//

    /// Name of the table (without extension or path).
    ///
    /// Used when decoding table fragments that don't have the full path context.
    /// For example, when decoding a table from a loose file or unnamed buffer.
    table_name: Option<&'a str>,

    //----------------------------//
    // Image-Related config data  //
    //----------------------------//

    /// Flag indicating the image is in DDS (DirectDraw Surface) format.
    ///
    /// When `true`, enables DDS-specific decoding and conversion to PNG for display.
    is_dds: bool,

    //------------------------------//
    // General-purpose config data //
    //------------------------------//

    /// Complete game information context.
    ///
    /// Provides game-specific settings, version info, and feature flags that may
    /// affect decoding behavior (e.g., format variations between game versions).
    game_info: Option<&'a GameInfo>,

    /// Original filename (with extension).
    ///
    /// Used for file type detection, logging, and error messages.
    file_name: Option<&'a str>,

    /// Total size of the file data in bytes.
    ///
    /// May differ from buffer size if dealing with partial data or compressed streams.
    data_size: u64,

    /// Skip automatic path cache generation for containers.
    ///
    /// When `true`, container decoders won't build the lowercase path cache automatically.
    /// You must manually call [`Container::paths_cache_generate()`] after decoding or
    /// case-insensitive lookups will fail.
    ///
    /// This is an optimization for bulk operations where you'll rebuild the cache once
    /// at the end instead of incrementally.
    skip_path_cache_generation: bool,
}

/// Additional context data for [`Encodeable::encode()`] operations.
///
/// This structure provides optional configuration and metadata that encoders may need
/// to properly serialize structured data to binary format. Different file types use
/// different subsets of these fields.
///
/// # Field Categories
///
/// - **Configuration toggles**: Control encoder behavior (test mode, date handling, GUIDs)
/// - **Optional config data**: Game info, compression settings
///
/// # Usage
///
/// ```ignore
/// use rpfm_lib::files::EncodeableExtraData;
/// use rpfm_lib::compression::CompressionFormat;
///
/// // For encoding a DB table with GUID
/// let extra_data = EncodeableExtraData::default()
///     .set_game_info(Some(&game_info))
///     .set_table_has_guid(true)
///     .set_regenerate_table_guid(true);
///
/// // For encoding with compression
/// let extra_data = EncodeableExtraData::default()
///     .set_compression_format(CompressionFormat::Lz4)
///     .set_game_info(Some(&game_info));
/// ```
///
/// # Common Configurations
///
/// - **DB Tables**: Use `table_has_guid` and `regenerate_table_guid` to control GUID handling
/// - **Containers (PackFiles)**: Use `compression_format` to enable compression
/// - **Testing**: Use `test_mode` and `nullify_dates` for deterministic output
/// - **ESF Files**: Use `disable_compression` for nested encoding
///
/// For specific requirements, consult each file type's documentation.
#[derive(Clone, Default, Getters, Setters)]
#[getset(get = "pub", set = "pub")]
pub struct EncodeableExtraData<'a> {

    //-----------------------//
    // Configuration toggles //
    //-----------------------//

    /// Enable test mode for deterministic output.
    ///
    /// When `true`, encoders may skip randomization or use fixed values for fields
    /// that would normally vary (like auto-generated IDs), making output reproducible
    /// for testing purposes.
    test_mode: bool,

    /// Zero out all date and timestamp fields.
    ///
    /// When `true`, any date or timestamp fields are written as `0` instead of their
    /// actual values. Used in conjunction with `test_mode` for reproducible output,
    /// or when dates should be reset.
    nullify_dates: bool,

    /// Include a GUID in the DB table header.
    ///
    /// When `true`, table encoders will write a GUID (Globally Unique Identifier) in
    /// the table header. Required for Shogun 2 and newer games. Must be `false` for Empire
    /// and Napoleon, as including a GUID crashes those games on load.
    table_has_guid: bool,

    /// Generate a new GUID for the table instead of preserving the existing one.
    ///
    /// When `true`, a fresh random GUID is generated. When `false`, the existing GUID
    /// (if any) is preserved. Only meaningful when `table_has_guid` is also `true`.
    regenerate_table_guid: bool,

    //-----------------------//
    // Optional config data  //
    //-----------------------//

    /// Complete game information context.
    ///
    /// Provides game-specific settings, version info, and feature flags that may
    /// affect encoding behavior (e.g., format variations between game versions).
    game_info: Option<&'a GameInfo>,

    /// Compression format to use when writing files.
    ///
    /// Specifies which compression algorithm to apply. Common formats include:
    /// - [`CompressionFormat::None`]: No compression
    /// - [`CompressionFormat::Lz4`]: Fast compression
    /// - [`CompressionFormat::Zstd`]: Modern compression (best ratio)
    /// - [`CompressionFormat::Lzma1`]: Legacy compression (older games)
    /// - [`CompressionFormat::Lz4Block`]: Raw Lz4 blocks (Shogun 2 Complete Edition PFH3 Packs only)
    ///
    /// The game info may override this based on what the target game supports.
    compression_format: CompressionFormat,

    /// Disable compression for nested encoding operations.
    ///
    /// When `true`, prevents compression even if `compression_format` is set. Used for
    /// ESF (Empire Save File) encoding where the outer container handles compression
    /// and inner structures should remain uncompressed to avoid double-compression.
    disable_compression: bool
}

//---------------------------------------------------------------------------//
//                           Trait Definitions
//---------------------------------------------------------------------------//

/// Generic trait for decoding binary data into structured types.
///
/// This trait provides a standardized interface for deserializing binary data from any
/// source implementing [`ReadBytes`]. All Total War file types
/// in RPFM implement this trait to enable consistent decoding behavior.
///
/// # Type Parameters
///
/// The trait is object-safe and requires `Send + Sync` to enable concurrent decoding operations.
///
/// # Examples
///
/// ```ignore
/// use rpfm_lib::files::{Decodeable, DecodeableExtraData};
/// use rpfm_lib::binary::ReadBytes;
///
/// let data = &[0x01, 0x02, 0x03, 0x04];
/// let extra_data = None;
/// let decoded = MyType::decode(&mut data.as_slice(), &extra_data)?;
/// ```
pub trait Decodeable: Send + Sync {

    /// Decodes binary data into the implementing type.
    ///
    /// This method reads from any source implementing [`ReadBytes`]
    /// and constructs an instance of the implementing type.
    ///
    /// # Parameters
    ///
    /// - `data`: A mutable reference to a type implementing [`ReadBytes`]
    /// - `extra_data`: Optional additional context needed for decoding (schemas, game version, etc.)
    ///
    /// # Returns
    ///
    /// Returns `Ok(Self)` on successful decoding, or an error if the data is malformed or
    /// insufficient context was provided.
    ///
    /// # Errors
    ///
    /// This function may return errors if:
    /// - The binary data is corrupted or malformed
    /// - Required schema information is missing from `extra_data`
    /// - The data stream ends unexpectedly
    fn decode<R: ReadBytes>(data: &mut R, extra_data: &Option<DecodeableExtraData>) -> Result<Self> where Self: Sized;
}

/// Generic trait for encoding structured types into binary data.
///
/// This trait provides a standardized interface for serializing structured data to any
/// destination implementing [`WriteBytes`]. All Total War file types
/// in RPFM implement this trait to enable consistent encoding behavior.
///
/// # Type Parameters
///
/// The trait is object-safe and requires `Send + Sync` to enable concurrent encoding operations.
///
/// # Examples
///
/// ```ignore
/// use rpfm_lib::files::{Encodeable, EncodeableExtraData};
/// use rpfm_lib::binary::WriteBytes;
///
/// let mut buffer = Vec::new();
/// let extra_data = None;
/// my_instance.encode(&mut buffer, &extra_data)?;
/// ```
pub trait Encodeable: Send + Sync {

    /// Encodes the implementing type into binary data.
    ///
    /// This method writes to any destination implementing [`WriteBytes`],
    /// serializing the instance's data in the appropriate Total War file format.
    ///
    /// # Parameters
    ///
    /// - `buffer`: A mutable reference to a type implementing [`WriteBytes`]
    /// - `extra_data`: Optional additional context needed for encoding (schemas, game version, etc.)
    ///
    /// # Returns
    ///
    /// Returns `Ok(())` on successful encoding, or an error if the encoding process fails.
    ///
    /// # Errors
    ///
    /// This function may return errors if:
    /// - Writing to the buffer fails
    /// - Required schema information is missing from `extra_data`
    /// - The data contains invalid values that cannot be serialized
    fn encode<W: WriteBytes>(&mut self, buffer: &mut W, extra_data: &Option<EncodeableExtraData>) -> Result<()>;
}

/// Interface for working with container-like files.
///
/// This trait provides a unified API for manipulating file containers such as PackFiles,
/// allowing implementors to store, retrieve, and manage collections of [`RFile`]s with
/// hierarchical path structures.
///
/// # Implementors
///
/// - `Pack`: Total War PackFile containers (`.pack` files)
/// - Other container formats that store multiple files
///
/// # Core Operations
///
/// The trait provides several categories of operations:
///
/// - **File access**: Get references to files by path, type, or pattern
/// - **Insertion**: Add files from disk or [`RFile`] instances
/// - **Extraction**: Write files to disk, optionally as TSV for DB/Loc files
/// - **Removal**: Delete files or folders by path
/// - **Queries**: Check existence, list folders, filter by type
///
/// # Path Handling
///
/// All paths use forward slashes (`/`) as separators, regardless of OS. Paths starting
/// with `/` are automatically normalized by removing the leading slash.
pub trait Container {

    /// Extracts files from the container to disk.
    ///
    /// This method writes files matching the provided [`ContainerPath`] to the filesystem,
    /// optionally preserving the container's folder structure.
    ///
    /// # Parameters
    ///
    /// - `container_path`: Path to file or folder within the container to extract
    /// - `destination_path`: Target directory on disk where files will be written
    /// - `keep_container_path_structure`: If `true`, preserves the container's folder hierarchy
    /// - `schema`: If provided, attempts to export DB/Loc files as TSV (falls back to binary on error)
    /// - `case_insensitive`: Enable case-insensitive folder matching (file extraction is always case-sensitive)
    /// - `keys_first`: When exporting to TSV, place key columns first
    /// - `extra_data`: Optional encoding context for binary files
    ///
    /// # Returns
    ///
    /// Returns a list of paths to the extracted files on disk.
    ///
    /// # Errors
    ///
    /// Returns an error if:
    /// - The specified container path doesn't exist
    /// - Disk I/O operations fail
    /// - File decoding fails (for TSV export)
    ///
    /// # Examples
    ///
    /// ```ignore
    /// // Extract a single file, preserving structure
    /// let paths = container.extract(
    ///     ContainerPath::File("db/units_tables/units.bin".to_string()),
    ///     Path::new("./output"),
    ///     true,  // Keep structure
    ///     &Some(schema),
    ///     false, // Case-sensitive
    ///     true,  // Keys first
    ///     &None,
    /// )?;
    ///
    /// // Extract entire folder as TSV
    /// let paths = container.extract(
    ///     ContainerPath::Folder("db/".to_string()),
    ///     Path::new("./output"),
    ///     false, // Flat extraction
    ///     &Some(schema),
    ///     true,  // Case-insensitive
    ///     true,
    ///     &None,
    /// )?;
    /// ```
    #[allow(clippy::too_many_arguments)]
    fn extract(&mut self,
        container_path: ContainerPath,
        destination_path: &Path,
        keep_container_path_structure: bool,
        schema: &Option<Schema>,
        case_insensitive: bool,
        keys_first: bool,
        extra_data: &Option<EncodeableExtraData>,
    ) -> Result<Vec<PathBuf>> {

        let mut extracted_paths = vec![];
        match container_path {
            ContainerPath::File(mut container_path) => {
                if container_path.starts_with('/') {
                    container_path.remove(0);
                }

                let destination_path = if keep_container_path_structure {
                    destination_path.to_owned().join(&container_path)
                } else {
                    destination_path.to_owned()
                };

                let mut destination_folder = destination_path.to_owned();
                destination_folder.pop();
                DirBuilder::new().recursive(true).create(&destination_folder)?;

                let rfile = self.files_mut().get_mut(&container_path).ok_or_else(|| RLibError::FileNotFound(container_path.to_string()))?;

                // If we want to extract as tsv and we got a db/loc, export to tsv.
                if let Some(schema) = schema {
                    if rfile.file_type() == FileType::DB || rfile.file_type() == FileType::Loc {
                        let mut destination_path_tsv = destination_path.to_owned();

                        // Make sure to NOT replace the extension if there is one, only append to it.
                        match destination_path_tsv.extension() {
                            Some(extension) => {
                                let extension = format!("{}.tsv", extension.to_string_lossy());
                                destination_path_tsv.set_extension(extension)
                            },
                            None => destination_path_tsv.set_extension("tsv"),
                        };

                        let result = rfile.tsv_export_to_path(&destination_path_tsv, schema, keys_first);

                        // If it fails to extract as tsv, extract as binary.
                        if result.is_err() {
                            warn!("File with path {} failed to extract as TSV. Extracting it as binary.", rfile.path_in_container_raw());

                            let extracted_path = rfile.sanitize_and_create_file(&destination_path, extra_data)?;
                            extracted_paths.push(extracted_path);
                        } else {
                            extracted_paths.push(destination_path_tsv);
                            result?;
                        }
                    } else {
                        let extracted_path = rfile.sanitize_and_create_file(&destination_path, extra_data)?;
                        extracted_paths.push(extracted_path);
                    }
                }

                // Otherwise, just write the binary data to disk.
                else {
                    let extracted_path = rfile.sanitize_and_create_file(&destination_path, extra_data)?;
                    extracted_paths.push(extracted_path);
                }
            }
            ContainerPath::Folder(mut container_path) => {
                if container_path.starts_with('/') {
                    container_path.remove(0);
                }

                let mut rfiles = self.files_by_path_mut(&ContainerPath::Folder(container_path.clone()), case_insensitive);
                for rfile in &mut rfiles {
                    let container_path = rfile.path_in_container_raw();
                    let destination_path = if keep_container_path_structure {
                        destination_path.to_owned().join(container_path)
                    } else {
                        destination_path.to_owned()
                    };

                    let mut destination_folder = destination_path.to_owned();
                    destination_folder.pop();
                    DirBuilder::new().recursive(true).create(&destination_folder)?;

                    // If we want to extract as tsv and we got a db/loc, export to tsv.
                    if let Some(schema) = schema {
                        if rfile.file_type() == FileType::DB || rfile.file_type() == FileType::Loc {
                            let mut destination_path_tsv = destination_path.to_owned();

                            // Make sure to NOT replace the extension if there is one, only append to it.
                            match destination_path_tsv.extension() {
                                Some(extension) => {
                                    let extension = format!("{}.tsv", extension.to_string_lossy());
                                    destination_path_tsv.set_extension(extension)
                                },
                                None => destination_path_tsv.set_extension("tsv"),
                            };

                            let result = rfile.tsv_export_to_path(&destination_path_tsv, schema, keys_first);

                            // If it fails to extract as tsv, extract as binary.
                            if result.is_err() {
                                warn!("File with path {} failed to extract as TSV. Extracting it as binary.", rfile.path_in_container_raw());

                                let extracted_path = rfile.sanitize_and_create_file(&destination_path, extra_data)?;
                                extracted_paths.push(extracted_path);
                            } else {
                                extracted_paths.push(destination_path_tsv);
                                result?;
                            }
                        } else {
                            let extracted_path = rfile.sanitize_and_create_file(&destination_path, extra_data)?;
                            extracted_paths.push(extracted_path);
                        }
                    }

                    // Otherwise, just write the binary data to disk.
                    else {
                        let extracted_path = rfile.sanitize_and_create_file(&destination_path, extra_data)?;
                        extracted_paths.push(extracted_path);
                    }

                }

                // If we're extracting the whole container, also extract any relevant metadata file associated with it.
                if container_path.is_empty() {
                    extracted_paths.append(&mut self.extract_metadata(destination_path)?);
                }
            }
        }

        Ok(extracted_paths)
    }

    /// Extracts container metadata as `.json` files.
    ///
    /// This method writes any metadata associated with the container (such as pack settings,
    /// notes, or configuration) to the specified destination directory.
    ///
    /// # Parameters
    ///
    /// - `destination_path`: Directory where metadata files will be written
    ///
    /// # Returns
    ///
    /// Returns a list of paths to the extracted metadata files.
    ///
    /// # Default Implementation
    ///
    /// The default implementation does nothing and returns an empty list. Container types
    /// with metadata should override this method.
    fn extract_metadata(&mut self, _destination_path: &Path) -> Result<Vec<PathBuf>> {
        Ok(vec![])
    }

    /// Inserts an [`RFile`] into the container.
    ///
    /// If a file with the same path already exists, it will be replaced.
    ///
    /// # Parameters
    ///
    /// - `file`: The [`RFile`] to insert
    ///
    /// # Returns
    ///
    /// Returns the [`ContainerPath`] of the inserted file, or `None` if insertion failed.
    fn insert(&mut self, file: RFile) -> Result<Option<ContainerPath>> {
        let path = file.path_in_container();
        let path_raw = file.path_in_container_raw();

        self.paths_cache_insert_path(path_raw);
        self.files_mut().insert(path_raw.to_owned(), file);
        Ok(Some(path))
    }

    /// Inserts a file from disk into the container.
    ///
    /// This method reads a file from the filesystem and inserts it at the specified path
    /// within the container. If a file already exists at that path, it will be replaced.
    ///
    /// # TSV Import
    ///
    /// If a [`Schema`] is provided and the source file has a `.tsv` extension, this method
    /// will attempt to import it as a binary DB/Loc file. If the conversion fails, it falls
    /// back to importing it as a plain text file.
    ///
    /// # Parameters
    ///
    /// - `source_path`: Path to the file on disk to import
    /// - `container_path_folder`: Target path within the container (folder or full path)
    /// - `schema`: Optional schema for TSV-to-binary conversion
    ///
    /// # Returns
    ///
    /// Returns the [`ContainerPath`] of the inserted file, or `None` if insertion failed.
    ///
    /// # Errors
    ///
    /// Returns an error if:
    /// - The source file cannot be read
    /// - File type detection fails
    ///
    /// # Path Behavior
    ///
    /// - If `container_path_folder` ends with `/` or is empty, the source filename is appended
    /// - Otherwise, `container_path_folder` is used as the full target path
    fn insert_file(&mut self, source_path: &Path, container_path_folder: &str, schema: &Option<Schema>) -> Result<Option<ContainerPath>> {
        let mut container_path_folder = container_path_folder.replace('\\', "/");
        if container_path_folder.starts_with('/') {
            container_path_folder.remove(0);
        }

        if container_path_folder.ends_with('/') || container_path_folder.is_empty() {
            let trimmed_path = source_path.file_name()
                .ok_or_else(|| RLibError::PathMissingFileName(source_path.to_string_lossy().to_string()))?
                .to_string_lossy().to_string();
            container_path_folder = container_path_folder.to_owned() + &trimmed_path;
        }

        // If tsv import is enabled, try to import the file to binary before adding it to the Container.
        let mut tsv_imported = false;
        let mut rfile = match source_path.extension() {
            Some(extension) if extension.to_string_lossy() == "tsv" => {
                tsv_imported = true;
                let rfile = RFile::tsv_import_from_path(source_path, schema);
                if let Err(error) = rfile {
                    warn!("File with path {} failed to import as TSV. Importing it as binary. If you're using the CLI - did you forget to provide schema with --tsv-as-binary flag? Error was: {}", source_path.to_string_lossy(), error);

                    tsv_imported = false;
                    RFile::new_from_file_path(source_path)
                } else {
                    rfile
                }
            }
            _ => RFile::new_from_file_path(source_path),
        }?;

        if !tsv_imported {
            rfile.set_path_in_container_raw(&container_path_folder);
        }

        // Make sure to guess the file type before inserting it.
        rfile.load()?;
        rfile.guess_file_type()?;

        self.insert(rfile)
    }

    /// Inserts an entire folder from disk into the container recursively.
    ///
    /// This method recursively scans a directory and imports all files, preserving
    /// the folder structure. Files with identical paths in the container are replaced.
    ///
    /// # TSV Import
    ///
    /// If a [`Schema`] is provided, `.tsv` files will be converted to binary DB/Loc format.
    /// If conversion fails, they're imported as plain text files.
    ///
    /// # Parameters
    ///
    /// - `source_path`: Path to the folder on disk to import
    /// - `container_path_folder`: Target folder path within the container
    /// - `ignored_paths`: Optional list of relative paths to exclude from import
    /// - `schema`: Optional schema for TSV-to-binary conversion
    /// - `include_base_folder`: If `true`, includes the source folder name in container paths
    ///
    /// # Returns
    ///
    /// Returns a list of all [`ContainerPath`]s that were inserted.
    ///
    /// # Errors
    ///
    /// Returns an error if:
    /// - The source directory cannot be read
    /// - Any file read or type detection fails
    ///
    /// # Folder Inclusion Behavior
    ///
    /// - `include_base_folder = false`: Contents of `source_path` go directly into `container_path_folder`
    /// - `include_base_folder = true`: A subfolder with the source folder's name is created first
    ///
    /// # Examples
    ///
    /// ```ignore
    /// // Import folder contents directly into "data/"
    /// container.insert_folder(
    ///     Path::new("./my_mod"),
    ///     "data/",
    ///     &None,
    ///     &Some(schema),
    ///     false  // Don't include "my_mod" folder name
    /// )?;
    ///
    /// // Import with ignored paths
    /// container.insert_folder(
    ///     Path::new("./my_mod"),
    ///     "data/",
    ///     &Some(vec![".git/", "node_modules/"]),
    ///     &None,
    ///     true  // Include "my_mod" folder name
    /// )?;
    /// ```
    fn insert_folder(&mut self, source_path: &Path, container_path_folder: &str, ignored_paths: &Option<Vec<&str>>, schema: &Option<Schema>, include_base_folder: bool) -> Result<Vec<ContainerPath>> {
        let mut container_path_folder = container_path_folder.replace('\\', "/");
        if !container_path_folder.is_empty() && !container_path_folder.ends_with('/') {
            container_path_folder.push('/');
        }

        if container_path_folder.starts_with('/') {
            container_path_folder.remove(0);
        }

        let mut source_path_without_base_folder = source_path.to_path_buf();
        source_path_without_base_folder.pop();

        let file_paths = files_from_subdir(source_path, true)?;
        let mut inserted_paths = Vec::with_capacity(file_paths.len());
        for file_path in file_paths {
            let trimmed_path = if include_base_folder {
                file_path.strip_prefix(&source_path_without_base_folder)?
            } else {
                file_path.strip_prefix(source_path)?
            }.to_string_lossy().replace('\\', "/");

            let file_container_path = container_path_folder.to_owned() + &trimmed_path;

            if let Some(ignored_paths) = ignored_paths {
                if ignored_paths.iter().any(|x| trimmed_path.starts_with(x)) {
                    continue;
                }
            }

            // If tsv import is enabled, try to import the file to binary before adding it to the Container.
            let mut tsv_imported = false;
            let mut rfile = match file_path.extension() {
                Some(extension) if extension.to_string_lossy() == "tsv" => {
                    tsv_imported = true;
                    let rfile = RFile::tsv_import_from_path(&file_path, schema);
                    if let Err(error) = rfile {
                        warn!("File with path {} failed to import as TSV. Importing it as binary. If you're using the CLI - did you forget to provide schema with --tsv-as-binary flag? Error was: {}", file_path.to_string_lossy(), error);

                        tsv_imported = false;
                        RFile::new_from_file_path(&file_path)
                    } else {
                        rfile
                    }
                }
                _ => RFile::new_from_file_path(&file_path),
            }?;

            if !tsv_imported {
                rfile.set_path_in_container_raw(&file_container_path);
            }

            // Make sure to guess the file type before inserting it.
            rfile.load()?;
            rfile.guess_file_type()?;

            if let Some(path) = self.insert(rfile)? {
                inserted_paths.push(path);
            }
        }

        Ok(inserted_paths)
    }

    /// Removes files matching the provided [`ContainerPath`] from the container.
    ///
    /// This method deletes files or entire folder hierarchies from the container.
    ///
    /// # Parameters
    ///
    /// - `path`: The container path to remove (file or folder)
    ///
    /// # Returns
    ///
    /// Returns a list of removed [`ContainerPath`]s, always using the [`File`](ContainerPath::File) variant.
    ///
    /// # Special Cases
    ///
    /// - `ContainerPath::Folder("")`: Represents the container root, deletes **all** files
    /// - `ContainerPath::File(...)`: Deletes a single file
    /// - `ContainerPath::Folder(...)`: Deletes all files under that folder (recursive)
    ///
    /// # Examples
    ///
    /// ```ignore
    /// // Remove a single file
    /// container.remove(&ContainerPath::File("data/units.bin".to_string()));
    ///
    /// // Remove entire folder
    /// container.remove(&ContainerPath::Folder("db/".to_string()));
    ///
    /// // Clear entire container
    /// container.remove(&ContainerPath::Folder("".to_string()));
    /// ```
    fn remove(&mut self, path: &ContainerPath) -> Vec<ContainerPath> {
        match path {
            ContainerPath::File(path) => {
                let mut path = path.to_owned();
                if path.starts_with('/') {
                    path.remove(0);
                }

                self.paths_cache_remove_path(&path);
                self.files_mut().remove(&path);
                vec![ContainerPath::File(path.to_owned())]
            },
            ContainerPath::Folder(path) => {
                let mut path = path.to_owned();
                if path.starts_with('/') {
                    path.remove(0);
                }

                // If the path is empty, we mean the root of the container, including everything on it.
                if path.is_empty() {
                    self.files_mut().clear();
                    vec![ContainerPath::Folder(String::new()); 1]
                }

                // Otherwise, it's a normal folder.
                else {
                    let mut path_full = path.to_owned();
                    path_full.push('/');

                    let paths_to_remove = self.files().par_iter()
                        .filter_map(|(key, _)| {

                            // Make sure to only pick folders, not files matching folder names or partial folder matches!
                            if key.starts_with(&path_full) {
                                Some(key.to_owned())
                            } else {
                                None
                            }
                        }).collect::<Vec<String>>();

                    paths_to_remove.iter().for_each(|path| {
                        self.paths_cache_remove_path(path);
                        self.files_mut().remove(path);
                    });

                    // Fix for when we try to delete empty folders.
                    if paths_to_remove.is_empty() {
                        vec![ContainerPath::Folder(path); 1]
                    } else {
                        paths_to_remove.par_iter().map(|path| ContainerPath::File(path.to_string())).collect()
                    }
                }
            }
        }
    }

    /// Returns the full path on disk where this container is stored.
    ///
    /// # Returns
    ///
    /// The absolute file path, or an empty string if the container is not backed by a disk file.
    fn disk_file_path(&self) -> &str;

    /// Returns the filename of the container on disk.
    ///
    /// Extracts just the filename portion from [`disk_file_path()`](Self::disk_file_path).
    ///
    /// # Returns
    ///
    /// The filename as a string, or an empty string if no disk path is set.
    fn disk_file_name(&self) -> String {
       PathBuf::from(self.disk_file_path()).file_name().unwrap_or_default().to_string_lossy().to_string()
    }

    /// Returns the byte offset of this container's data within its disk file.
    ///
    /// This is used for nested containers (e.g., a container embedded within another file).
    ///
    /// # Returns
    ///
    /// The offset in bytes, or `0` if the container starts at the beginning of the file.
    fn disk_file_offset(&self) -> u64;

    /// Checks if a file with the specified path exists in the container.
    ///
    /// # Parameters
    ///
    /// - `path`: The file path to check (case-sensitive)
    ///
    /// # Returns
    ///
    /// `true` if the file exists, `false` otherwise.
    fn has_file(&self, path: &str) -> bool {
        self.files().get(path).is_some()
    }

    /// Checks if a non-empty folder exists at the specified path.
    ///
    /// A folder is considered to exist if there is at least one file whose path starts
    /// with the provided folder path.
    ///
    /// # Parameters
    ///
    /// - `path`: The folder path to check
    ///
    /// # Returns
    ///
    /// `true` if the folder exists and contains files, `false` otherwise.
    ///
    /// # Note
    ///
    /// Empty string always returns `false`. Paths are normalized to end with `/` for matching.
    fn has_folder(&self, path: &str) -> bool {
        if path.is_empty() {
           false
        } else {

            // Make sure we don't trigger false positives due to similarly started files/folders.
            let path = if path.ends_with('/') {
                path.to_string()
            } else {
                let mut path = path.to_string();
                path.push('/');
                path
            };

            self.files().keys().any(|x| x.starts_with(&path) && x.len() > path.len())
        }
    }

    /// Returns a reference to an [`RFile`] in the container by path.
    ///
    /// # Parameters
    ///
    /// - `path`: The file path to look up
    /// - `case_insensitive`: If `true`, performs case-insensitive matching
    ///
    /// # Returns
    ///
    /// `Some(&RFile)` if the file exists, `None` otherwise.
    fn file(&self, path: &str, case_insensitive: bool) -> Option<&RFile> {
        if case_insensitive {
            let lower = path.to_lowercase();
            self.paths_cache().get(&lower).and_then(|paths| self.files().get(&paths[0]))
        } else {
            self.files().get(path)
        }
    }

    /// Returns a mutable reference to an [`RFile`] in the container by path.
    ///
    /// # Parameters
    ///
    /// - `path`: The file path to look up
    /// - `case_insensitive`: If `true`, performs case-insensitive matching
    ///
    /// # Returns
    ///
    /// `Some(&mut RFile)` if the file exists, `None` otherwise.
    fn file_mut(&mut self, path: &str, case_insensitive: bool) -> Option<&mut RFile> {
        if case_insensitive {
            let lower = path.to_lowercase();
            self.paths_cache().get(&lower).cloned().and_then(|paths| self.files_mut().get_mut(&paths[0]))
        } else {
            self.files_mut().get_mut(path)
        }
    }

    /// Returns a reference to the internal file map.
    ///
    /// The map uses file paths as keys and [`RFile`]s as values.
    fn files(&self) -> &HashMap<String, RFile>;

    /// Returns a mutable reference to the internal file map.
    ///
    /// The map uses file paths as keys and [`RFile`]s as values.
    fn files_mut(&mut self) -> &mut HashMap<String, RFile>;

    /// Returns references to all files of the specified types.
    ///
    /// # Parameters
    ///
    /// - `file_types`: Slice of [`FileType`]s to filter by
    ///
    /// # Returns
    ///
    /// A vector of references to matching files.
    fn files_by_type(&self, file_types: &[FileType]) -> Vec<&RFile> {
        self.files()
            .iter()
            .filter(|(_, file)| file_types.contains(&file.file_type))
            .map(|(_, file)| file)
            .collect()
    }

    /// Returns mutable references to all files of the specified types.
    ///
    /// # Parameters
    ///
    /// - `file_types`: Slice of [`FileType`]s to filter by
    ///
    /// # Returns
    ///
    /// A vector of mutable references to matching files.
    fn files_by_type_mut(&mut self, file_types: &[FileType]) -> Vec<&mut RFile> {
        self.files_mut().par_iter_mut().filter(|(_, file)| file_types.contains(&file.file_type)).map(|(_, file)| file).collect()
    }

    /// Returns references to files matching the provided [`ContainerPath`].
    ///
    /// # Parameters
    ///
    /// - `path`: The container path to match (file or folder)
    /// - `case_insensitive`: Enable case-insensitive matching
    ///
    /// # Returns
    ///
    /// A vector of references to matching files.
    ///
    /// # Special Cases
    ///
    /// `ContainerPath::Folder("")` represents the container root and returns **all** files.
    fn files_by_path(&self, path: &ContainerPath, case_insensitive: bool) -> Vec<&RFile> {
        match path {
            ContainerPath::File(path) => self.file(path, case_insensitive).map(|file| vec![file]).unwrap_or(vec![]),
            ContainerPath::Folder(path) => {

                // If the path is empty, get everything.
                if path.is_empty() {
                    self.files().values().collect()
                }

                else {
                    let mut path = if case_insensitive { path.to_lowercase() } else { path.to_owned() };
                    if !path.ends_with('/') {
                        path.push('/');
                    }

                    // Otherwise, only get the files under our folder.
                    let real_paths = self.paths_cache()
                        .par_iter()
                        .filter_map(|(lower_path, real_paths)| if lower_path.starts_with(&path) { Some(real_paths) } else { None })
                        .map(|paths| paths.iter().map(|path| ContainerPath::File(path.to_owned())).collect::<Vec<_>>())
                        .flatten()
                        .collect::<Vec<_>>();

                    self.files_by_paths(&real_paths, false)
                }
            },
        }
    }

    /// Returns mutable references to files matching the provided [`ContainerPath`].
    ///
    /// # Parameters
    ///
    /// - `path`: The container path to match (file or folder)
    /// - `case_insensitive`: Enable case-insensitive matching
    ///
    /// # Returns
    ///
    /// A vector of mutable references to matching files.
    ///
    /// # Special Cases
    ///
    /// `ContainerPath::Folder("")` represents the container root and returns **all** files.
    fn files_by_path_mut(&mut self, path: &ContainerPath, case_insensitive: bool) -> Vec<&mut RFile> {
        match path {
            ContainerPath::File(path) => self.file_mut(path, case_insensitive).map(|file| vec![file]).unwrap_or(vec![]),
            ContainerPath::Folder(path) => {

                // If the path is empty, get everything.
                if path.is_empty() {
                    self.files_mut().values_mut().collect()
                }

                // Otherwise, only get the files under our folder.
                else {
                    self.files_mut().par_iter_mut()
                        .filter_map(|(key, file)|
                            if case_insensitive {
                                if starts_with_case_insensitive(key, path) { Some(file) } else { None }
                            } else if key.starts_with(path) {
                                Some(file)
                            } else {
                                None
                            }
                        ).collect::<Vec<&mut RFile>>()
                }
            },
        }
    }

    /// Returns references to files matching any of the provided [`ContainerPath`]s.
    ///
    /// # Parameters
    ///
    /// - `paths`: Slice of container paths to match
    /// - `case_insensitive`: Enable case-insensitive matching
    ///
    /// # Returns
    ///
    /// A vector of references to all matching files (may contain duplicates if paths overlap).
    fn files_by_paths(&self, paths: &[ContainerPath], case_insensitive: bool) -> Vec<&RFile> {
        paths.iter()
            .flat_map(|path| self.files_by_path(path, case_insensitive))
            .collect()
    }

    /// Returns mutable references to files matching any of the provided [`ContainerPath`]s.
    ///
    /// This method should be used instead of [`files_by_path_mut()`](Self::files_by_path_mut)
    /// when you need mutable references to files across multiple different paths, as it
    /// properly handles the borrowing requirements.
    ///
    /// # Parameters
    ///
    /// - `paths`: Slice of container paths to match
    /// - `case_insensitive`: Enable case-insensitive matching
    ///
    /// # Returns
    ///
    /// A vector of mutable references to all matching files (no duplicates).
    fn files_by_paths_mut(&mut self, paths: &[ContainerPath], case_insensitive: bool) -> Vec<&mut RFile> {
        self.files_mut()
            .iter_mut()
            .filter(|(file_path, _)| {
                paths.iter().any(|path| {
                    match path {
                        ContainerPath::File(path) => {
                            if case_insensitive {
                                caseless::canonical_caseless_match_str(file_path, path)
                            } else {
                                file_path == &path
                            }
                        }
                        ContainerPath::Folder(path) => {
                            if case_insensitive {
                                starts_with_case_insensitive(file_path, path)
                            } else {
                                file_path.starts_with(path)
                            }
                        }
                    }
                })
            })
            .map(|(_, file)| file)
            .collect()
    }

    /// Returns references to files matching both type and path criteria.
    ///
    /// This is a filtered combination of [`files_by_type()`](Self::files_by_type) and
    /// [`files_by_paths()`](Self::files_by_paths).
    ///
    /// # Parameters
    ///
    /// - `file_types`: Slice of [`FileType`]s to filter by
    /// - `paths`: Slice of [`ContainerPath`]s to match
    /// - `case_insensitive`: Enable case-insensitive path matching
    ///
    /// # Returns
    ///
    /// A vector of references to files matching both the type and path criteria.
    fn files_by_type_and_paths(&self, file_types: &[FileType], paths: &[ContainerPath], case_insensitive: bool) -> Vec<&RFile> {
        paths.iter()
            .flat_map(|path| self.files_by_path(path, case_insensitive)
                .into_iter()
                .filter(|file| file_types.contains(&file.file_type()))
                .collect::<Vec<_>>()
            ).collect()
    }

    /// Returns mutable references to files matching both type and path criteria.
    ///
    /// This is a filtered combination of [`files_by_type_mut()`](Self::files_by_type_mut) and
    /// [`files_by_paths_mut()`](Self::files_by_paths_mut).
    ///
    /// # Parameters
    ///
    /// - `file_types`: Slice of [`FileType`]s to filter by
    /// - `paths`: Slice of [`ContainerPath`]s to match
    /// - `case_insensitive`: Enable case-insensitive path matching
    ///
    /// # Returns
    ///
    /// A vector of mutable references to files matching both the type and path criteria.
    fn files_by_type_and_paths_mut(&mut self, file_types: &[FileType], paths: &[ContainerPath], case_insensitive: bool) -> Vec<&mut RFile> {
        self.files_by_paths_mut(paths, case_insensitive).into_iter().filter(|file| file_types.contains(&file.file_type())).collect()
    }

    /// Regenerates the internal paths cache from the current file list.
    ///
    /// The paths cache maps lowercase paths to their actual casing variants, enabling
    /// efficient case-insensitive lookups. This method should be called after bulk
    /// modifications to the file list.
    fn paths_cache_generate(&mut self) {
        self.paths_cache_mut().clear();

        let mut cache: HashMap<String, Vec<String>> = HashMap::new();
        self.files().keys().for_each(|path| {
            let lower = path.to_lowercase();
            match cache.get_mut(&lower) {
                Some(paths) => paths.push(path.to_owned()),
                None => { cache.insert(lower, vec![path.to_owned()]); },
            }
        });

        *self.paths_cache_mut() = cache;
    }

    /// Adds a single path to the paths cache.
    ///
    /// This is more efficient than regenerating the entire cache when adding individual files.
    ///
    /// # Parameters
    ///
    /// - `path`: The file path to add (with original casing)
    fn paths_cache_insert_path(&mut self, path: &str) {
        let path_lower = path.to_lowercase();
        match self.paths_cache_mut().get_mut(&path_lower) {
            Some(paths) => if paths.iter().all(|x| x != path) {
                paths.push(path.to_owned());
            }
            None => { self.paths_cache_mut().insert(path_lower, vec![path.to_owned()]); }
        }
    }

    /// Removes a single path from the paths cache.
    ///
    /// This is more efficient than regenerating the entire cache when removing individual files.
    ///
    /// # Parameters
    ///
    /// - `path`: The file path to remove (with original casing)
    ///
    /// # Note
    ///
    /// Reserved paths (notes, settings) are automatically skipped.
    fn paths_cache_remove_path(&mut self, path: &str) {
        let path_lower = path.to_lowercase();

        // Skip reserved paths when using this.
        if path_lower == RESERVED_NAME_NOTES || path_lower == RESERVED_NAME_SETTINGS || path_lower == RESERVED_NAME_DEPENDENCIES_MANAGER_V2 {
            return;
        }

        match self.paths_cache_mut().get_mut(&path_lower) {
            Some(paths) => {
                match paths.iter().position(|x| x == path) {
                    Some(pos) => {
                        paths.remove(pos);
                        if paths.is_empty() {
                            self.paths_cache_mut().remove(&path_lower);
                        }
                    },
                    None => { warn!("remove_path received a valid path, but we don't have casing equivalence for it. This is a bug. {path_lower}, {path}"); },
                }
            }
            None => { warn!("remove_path received an invalid path. This is a bug. {path_lower}, {path}"); },
        }
    }

    /// Returns the paths cache mapping lowercase paths to their original-cased variants.
    ///
    /// This cache enables efficient case-insensitive file lookups. The map structure is:
    /// `lowercase_path -> vec![OriginalCased1, OriginalCased2, ...]`
    ///
    /// # Important
    ///
    /// If you manipulate the file list directly (via [`files_mut()`](Self::files_mut)),
    /// you **must** update this cache using [`paths_cache_insert_path()`](Self::paths_cache_insert_path),
    /// [`paths_cache_remove_path()`](Self::paths_cache_remove_path), or
    /// [`paths_cache_generate()`](Self::paths_cache_generate).
    fn paths_cache(&self) -> &HashMap<String, Vec<String>>;

    /// Returns a mutable reference to the paths cache.
    ///
    /// See [`paths_cache()`](Self::paths_cache) for details on cache structure and maintenance.
    fn paths_cache_mut(&mut self) -> &mut HashMap<String, Vec<String>>;

    /// Returns a set of all folder paths contained within the container.
    ///
    /// This method analyzes file paths to extract unique folder hierarchies.
    ///
    /// # Returns
    ///
    /// A set of folder paths (without trailing slashes). Root-level files contribute no entries.
    fn paths_folders_raw(&self) -> HashSet<String> {
        self.files()
            .par_iter()
            .filter_map(|(path, _)| {
                let file_path_split = path.split('/').collect::<Vec<&str>>();
                let folder_path_len = file_path_split.len() - 1;
                if folder_path_len == 0 {
                    None
                } else {

                    let mut paths = Vec::with_capacity(folder_path_len);

                    for (index, folder) in file_path_split.iter().enumerate() {
                        if index < path.len() - 1 && !folder.is_empty() {
                            paths.push(file_path_split[0..=index].join("/"))
                        }
                    }

                    Some(paths)
                }
            })
            .flatten()
            .collect::<HashSet<String>>()
    }

    /// This method returns the list of [ContainerPath] corresponding to RFiles within the provided Container.
    fn paths(&self) -> Vec<ContainerPath> {
        self.files()
            .par_iter()
            .map(|(path, _)| ContainerPath::File(path.to_owned()))
            .collect()
    }

    /// This method returns the list of paths (as [&str]) corresponding to RFiles within the provided Container.
    fn paths_raw(&self) -> Vec<&str> {
        self.files()
            .par_iter()
            .map(|(path, _)| &**path)
            .collect()
    }

    /// This function returns the list of paths (as [String]) corresponding to RFiles that match the provided [ContainerPath].
    fn paths_raw_from_container_path(&self, path: &ContainerPath) -> Vec<String> {
        match path {
            ContainerPath::File(path) => vec![path.to_owned(); 1],
            ContainerPath::Folder(path) => {

                // If the path is empty, get everything.
                if path.is_empty() {
                    self.paths_raw().iter().map(|x| x.to_string()).collect()
                }

                // Otherwise, only get the paths under our folder.
                else {
                    self.files().par_iter()
                        .filter_map(|(key, file)|
                            if key.starts_with(path) {
                                Some(file.path_in_container_raw().to_owned())
                            } else {
                                None
                            }
                        ).collect::<Vec<String>>()
                }
            },
        }
    }

    /// This method returns the `Last modified date` stored on the provided Container, in seconds.
    ///
    /// A default implementation that returns `0` is provided for Container types that don't support internal timestamps.
    ///
    /// Implementors should return `0` if the Container doesn't have a file on disk yet.
    fn internal_timestamp(&self) -> u64 {
       0
    }

    /// This method returns the `Last modified date` the filesystem reports for the container file, in seconds.
    ///
    /// Implementors should return `0` if the Container doesn't have a file on disk yet.
    fn local_timestamp(&self) -> u64;

    /// This function preloads to memory any lazy-loaded RFile within this container.
    fn preload(&mut self) -> Result<()> {
        self.files_mut()
            .into_par_iter()
            .try_for_each(|(_, rfile)| rfile.encode(&None, false, true, false).map(|_| ()))
    }

    /// This function allows you to *move* multiple RFiles or folders of RFiles from one folder to another.
    ///
    /// It returns a list with all the new [ContainerPath].
    fn move_paths(&mut self, in_out_paths: &[(ContainerPath, ContainerPath)]) -> Result<Vec<(ContainerPath, ContainerPath)>> {
        let mut successes = vec![];
        for (source_path, destination_path) in in_out_paths {
            successes.append(&mut self.move_path(source_path, destination_path)?);
        }

        Ok(successes)
    }

    /// This function allows you to *move* any RFile or folder of RFiles from one folder to another.
    ///
    /// It returns a list with all the new [ContainerPath].
    fn move_path(&mut self, source_path: &ContainerPath, destination_path: &ContainerPath) -> Result<Vec<(ContainerPath, ContainerPath)>> {
        match source_path {
            ContainerPath::File(source_path) => match destination_path {
                ContainerPath::File(destination_path) => {
                    if destination_path.is_empty() {
                        return Err(RLibError::EmptyDestiny);
                    }

                    self.paths_cache_remove_path(source_path);
                    let mut moved = self
                        .files_mut()
                        .remove(source_path)
                        .ok_or_else(|| RLibError::FileNotFound(source_path.to_string()))?;

                    moved.set_path_in_container_raw(destination_path);

                    self.insert(moved).map(|x| match x {
                        Some(x) => vec![(ContainerPath::File(source_path.to_string()), x); 1],
                        None => Vec::with_capacity(0)
                    })
                },
                ContainerPath::Folder(_) => unreachable!("move_path_1"),
            },
            ContainerPath::Folder(source_path) => match destination_path {
                ContainerPath::File(_) => unreachable!("move_path_2"),
                ContainerPath::Folder(destination_path) => {
                    if destination_path.is_empty() {
                        return Err(RLibError::EmptyDestiny);
                    }

                    // Fix to avoid false positives.
                    let mut source_path_end = source_path.to_owned();
                    if !source_path_end.ends_with('/') {
                        source_path_end.push('/');
                    }

                    let moved_paths = self.files()
                        .par_iter()
                        .filter_map(|(path, _)| if path.starts_with(&source_path_end) { Some(path.to_owned()) } else { None })
                        .collect::<Vec<_>>();

                    let moved = moved_paths.iter()
                        .filter_map(|x| {
                            self.paths_cache_remove_path(x);
                            self.files_mut().remove(x)
                        })
                        .collect::<Vec<_>>();

                    let mut new_paths = Vec::with_capacity(moved.len());
                    for mut moved in moved {
                        let old_path = moved.path_in_container();
                        let new_path = moved.path_in_container_raw().replacen(source_path, destination_path, 1);
                        moved.set_path_in_container_raw(&new_path);

                        if let Some(new_path) = self.insert(moved)? {
                            new_paths.push((old_path, new_path));
                        }
                    }

                    Ok(new_paths)
                },
            },
        }
    }

    /// This function removes all not-in-memory-already Files from the Container.
    ///
    /// Used for removing possibly corrupted RFiles from the Container in order to sanitize it.
    ///
    /// BE CAREFUL WITH USING THIS. IT MAY (PROBABLY WILL) CAUSE DATA LOSSES.
    fn clean_undecoded(&mut self) {
        self.files_mut().retain(|_, file| file.decoded().is_ok() || file.cached().is_ok());
    }
}

//----------------------------------------------------------------//
//                        Implementations
//----------------------------------------------------------------//

impl RFile {

    /// This function creates a RFile from a lazy-loaded file inside a Container.
    ///
    /// About the parameters:
    /// - `container`: The container this RFile is on.
    /// - `size`: Size in bytes of the RFile.
    /// - `is_compressed`: If the RFile is compressed.
    /// - `is_encrypted`: If the RFile is encrypted.
    /// - `data_pos`: Byte offset of the data from the beginning of the Container.
    /// - `file_timestamp`: Timestamp of this specific file (not of the container, but the file). If it doesn't have one, pass 0.
    /// - `path_in_container`: Path of the RFile in the container.
    ///
    /// NOTE: Remember to call `guess_file_type` after this to properly set the FileType.
    pub fn new_from_container<C: Container>(
        container: &C,
        size: u64,
        is_compressed: bool,
        is_encrypted: Option<PFHVersion>,
        data_pos: u64,
        file_timestamp: u64,
        path_in_container: &str,
    ) -> Result<Self> {
        let on_disk = OnDisk {
            path: container.disk_file_path().to_owned(),
            timestamp: container.local_timestamp(),
            start: container.disk_file_offset() + data_pos,
            size,
            is_compressed,
            lz4_block_uncompressed_size: None,
            is_encrypted,
        };

        let rfile = Self {
            path: path_in_container.to_owned(),
            timestamp: if file_timestamp == 0 { None } else { Some(file_timestamp) },
            file_type: FileType::Unknown,
            container_name: Some(container.disk_file_name()),
            data: RFileInnerData::OnDisk(on_disk)
        };

        Ok(rfile)
    }

    /// This function creates a RFile from a path on disk.
    ///
    /// This may fail if the file doesn't exist or errors out when trying to be read for metadata.
    ///
    /// NOTE: Remember to call `guess_file_type` after this to properly set the FileType.
    pub fn new_from_file(path: &str) -> Result<Self> {
        let path_checked = PathBuf::from(path);
        if !path_checked.is_file() {
            return Err(RLibError::FileNotFound(path.to_owned()));
        }

        let mut file = File::open(path)?;
        let on_disk = OnDisk {
            path: path.to_owned(),
            timestamp: last_modified_time_from_file(&file)?,
            start: 0,
            size: file.len()?,
            is_compressed: false,
            lz4_block_uncompressed_size: None,
            is_encrypted: None,
        };


        let rfile = Self {
            path: path.to_owned(),
            timestamp: Some(on_disk.timestamp),
            file_type: FileType::Unknown,
            container_name: None,
            data: RFileInnerData::OnDisk(on_disk)
        };

        Ok(rfile)
    }

    /// This function sets the uncompressed size of a lazy-loaded RFile compressed with [`CompressionFormat::Lz4Block`].
    ///
    /// That format cannot be auto-detected, so this is required to decompress it. Does nothing if the RFile is not lazy-loaded.
    pub(crate) fn set_lz4_block_uncompressed_size(&mut self, uncompressed_size: u64) {
        if let RFileInnerData::OnDisk(ref mut on_disk) = self.data {
            on_disk.lz4_block_uncompressed_size = Some(uncompressed_size);
        }
    }

    /// This function creates a RFile from a path on disk.
    ///
    /// This may fail if the file doesn't exist or errors out when trying to be read for metadata.
    ///
    /// NOTE: Remember to call `guess_file_type` after this to properly set the FileType.
    pub fn new_from_file_path(path: &Path) -> Result<Self> {
        let path = path.to_string_lossy().to_string();
        Self::new_from_file(&path)
    }

    /// This function creates a RFile from raw data on memory.
    ///
    /// NOTE: Remember to call `guess_file_type` after this to properly set the FileType.
    pub fn new_from_vec(data: &[u8], file_type: FileType, timestamp: u64, path: &str) -> Self {
        Self {
            path: path.to_owned(),
            timestamp: if timestamp == 0 { None } else { Some(timestamp) },
            file_type,
            container_name: None,
            data: RFileInnerData::Cached(data.to_vec())
        }
    }

    /// This function creates a RFile from an RFileDecoded on memory.
    ///
    /// NOTE: Remember to call `guess_file_type` after this to properly set the FileType.
    pub fn new_from_decoded(data: &RFileDecoded, timestamp: u64, path: &str) -> Self {
        Self {
            path: path.to_owned(),
            timestamp: if timestamp == 0 { None } else { Some(timestamp) },
            file_type: FileType::from(data),
            container_name: None,
            data: RFileInnerData::Decoded(Box::new(data.clone()))
        }
    }

    /// This function returns a reference to the cached data of an RFile, if said RFile has been cached. If not, it returns an error.
    ///
    /// Useful for accessing preloaded data.
    pub fn cached(&self) -> Result<&[u8]> {
        match self.data {
            RFileInnerData::Cached(ref data) => Ok(data),
            _ => Err(RLibError::FileNotCached(self.path_in_container_raw().to_string()))
        }
    }

    /// This function returns a mutable reference to the cached data of an RFile, if said RFile has been cached. If not, it returns an error.
    ///
    /// Useful for accessing preloaded data.
    pub fn cached_mut(&mut self) -> Result<&mut Vec<u8>> {
        match self.data {
            RFileInnerData::Cached(ref mut data) => Ok(data),
            _ => Err(RLibError::FileNotCached(self.path_in_container_raw().to_string()))
        }
    }

    /// This function returns a reference to the decoded data of an RFile, if said RFile has been decoded. If not, it returns an error.
    ///
    /// Useful for accessing preloaded data.
    pub fn decoded(&self) -> Result<&RFileDecoded> {
        match self.data {
            RFileInnerData::Decoded(ref data) => Ok(data),
            _ => Err(RLibError::FileNotDecoded(self.path_in_container_raw().to_string()))
        }
    }

    /// This function returns a mutable reference to the decoded data of an RFile, if said RFile has been decoded. If not, it returns an error.
    ///
    /// Useful for accessing preloaded data.
    pub fn decoded_mut(&mut self) -> Result<&mut RFileDecoded> {
        match self.data {
            RFileInnerData::Decoded(ref mut data) => Ok(data),
            _ => Err(RLibError::FileNotDecoded(self.path_in_container_raw().to_string()))
        }
    }

    /// This function replace any data a RFile has with the provided raw data.
    pub fn set_cached(&mut self, data: &[u8]) {
        self.data = RFileInnerData::Cached(data.to_vec());
    }

    /// This function allows to replace the inner decoded data of a RFile with another. It'll fail if the decoded data is not valid for the file's type.
    pub fn set_decoded(&mut self, decoded: RFileDecoded) -> Result<()> {
        match (self.file_type(), &decoded) {
            (FileType::Anim, &RFileDecoded::Anim(_)) |
            (FileType::AnimFragmentBattle, &RFileDecoded::AnimFragmentBattle(_)) |
            (FileType::AnimPack, &RFileDecoded::AnimPack(_)) |
            (FileType::AnimsTable, &RFileDecoded::AnimsTable(_)) |
            (FileType::Atlas, &RFileDecoded::Atlas(_)) |
            (FileType::Audio, &RFileDecoded::Audio(_)) |
            (FileType::BMD, &RFileDecoded::BMD(_)) |
            (FileType::BMDVegetation, &RFileDecoded::BMDVegetation(_)) |
            (FileType::Dat, &RFileDecoded::Dat(_)) |
            (FileType::DB, &RFileDecoded::DB(_)) |
            (FileType::ESF, &RFileDecoded::ESF(_)) |
            (FileType::Font, &RFileDecoded::Font(_)) |
            (FileType::GroupFormations, &RFileDecoded::GroupFormations(_)) |
            (FileType::HlslCompiled, &RFileDecoded::HlslCompiled(_)) |
            (FileType::Image, &RFileDecoded::Image(_)) |
            (FileType::Loc, &RFileDecoded::Loc(_)) |
            (FileType::MatchedCombat, &RFileDecoded::MatchedCombat(_)) |
            (FileType::Pack, &RFileDecoded::Pack(_)) |
            (FileType::PortraitSettings, &RFileDecoded::PortraitSettings(_)) |
            (FileType::RigidModel, &RFileDecoded::RigidModel(_)) |
            (FileType::SoundBank, &RFileDecoded::SoundBank(_)) |
            (FileType::Text, &RFileDecoded::Text(_)) |
            (FileType::TradeRoutes, &RFileDecoded::TradeRoutes(_)) |
            (FileType::UIC, &RFileDecoded::UIC(_)) |
            (FileType::UnitVariant, &RFileDecoded::UnitVariant(_)) |
            (FileType::Video, &RFileDecoded::Video(_)) |
            (FileType::VMD, &RFileDecoded::VMD(_)) |
            (FileType::WSModel, &RFileDecoded::WSModel(_)) |
            (FileType::Unknown, &RFileDecoded::Unknown(_)) => self.data = RFileInnerData::Decoded(Box::new(decoded)),
            _ => return Err(RLibError::DecodedDataDoesNotMatchFileType(self.file_type(), From::from(&decoded)))
        }

        Ok(())
    }

    /// This function decodes an RFile from binary data, optionally caching and returning the decoded RFile.
    ///
    /// About the arguments:
    ///
    /// - `extra_data`: any data needed to decode specific file types. Check each file type for info about what do each file type need.
    /// - `keep_in_cache`: if true, the data will be cached on memory.
    /// - `return_data`: if true, the decoded data will be returned.
    ///
    /// NOTE: Passing `keep_in_cache` and `return_data` at false causes this function to decode the RFile and
    /// immediately drop the resulting data.
    pub fn decode(&mut self, extra_data: &Option<DecodeableExtraData>, keep_in_cache: bool, return_data: bool) -> Result<Option<RFileDecoded>> {
        let mut already_decoded = false;
        let decoded = match &self.data {

            // If the data is already decoded, just return a copy of it.
            RFileInnerData::Decoded(data) => {
                already_decoded = true;

                // Microoptimization: don't clone data if we're not going to use it.
                if !return_data {
                    return Ok(None);
                }

                *data.clone()
            },

            // If the data is on memory but not yet decoded, decode it.
            RFileInnerData::Cached(data) => {

                // Copy the provided extra data (if any), then replace the file-specific stuff.
                let mut extra_data = match extra_data {
                    Some(extra_data) => extra_data.clone(),
                    None => DecodeableExtraData::default(),
                };
                extra_data.file_name = self.file_name();
                extra_data.data_size = data.len() as u64;

                // Some types require extra data specific for them to be added to the extra data before decoding.
                let mut data = Cursor::new(data);
                match self.file_type {
                    FileType::Anim => RFileDecoded::Anim(Anim::decode(&mut data, &Some(extra_data))?),
                    FileType::AnimFragmentBattle => RFileDecoded::AnimFragmentBattle(AnimFragmentBattle::decode(&mut data, &Some(extra_data))?),
                    FileType::AnimPack => RFileDecoded::AnimPack(AnimPack::decode(&mut data, &Some(extra_data))?),
                    FileType::AnimsTable => RFileDecoded::AnimsTable(AnimsTable::decode(&mut data, &Some(extra_data))?),
                    FileType::Atlas => RFileDecoded::Atlas(Atlas::decode(&mut data, &Some(extra_data))?),
                    FileType::Audio => RFileDecoded::Audio(Audio::decode(&mut data, &Some(extra_data))?),
                    FileType::BMD => RFileDecoded::BMD(Box::new(Bmd::decode(&mut data, &Some(extra_data))?)),
                    FileType::BMDVegetation => RFileDecoded::BMDVegetation(BmdVegetation::decode(&mut data, &Some(extra_data))?),
                    FileType::Dat => RFileDecoded::Dat(Dat::decode(&mut data, &Some(extra_data))?),
                    FileType::DB => {

                        if extra_data.table_name.is_none() {
                            extra_data.table_name = self.db_table_name_from_path();
                        }
                        RFileDecoded::DB(DB::decode(&mut data, &Some(extra_data))?)
                    },
                    FileType::ESF => RFileDecoded::ESF(ESF::decode(&mut data, &Some(extra_data))?),
                    FileType::Font => RFileDecoded::Font(Font::decode(&mut data, &Some(extra_data))?),
                    FileType::GroupFormations => RFileDecoded::GroupFormations(GroupFormations::decode(&mut data, &Some(extra_data))?),
                    FileType::HlslCompiled => RFileDecoded::HlslCompiled(HlslCompiled::decode(&mut data, &Some(extra_data))?),
                    FileType::Image => {

                        if self.path.ends_with(".dds") {
                            extra_data.is_dds = true;
                        }

                        RFileDecoded::Image(Image::decode(&mut data, &Some(extra_data))?)
                    },
                    FileType::Loc => RFileDecoded::Loc(Loc::decode(&mut data, &Some(extra_data))?),
                    FileType::MatchedCombat => RFileDecoded::MatchedCombat(MatchedCombat::decode(&mut data, &Some(extra_data))?),
                    FileType::Pack => RFileDecoded::Pack(Pack::decode(&mut data, &Some(extra_data))?),
                    FileType::PortraitSettings => RFileDecoded::PortraitSettings(PortraitSettings::decode(&mut data, &Some(extra_data))?),
                    FileType::RigidModel => RFileDecoded::RigidModel(RigidModel::decode(&mut data, &Some(extra_data))?),
                    FileType::SoundBank => RFileDecoded::SoundBank(SoundBank::decode(&mut data, &Some(extra_data))?),
                    FileType::Text => RFileDecoded::Text(Text::decode(&mut data, &Some(extra_data))?),
                    FileType::TradeRoutes => RFileDecoded::TradeRoutes(TradeRoutes::decode(&mut data, &Some(extra_data))?),
                    FileType::UIC => RFileDecoded::UIC(UIC::decode(&mut data, &Some(extra_data))?),
                    FileType::UnitVariant => RFileDecoded::UnitVariant(UnitVariant::decode(&mut data, &Some(extra_data))?),
                    FileType::Unknown => RFileDecoded::Unknown(Unknown::decode(&mut data, &Some(extra_data))?),
                    FileType::Video => RFileDecoded::Video(Video::decode(&mut data, &Some(extra_data))?),
                    FileType::VMD => RFileDecoded::VMD(Text::decode(&mut data, &Some(extra_data))?),
                    FileType::WSModel => RFileDecoded::WSModel(Text::decode(&mut data, &Some(extra_data))?),
                }
            },

            // If the data is not yet in memory, it depends:
            // - If it's something we can lazy-load and we want to, decode it directly from disk.
            // - If it's not, load it to memory and decode it from there.
            RFileInnerData::OnDisk(data) => {

                // Copy the provided extra data (if any), then replace the file-specific stuff.
                let raw_data = data.read()?;
                let mut extra_data = match extra_data {
                    Some(extra_data) => extra_data.clone(),
                    None => DecodeableExtraData::default(),
                };

                extra_data.file_name = self.file_name();
                extra_data.data_size = raw_data.len() as u64;

                // These are the easy types: just load the data to memory, and decode.
                let mut data = Cursor::new(raw_data);
                match self.file_type {
                    FileType::Anim => RFileDecoded::Anim(Anim::decode(&mut data, &Some(extra_data))?),
                    FileType::AnimFragmentBattle => RFileDecoded::AnimFragmentBattle(AnimFragmentBattle::decode(&mut data, &Some(extra_data))?),
                    FileType::AnimsTable => RFileDecoded::AnimsTable(AnimsTable::decode(&mut data, &Some(extra_data))?),
                    FileType::AnimPack => RFileDecoded::AnimPack(AnimPack::decode(&mut data, &Some(extra_data))?),
                    FileType::Atlas => RFileDecoded::Atlas(Atlas::decode(&mut data, &Some(extra_data))?),
                    FileType::Audio => RFileDecoded::Audio(Audio::decode(&mut data, &Some(extra_data))?),
                    FileType::BMD => RFileDecoded::BMD(Box::new(Bmd::decode(&mut data, &Some(extra_data))?)),
                    FileType::BMDVegetation => RFileDecoded::BMDVegetation(BmdVegetation::decode(&mut data, &Some(extra_data))?),
                    FileType::Dat => RFileDecoded::Dat(Dat::decode(&mut data, &Some(extra_data))?),
                    FileType::DB => {

                        if extra_data.table_name.is_none() {
                            extra_data.table_name = self.db_table_name_from_path();
                        }
                        RFileDecoded::DB(DB::decode(&mut data, &Some(extra_data))?)
                    },
                    FileType::ESF => RFileDecoded::ESF(ESF::decode(&mut data, &Some(extra_data))?),
                    FileType::Font => RFileDecoded::Font(Font::decode(&mut data, &Some(extra_data))?),
                    FileType::GroupFormations => RFileDecoded::GroupFormations(GroupFormations::decode(&mut data, &Some(extra_data))?),
                    FileType::HlslCompiled => RFileDecoded::HlslCompiled(HlslCompiled::decode(&mut data, &Some(extra_data))?),
                    FileType::Image => {

                        if self.path.ends_with(".dds") {
                            extra_data.is_dds = true;
                        }

                        RFileDecoded::Image(Image::decode(&mut data, &Some(extra_data))?)
                    },
                    FileType::Loc => RFileDecoded::Loc(Loc::decode(&mut data, &Some(extra_data))?),
                    FileType::MatchedCombat => RFileDecoded::MatchedCombat(MatchedCombat::decode(&mut data, &Some(extra_data))?),
                    FileType::Pack => RFileDecoded::Pack(Pack::decode(&mut data, &Some(extra_data))?),
                    FileType::PortraitSettings => RFileDecoded::PortraitSettings(PortraitSettings::decode(&mut data, &Some(extra_data))?),
                    FileType::RigidModel => RFileDecoded::RigidModel(RigidModel::decode(&mut data, &Some(extra_data))?),
                    FileType::SoundBank => RFileDecoded::SoundBank(SoundBank::decode(&mut data, &Some(extra_data))?),
                    FileType::Text => RFileDecoded::Text(Text::decode(&mut data, &Some(extra_data))?),
                    FileType::TradeRoutes => RFileDecoded::TradeRoutes(TradeRoutes::decode(&mut data, &Some(extra_data))?),
                    FileType::UIC => RFileDecoded::UIC(UIC::decode(&mut data, &Some(extra_data))?),
                    FileType::UnitVariant => RFileDecoded::UnitVariant(UnitVariant::decode(&mut data, &Some(extra_data))?),
                    FileType::Unknown => RFileDecoded::Unknown(Unknown::decode(&mut data, &Some(extra_data))?),
                    FileType::Video => RFileDecoded::Video(Video::decode(&mut data, &Some(extra_data))?),
                    FileType::VMD => RFileDecoded::VMD(Text::decode(&mut data, &Some(extra_data))?),
                    FileType::WSModel => RFileDecoded::WSModel(Text::decode(&mut data, &Some(extra_data))?),
                }
            },
        };

        // If we're returning data, clone it. If not, skip the clone.
        if !already_decoded && keep_in_cache && return_data {
            self.data = RFileInnerData::Decoded(Box::new(decoded.clone()));
        } else if !already_decoded && keep_in_cache && !return_data{
            self.data = RFileInnerData::Decoded(Box::new(decoded));
            return Ok(None)
        }

        if return_data {
            Ok(Some(decoded))
        } else {
            Ok(None)
        }
    }

    /// This function encodes an RFile to binary, optionally caching and returning the data.
    ///
    /// About the arguments:
    /// - `extra_data`: any data needed to encode specific file types. Check each file type for info about what do each file type need.
    /// - `move_decoded_to_cache`: if true, the decoded data will be dropped in favor of undecoded cached data.
    /// - `move_undecoded_to_cache`: if true, the data will be cached on memory.
    /// - `return_data`: if true, the data will be returned.
    pub fn encode(&mut self, extra_data: &Option<EncodeableExtraData>, move_decoded_to_cache: bool, move_undecoded_to_cache: bool, return_data: bool) -> Result<Option<Vec<u8>>> {
        let mut previously_decoded = false;
        let mut already_encoded = false;
        let mut previously_undecoded = false;

        let encoded = match &mut self.data {
            RFileInnerData::Decoded(data) => {
                previously_decoded = true;
                let mut buffer = vec![];
                match &mut **data {
                    RFileDecoded::Anim(data) => data.encode(&mut buffer, extra_data)?,
                    RFileDecoded::AnimFragmentBattle(data) => data.encode(&mut buffer, extra_data)?,
                    RFileDecoded::AnimPack(data) => data.encode(&mut buffer, extra_data)?,
                    RFileDecoded::AnimsTable(data) => data.encode(&mut buffer, extra_data)?,
                    RFileDecoded::Atlas(data) => data.encode(&mut buffer, extra_data)?,
                    RFileDecoded::Audio(data) => data.encode(&mut buffer, extra_data)?,
                    RFileDecoded::BMD(data) => data.encode(&mut buffer, extra_data)?,
                    RFileDecoded::BMDVegetation(data) => data.encode(&mut buffer, extra_data)?,
                    RFileDecoded::Dat(data) => data.encode(&mut buffer, extra_data)?,
                    RFileDecoded::DB(data) => data.encode(&mut buffer, extra_data)?,
                    RFileDecoded::ESF(data) => data.encode(&mut buffer, extra_data)?,
                    RFileDecoded::Font(data) => data.encode(&mut buffer, extra_data)?,
                    RFileDecoded::GroupFormations(data) => data.encode(&mut buffer, extra_data)?,
                    RFileDecoded::HlslCompiled(data) => data.encode(&mut buffer, extra_data)?,
                    RFileDecoded::Image(data) => data.encode(&mut buffer, extra_data)?,
                    RFileDecoded::Loc(data) => data.encode(&mut buffer, extra_data)?,
                    RFileDecoded::MatchedCombat(data) => data.encode(&mut buffer, extra_data)?,
                    RFileDecoded::Pack(data) => data.encode(&mut buffer, extra_data)?,
                    RFileDecoded::PortraitSettings(data) => data.encode(&mut buffer, extra_data)?,
                    RFileDecoded::RigidModel(data) => data.encode(&mut buffer, extra_data)?,
                    RFileDecoded::SoundBank(data) => data.encode(&mut buffer, extra_data)?,
                    RFileDecoded::Text(data) => data.encode(&mut buffer, extra_data)?,
                    RFileDecoded::TradeRoutes(data) => data.encode(&mut buffer, extra_data)?,
                    RFileDecoded::UIC(data) => data.encode(&mut buffer, extra_data)?,
                    RFileDecoded::UnitVariant(data) => data.encode(&mut buffer, extra_data)?,
                    RFileDecoded::Unknown(data) => data.encode(&mut buffer, extra_data)?,
                    RFileDecoded::Video(data) => data.encode(&mut buffer, extra_data)?,
                    RFileDecoded::VMD(data) => data.encode(&mut buffer, extra_data)?,
                    RFileDecoded::WSModel(data) => data.encode(&mut buffer, extra_data)?,
                }

                buffer
            },
            RFileInnerData::Cached(data) => {
                already_encoded = true;
                data.to_vec()
            },
            RFileInnerData::OnDisk(data) => {
                previously_undecoded = true;
                data.read()?
            },
        };

        // If the RFile was already decoded.
        if previously_decoded {
            if move_decoded_to_cache {
                if return_data {
                    self.data = RFileInnerData::Cached(encoded.to_vec());
                    Ok(Some(encoded))
                } else {
                    self.data = RFileInnerData::Cached(encoded);
                    Ok(None)
                }
            } else if return_data {
                Ok(Some(encoded))
            } else {
                Ok(None)
            }
        }

        // If the RFile was not even loaded.
        else if previously_undecoded {
            if move_undecoded_to_cache {
                if return_data {
                    self.data = RFileInnerData::Cached(encoded.to_vec());
                    Ok(Some(encoded))
                } else {
                    self.data = RFileInnerData::Cached(encoded);
                    Ok(None)
                }
            } else if return_data {
                Ok(Some(encoded))
            } else {
                Ok(None)
            }
        }

        // If the RFile was already encoded and loaded.
        else if already_encoded && return_data {
            Ok(Some(encoded))
        } else {
            Ok(None)
        }
    }

    /// This function loads the data of an RFile to memory if it's not yet loaded.
    ///
    /// If it has already been loaded either to cache, or for decoding, this does nothing.
    pub fn load(&mut self) -> Result<()> {
       let loaded = match &self.data {
            RFileInnerData::Decoded(_) |
            RFileInnerData::Cached(_) => return Ok(()),
            RFileInnerData::OnDisk(data) => data.read()?,
        };

        // Piece of code to find text files we do not support yet. Needs enabling the content_inspector crate.
        #[cfg(feature = "enable_content_inspector")]
        if self.file_type() == FileType::Unknown && !loaded.is_empty() && content_inspector::inspect(&loaded).is_text() {
            dbg!(self.path_in_container_raw());
        }

        self.data = RFileInnerData::Cached(loaded);
        Ok(())
    }

    /// This function returns a copy of the `Last modified date` of this RFile, if any.
    pub fn timestamp(&self) -> Option<u64> {
        self.timestamp
    }

    /// This function returns a copy of the FileType of this RFile.
    pub fn file_type(&self) -> FileType {
        self.file_type
    }

    /// This function returns the file name if this RFile, if it has one.
    pub fn file_name(&self) -> Option<&str> {
        self.path_in_container_raw().split('/').next_back()
    }

    /// This function returns the file name of the container this RFile originates from, if any.
    pub fn container_name(&self) -> &Option<String> {
        &self.container_name
    }

    /// This function returns the [ContainerPath] corresponding to this file.
    pub fn path_in_container(&self) -> ContainerPath {
        ContainerPath::File(self.path.to_owned())
    }

    /// This function returns the [ContainerPath] corresponding to this file as an [&str].
    pub fn path_in_container_raw(&self) -> &str {
        &self.path
    }

    /// This function returns the [ContainerPath] corresponding to this file as a [Vec] of [&str].
    pub fn path_in_container_split(&self) -> Vec<&str> {
        self.path.split('/').collect()
    }

    /// This function the *table_name* of this file (the folder that contains this file) if this file is a DB table.
    ///
    /// It returns None of the file provided is not a DB Table.
    pub fn db_table_name_from_path(&self) -> Option<&str> {
        let split_path = self.path.split('/').collect::<Vec<_>>();
        let start_lower = split_path[0].to_lowercase();
        if split_path.len() == 3 && (start_lower == "db" || start_lower == "ceo_db") {
            Some(split_path[1])
        } else {
            None
        }
    }

    /// This function sets the [ContainerPath] of the provided RFile to the provided path..
    pub fn set_path_in_container_raw(&mut self, path: &str) {
        self.path = path.to_owned();
    }

    /// This function returns if the RFile can be compressed or not.
    pub fn is_compressible(&self, game_info: &GameInfo) -> bool {
        !game_info.compression_formats_supported().is_empty() &&

        // These files are needed in plain text for this lib to read them.
        self.file_name() != Some(RESERVED_NAME_DEPENDENCIES_MANAGER_V2) &&
        self.file_name() != Some(RESERVED_NAME_DEPENDENCIES_MANAGER) &&
        self.file_name() != Some(RESERVED_NAME_SETTINGS) &&
        self.file_name() != Some(RESERVED_NAME_NOTES) &&

        // These files either do not benefit from compression (video), may cause issues due to compression (audio not working)
        // or may cause overhead due to decompression (models).
        !matches!(self.file_type, FileType::Audio | FileType::Dat | FileType::RigidModel | FileType::SoundBank | FileType::Video) &&

        // We can only compress files if the game supports them. And only in WH3 (and newer games?) is the table compression bug fixed.
        (
            !matches!(self.file_type, FileType::DB | FileType::Loc) || (
                game_info.key() != KEY_PHARAOH_DYNASTIES &&
                game_info.key() != KEY_PHARAOH &&
                game_info.key() != KEY_TROY &&
                game_info.key() != KEY_THREE_KINGDOMS &&
                game_info.key() != KEY_WARHAMMER_2
            )
        )
    }

    /// This function guesses the [`FileType`] of the provided RFile and stores it on it for later queries.
    ///
    /// The way it works is: first it tries to guess it by extension (fast), then by full path (not as fast), then by data (slow and it may fail on lazy-loaded files).
    ///
    /// This may fail for some files, so if you doubt set the type manually.
    pub fn guess_file_type(&mut self) -> Result<()> {

        // First, try with extensions.
        let path = self.path.to_lowercase();

        if path.ends_with(pack::EXTENSION) {
            self.file_type = FileType::Pack;
        }

        else if path.ends_with(loc::EXTENSION) {
            self.file_type = FileType::Loc;
        }

        else if path.ends_with(rigidmodel::EXTENSION) {
            self.file_type = FileType::RigidModel
        }

        else if path.ends_with(animpack::EXTENSION) {
            self.file_type = FileType::AnimPack
        }

        else if path.ends_with(anim::EXTENSION) {
            self.file_type = FileType::Anim
        }

        else if path.ends_with(video::EXTENSION) {
            self.file_type = FileType::Video;
        }

        else if path.ends_with(dat::EXTENSION) {
            self.file_type = FileType::Dat;
        }

        else if path.ends_with(font::EXTENSION) {
            self.file_type = FileType::Font;
        }

        else if audio::EXTENSIONS.iter().any(|x| path.ends_with(x)) {
            self.file_type = FileType::Audio;
        }

        // TODO: detect bin files for maps and tile maps.
        else if bmd::EXTENSIONS.iter().any(|x| path.ends_with(x)) {
            self.file_type = FileType::BMD;
        }

        else if bmd_vegetation::EXTENSIONS.iter().any(|x| path.ends_with(x)) {
            self.file_type = FileType::BMDVegetation;
        }

        else if path.ends_with(sound_bank::EXTENSION) {
            self.file_type = FileType::SoundBank;
        }

        else if image::EXTENSIONS.iter().any(|x| path.ends_with(x)) {
            self.file_type = FileType::Image;
        }

        else if cfg!(feature = "support_uic") && path.starts_with(uic::BASE_PATH) && uic::EXTENSIONS.iter().any(|x| path.ends_with(x) || !path.contains('.')) {
            self.file_type = FileType::UIC;
        }

        else if path.ends_with(text::EXTENSION_VMD.0) {
            self.file_type = FileType::VMD;
        }

        else if path.ends_with(text::EXTENSION_WSMODEL.0) {
            self.file_type = FileType::WSModel;
        }

        else if text::EXTENSIONS.iter().any(|(x, _)| path.ends_with(x)) {
            self.file_type = FileType::Text;
        }

        else if path.ends_with(unit_variant::EXTENSION) {
            self.file_type = FileType::UnitVariant
        }

        else if path == group_formations::PATH {
            self.file_type = FileType::GroupFormations;
        }

        else if esf::EXTENSIONS.iter().any(|x| path.ends_with(x)) {
            self.file_type = FileType::ESF;
        }

        // If that failed, try types that need to be in a specific path.
        else if matched_combat::BASE_PATHS.iter().any(|x| path.starts_with(*x)) && path.ends_with(matched_combat::EXTENSION) {
            self.file_type = FileType::MatchedCombat;
        }

        else if path.starts_with(anims_table::BASE_PATH) && path.ends_with(anims_table::EXTENSION) {
            self.file_type = FileType::AnimsTable;
        }

        else if path.ends_with(anim_fragment_battle::EXTENSION_OLD) || (path.starts_with(anim_fragment_battle::BASE_PATH) && path.contains(anim_fragment_battle::MID_PATH) && path.ends_with(anim_fragment_battle::EXTENSION_NEW)) {
            self.file_type = FileType::AnimFragmentBattle;
        }

        // If that failed, check if it's in a folder which is known to only have specific files.
        // Microoptimization: check the path before using the regex. Regex is very, VERY slow.
        else if (path.starts_with("db/") && REGEX_DB.is_match(&path)) || (path.starts_with("ceo_db/") && REGEX_CEO_DB.is_match(&path))  {
            self.file_type = FileType::DB;
        }

        else if path.ends_with(portrait_settings::EXTENSION) && REGEX_PORTRAIT_SETTINGS.is_match(&path) {
            self.file_type = FileType::PortraitSettings;
        }

        else if path.ends_with(atlas::EXTENSION) {
            self.file_type = FileType::Atlas;
        }

        else if path.ends_with(hlsl_compiled::EXTENSION) {
            self.file_type = FileType::HlslCompiled;
        }

        else if path.ends_with(trade_routes::EXTENSION) {
            self.file_type = FileType::TradeRoutes;
        }

        // If we reach this... we're clueless. Leave it unknown.
        else {
            self.file_type = FileType::Unknown;
        }

        Ok(())
    }

    /// This function allows to import a TSV file on the provided Path into a binary database file.
    ///
    /// It requires the path on disk of the TSV file and the Schema to use. Schema is only needed for DB tables.
    pub fn tsv_import_from_path(path: &Path, schema: &Option<Schema>) -> Result<Self> {

        // We want the reader to have no quotes, tab as delimiter and custom headers, because otherwise
        // Excel, Libreoffice and all the programs that edit this kind of files break them on save.
        let mut reader = ReaderBuilder::new()
            .delimiter(b'\t')
            .quoting(false)
            .has_headers(true)
            .flexible(true)
            .from_path(path)?;

        // If we successfully load the TSV file into a reader, check the first line to get the column list and order.
        let field_order = reader.headers()?
            .iter()
            .enumerate()
            .map(|(x, y)| (x as u32, y.to_owned()))
            .collect::<HashMap<u32, String>>();

        // Get the record iterator so we can check the metadata from the second row.
        let mut records = reader.records();
        let (table_type, table_version, file_path) = match records.next() {
            Some(Ok(record)) => {
                let metadata = match record.get(0) {
                    Some(metadata) => metadata.split(';').map(|x| x.to_owned()).collect::<Vec<String>>(),
                    None => return Err(RLibError::ImportTSVWrongTypeTable),
                };

                let table_type = match metadata.first() {
                    Some(table_type) => {
                        let mut table_type = table_type.to_owned();
                        if table_type.starts_with('#') {
                            table_type.remove(0);
                        }
                        table_type
                    },
                    None => return Err(RLibError::ImportTSVWrongTypeTable),
                };

                let table_version = match metadata.get(1) {
                    Some(table_version) => table_version.parse::<i32>().map_err(|_| RLibError::ImportTSVInvalidVersion)?,
                    None => return Err(RLibError::ImportTSVInvalidVersion),
                };

                let file_path = match metadata.get(2) {
                    Some(file_path) => file_path.replace('\\', "/"),
                    None => return Err(RLibError::ImportTSVInvalidOrMissingPath),
                };

                (table_type, table_version, file_path)
            }
            Some(Err(_)) |
            None => return Err(RLibError::ImportTSVIncorrectRow(1, 0)),
        };

        // Once we get the metadata, we know what kind of file we have. Create it and pass the records.
        let decoded = match &*table_type {
            loc::TSV_NAME_LOC | loc::TSV_NAME_LOC_OLD => {
                let decoded = Loc::tsv_import(records, &field_order)?;
                RFileDecoded::Loc(decoded)
            }

            // Any other name is assumed to be a db table.
            _ => {
                match schema {
                    Some(schema) => {
                        let decoded = DB::tsv_import(records, &field_order, schema, &table_type, table_version)?;
                        RFileDecoded::DB(decoded)
                    },
                    None => return Err(RLibError::SchemaNotProvided),
                }
            }
        };

        let rfile = RFile::new_from_decoded(&decoded, 0, &file_path);
        Ok(rfile)
    }

    /// This function allows to export a RFile into a TSV file on disk.
    ///
    /// Only supported for DB and Loc files.
    pub fn tsv_export_to_path(&mut self, path: &Path, schema: &Schema, keys_first: bool) -> Result<()> {

        // Sanitize the path before creating the file
        let sanitized_path = sanitize_path(path);

        // Make sure the folder actually exists.
        let mut folder_path = sanitized_path.to_path_buf();
        folder_path.pop();
        DirBuilder::new().recursive(true).create(&folder_path)?;

        // We want the writer to have no quotes, tab as delimiter and custom headers, because otherwise
        // Excel, Libreoffice and all the programs that edit this kind of files break them on save.
        let mut writer = WriterBuilder::new()
            .delimiter(b'\t')
            .quote_style(QuoteStyle::Never)
            .has_headers(false)
            .flexible(true)
            .from_path(&sanitized_path)?;

        let mut extra_data = DecodeableExtraData::default();
        extra_data.set_schema(Some(schema));

        let extra_data = Some(extra_data);

        // If it fails in decoding, delete the tsv file.
        let file = self.decode(&extra_data, false, true);
        if let Err(error) = file {
            let _ = std::fs::remove_file(&sanitized_path);
            return Err(error);
        }

        let file = match file?.unwrap() {
            RFileDecoded::DB(table) => table.tsv_export(&mut writer, self.path_in_container_raw(), keys_first),
            RFileDecoded::Loc(table) => table.tsv_export(&mut writer, self.path_in_container_raw()),
            _ => Err(RLibError::ExportTSVUnsupportedFileType),
        };

        // If the tsv export failed, delete the tsv file.
        if file.is_err() {
            let _ = std::fs::remove_file(&sanitized_path);
        }

        file
    }

    /// This function tries to merge multiple files into one.
    ///
    /// All files must be of the same type and said type must support merging.
    pub fn merge(sources: &[&Self], path: &str) -> Result<Self> {
        if sources.len() <= 1 {
            return Err(RLibError::RFileMergeOnlyOneFileProvided);
        }

        let mut file_types = sources.iter().map(|file| file.file_type()).collect::<Vec<_>>();
        file_types.sort();
        file_types.dedup();

        if file_types.len() > 1 {
            return Err(RLibError::RFileMergeDifferentTypes);
        }

        match file_types[0] {
            FileType::DB => {
                let files = sources.iter().filter_map(|file| if let Ok(RFileDecoded::DB(table)) = file.decoded() { Some(table) } else { None }).collect::<Vec<_>>();
                let data = RFileDecoded::DB(DB::merge(&files)?);
                Ok(Self::new_from_decoded(&data, current_time()?, path))
            },
            FileType::Loc => {
                let files = sources.iter().filter_map(|file| if let Ok(RFileDecoded::Loc(table)) = file.decoded() { Some(table) } else { None }).collect::<Vec<_>>();
                let data = RFileDecoded::Loc(Loc::merge(&files)?);
                Ok(Self::new_from_decoded(&data, current_time()?, path))
            },
            _ => Err(RLibError::RFileMergeNotSupportedForType(file_types[0].to_string())),
        }
    }

    /// This function tries to update a file to a new version of the same file's format.
    ///
    /// Files used by this function are expected to be pre-decoded.
    pub fn update(&mut self, definition: &Option<Definition>) -> Result<()> {
        match self.decoded_mut() {
            Ok(RFileDecoded::DB(file)) => match definition {
                Some(definition) => file.update(definition),
                None => return Err(RLibError::RawTableMissingDefinition),
            }
            _ => return Err(RLibError::FileNotDecoded(self.path_in_container_raw().to_string())),
        }

        Ok(())
    }

    /// This function returns the data hash of the file.
    pub fn data_hash(&mut self, extra_data: &Option<EncodeableExtraData>) -> Result<u64> {
        Ok(match self.data {
            RFileInnerData::Decoded(_) => checksum(CrcAlgorithm::Crc32Iscsi, &self.encode(extra_data, false, false, true)?.unwrap()),
            RFileInnerData::Cached(ref data) => checksum(CrcAlgorithm::Crc32Iscsi, data),
            RFileInnerData::OnDisk(ref on_disk) => checksum(CrcAlgorithm::Crc32Iscsi, &on_disk.read()?),
        })
    }

    /// Sanitizes a destination path and creates a file with the sanitized path.
    /// Logs a message if the filename was changed due to invalid Windows characters.
    pub fn sanitize_and_create_file(&mut self, destination_path: &Path, extra_data: &Option<EncodeableExtraData>) -> Result<PathBuf> {
        let sanitized_destination_path = sanitize_path(destination_path);

        if sanitized_destination_path != destination_path {
            warn!("Filename sanitized from '{}' to '{}' due to invalid Windows characters",
                  destination_path.to_owned().file_name().unwrap_or_default().to_string_lossy(),
                  sanitized_destination_path.file_name().unwrap_or_default().to_string_lossy());
        }

        // Encode first, then create the destination file. This way, if the source happens to be
        // the same path we're writing to (e.g. MyMod Import/Export), we read the source bytes
        // before File::create truncates it. Avoids needing to pre-load OnDisk files into memory.
        let data = self.encode(extra_data, false, false, true)?.unwrap();
        let mut file = BufWriter::new(File::create(&sanitized_destination_path)?);
        file.write_all(&data)?;
        Ok(sanitized_destination_path)
    }

    /// Function to encode a file using data from a external path, effectively replacing its data.
    /// without replacing the file's metadata.
    ///
    /// NOTE: If TSV is detected and fails to import, this returns an error.
    pub fn encode_from_external_data(&mut self, schema: &Option<Schema>, external_path: &Path) -> Result<()> {

        let mut file = BufReader::new(File::open(external_path)?);
        let mut data = vec![];
        file.read_to_end(&mut data)?;

        // If we're dealing with a TSV, make sure to import it before setting up the data.
        match external_path.extension() {
            Some(extension) => {
                if extension.to_string_lossy() == "tsv" {
                    match RFile::tsv_import_from_path(external_path, schema) {
                        Ok(rfile) => self.set_decoded(rfile.decoded()?.clone())?,
                        Err(_) => self.set_cached(&data),
                    }
                } else {
                    self.set_cached(&data);
                }
            }
            None => self.set_cached(&data),
        }

        // If they're tables, make sure they're left decoded in memory.
        if self.file_type() == FileType::DB || self.file_type() == FileType::Loc {
            if let Some(ref schema) = schema {
                let mut extra_data = DecodeableExtraData::default();
                extra_data.set_schema(Some(schema));
                let extra_data = Some(extra_data);
                let _ = self.decode(&extra_data, true, false);
            }
        }

        Ok(())
    }
}

impl OnDisk {

    /// This function tries to read and return the raw data of an RFile.
    ///
    /// This returns the data uncompressed and unencrypted.
    fn read(&self) -> Result<Vec<u8>> {

        // Date check, to ensure the source file or container hasn't been modified since we got the indexes to read it.
        let mut file = BufReader::new(File::open(&self.path)?);
        let timestamp = last_modified_time_from_file(file.get_ref())?;
        if timestamp != self.timestamp {
            return Err(RLibError::FileSourceChanged(self.path.clone()));
        }

        // Read the data from disk.
        let mut data = vec![0; self.size as usize];
        file.seek(SeekFrom::Start(self.start))?;
        file.read_exact(&mut data)?;

        // If the data is encrypted, decrypt it.
        if let Some(pfh_version) = self.is_encrypted {
            data = Cursor::new(data).decrypt(pfh_version)?;
        }

        // If the data is compressed. decompress it.
        if self.is_compressed {
            data = match self.lz4_block_uncompressed_size {
                Some(uncompressed_size) => data.as_slice().decompress_lz4_block(uncompressed_size as usize)?,
                None => data.as_slice().decompress()?,
            };
        }

        Ok(data)
    }
}

impl ContainerPath {

    /// This function returns true if the provided [ContainerPath] corresponds to a file.
    pub fn is_file(&self) -> bool {
        matches!(self, ContainerPath::File(_))
    }

    /// This function returns true if the provided [ContainerPath] corresponds to a folder.
    pub fn is_folder(&self) -> bool {
        matches!(self, ContainerPath::Folder(_))
    }

    /// This function returns true if the provided [ContainerPath] corresponds to a root Pack.
    pub fn is_pack(&self) -> bool {
        match self {
            ContainerPath::Folder(path) => path.is_empty(),
            _ => false,
        }
    }

    /// This function returns a reference to the path stored within the provided [ContainerPath].
    pub fn path_raw(&self) -> &str {
        match self {
            Self::File(ref path) => path,
            Self::Folder(ref path) => path,
        }
    }

    /// This function returns the last item of the provided [ContainerPath], if any.
    pub fn name(&self) -> Option<&str> {
        self.path_raw().split('/').next_back()
    }

    /// This function the *table_name* of this file (the folder that contains this file) if this file is a DB table.
    ///
    /// It returns None of the file provided is not a DB Table.
    pub fn db_table_name_from_path(&self) -> Option<&str> {
        let split_path = self.path_raw().split('/').collect::<Vec<_>>();
        let start_lower = split_path[0].to_lowercase();
        if split_path.len() == 3 && (start_lower == "db" || start_lower == "ceo_db") {
            Some(split_path[1])
        } else {
            None
        }
    }

    /// This function returns the `db/` and `ceo_db/` folder paths for the provided *table_name*.
    ///
    /// Use it with `files_by_paths` instead of hardcoding the `db/` prefix.
    pub fn db_table_folders(table_name: &str) -> Vec<ContainerPath> {
        vec![
            ContainerPath::Folder(format!("db/{table_name}")),
            ContainerPath::Folder(format!("ceo_db/{table_name}")),
        ]
    }

    /// This function returns the path of the parent folder of the provided [ContainerPath].
    ///
    /// If the provided [ContainerPath] corresponds to a Container root, the path returned will be the current one.
    pub fn parent_path(&self) -> String {
        match self {
            ContainerPath::File(path) |
            ContainerPath::Folder(path) => {
                if path.is_empty() || (path.chars().count() == 1 && path.starts_with('/')) {
                    path.to_owned()
                } else {
                    let mut path_split = path.split('/').collect::<Vec<_>>();
                    path_split.pop();
                    path_split.join("/")
                }
            },
        }
    }

    /// This function removes collided items from the provided list of [ContainerPath].
    ///
    /// This means, if you have a file and a folder containing the file, it removes the file.
    pub fn dedup(paths: &[Self]) -> Vec<Self> {

        // As this operation can get very expensive very fast, we first check if we have a path containing the root of the container.
        let root = ContainerPath::Folder("".to_string());
        if paths.contains(&root) {
            return vec![root; 1];
        }

        // If we don't have the root of the container, second optimization: check if we have at least one folder.
        // If not, we just need to dedup the file list.
        if !paths.par_iter().any(|item| matches!(item, ContainerPath::Folder(_))) {
            let mut paths = paths.to_vec();
            paths.sort();
            paths.dedup();
            return paths;
        }

        // If we reached this point, we have a mix of files and folders, or only folders.
        // In any case, we need to filter them, then dedup the resultant paths.
        let items_to_remove = paths.par_iter().filter(|item_type_to_add| {
            match item_type_to_add {

                // If it's a file, we have to check if there is a folder containing it.
                ContainerPath::File(ref path_to_add) => {
                    !paths.par_iter()
                        .any(|item_type| {

                        // If the other one is a folder that contains it, dont add it.
                        item_type.is_folder() && path_to_add.starts_with(item_type.path_raw())
                    })
                }

                // If it's a folder, we have to check if there is already another folder containing it.
                ContainerPath::Folder(ref path_to_add) => {
                    !paths.par_iter()
                        .any(|item_type| {

                        // If the other one is a folder that contains it, dont add it.
                        let path = item_type.path_raw();
                        item_type.is_folder() && path_to_add.starts_with(path) && path_to_add.len() > path.len() && path_to_add.is_char_boundary(path.len()) && path_to_add.as_bytes()[path.len()] == b'/'
                    })
                }
            }
        }).cloned().collect::<Vec<Self>>();

        let mut paths = paths.to_vec();
        paths.retain(|x| items_to_remove.contains(x));
        paths.sort();
        paths.dedup();
        paths
    }
}

impl Ord for ContainerPath {
    fn cmp(&self, other: &Self) -> Ordering {
        match self {
            ContainerPath::File(a) => match other {
                ContainerPath::File(b) => a.cmp(b),
                ContainerPath::Folder(_) => Ordering::Less,
            }
            ContainerPath::Folder(a) => match other {
                ContainerPath::File(_) => Ordering::Greater,
                ContainerPath::Folder(b) => a.cmp(b),
            }
        }
    }
}

impl PartialOrd for ContainerPath {
    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
        Some(self.cmp(other))
    }
}

impl Display for FileType {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self {
            FileType::Anim => write!(f, "Anim"),
            FileType::AnimFragmentBattle => write!(f, "AnimFragmentBattle"),
            FileType::AnimPack => write!(f, "AnimPack"),
            FileType::AnimsTable => write!(f, "AnimsTable"),
            FileType::Atlas => write!(f, "Atlas"),
            FileType::Audio => write!(f, "Audio"),
            FileType::BMD => write!(f, "Battle Map Definition"),
            FileType::BMDVegetation => write!(f, "Battle Map Definition (Vegetation)"),
            FileType::Dat => write!(f, "Dat Audio File"),
            FileType::DB => write!(f, "DB Table"),
            FileType::ESF => write!(f, "ESF"),
            FileType::Font => write!(f, "Font"),
            FileType::HlslCompiled => write!(f, "Hlsl Compiled"),
            FileType::GroupFormations => write!(f, "Group Formations"),
            FileType::Image => write!(f, "Image"),
            FileType::Loc => write!(f, "Loc Table"),
            FileType::MatchedCombat => write!(f, "Matched Combat"),
            FileType::Pack => write!(f, "PackFile"),
            FileType::PortraitSettings => write!(f, "Portrait Settings"),
            FileType::RigidModel => write!(f, "RigidModel"),
            FileType::SoundBank => write!(f, "SoundBank"),
            FileType::Text => write!(f, "Text"),
            FileType::TradeRoutes => write!(f, "Trade Routes"),
            FileType::UIC => write!(f, "UI Component"),
            FileType::UnitVariant => write!(f, "Unit Variant"),
            FileType::Unknown => write!(f, "Unknown"),
            FileType::Video => write!(f, "Video"),
            FileType::VMD => write!(f, "VMD"),
            FileType::WSModel => write!(f, "WSModel"),
        }
    }
}

impl From<&str> for FileType {
    fn from(value: &str) -> Self {
        match value {
            "Anim" => FileType::Anim,
            "AnimFragmentBattle" => FileType::AnimFragmentBattle,
            "AnimPack" => FileType::AnimPack,
            "AnimsTable" => FileType::AnimsTable,
            "Atlas" => FileType::Atlas,
            "Audio" => FileType::Audio,
            "BMD" => FileType::BMD,
            "BMDVegetation" => FileType::BMDVegetation,
            "Dat" => FileType::Dat,
            "DB" => FileType::DB,
            "ESF" => FileType::ESF,
            "Font" => FileType::Font,
            "HlslCompiled" => FileType::HlslCompiled,
            "GroupFormations" => FileType::GroupFormations,
            "Image" => FileType::Image,
            "Loc" => FileType::Loc,
            "MatchedCombat" => FileType::MatchedCombat,
            "Pack" => FileType::Pack,
            "PortraitSettings" => FileType::PortraitSettings,
            "RigidModel" => FileType::RigidModel,
            "SoundBank" => FileType::SoundBank,
            "Text" => FileType::Text,
            "TradeRoutes" => FileType::TradeRoutes,
            "UIC" => FileType::UIC,
            "UnitVariant" => FileType::UnitVariant,
            "Unknown" => FileType::Unknown,
            "Video" => FileType::Video,
            "VMD" => FileType::VMD,
            "WSModel" => FileType::WSModel,
            _ => unimplemented!(),
        }
    }
}

impl From<FileType> for String {
    fn from(value: FileType) -> String {
        match value {
            FileType::Anim => "Anim",
            FileType::AnimFragmentBattle => "AnimFragmentBattle",
            FileType::AnimPack => "AnimPack",
            FileType::AnimsTable => "AnimsTable",
            FileType::Atlas => "Atlas",
            FileType::Audio => "Audio",
            FileType::BMD => "BMD",
            FileType::BMDVegetation => "BMD Vegetation",
            FileType::Dat => "Dat",
            FileType::DB => "DB",
            FileType::ESF => "ESF",
            FileType::Font => "Font",
            FileType::HlslCompiled => "HlslCompiled",
            FileType::GroupFormations => "GroupFormations",
            FileType::Image => "Image",
            FileType::Loc => "Loc",
            FileType::MatchedCombat => "MatchedCombat",
            FileType::Pack => "Pack",
            FileType::PortraitSettings => "PortraitSettings",
            FileType::RigidModel => "RigidModel",
            FileType::SoundBank => "SoundBank",
            FileType::Text => "Text",
            FileType::TradeRoutes => "TradeRoutes",
            FileType::UIC => "UIC",
            FileType::UnitVariant => "UnitVariant",
            FileType::Unknown => "Unknown",
            FileType::Video => "Video",
            FileType::VMD => "VMD",
            FileType::WSModel => "WSModel",
        }.to_owned()
    }
}

impl From<&RFileDecoded> for FileType {
    fn from(file: &RFileDecoded) -> Self {
        match file {
            RFileDecoded::Anim(_) => Self::Anim,
            RFileDecoded::AnimFragmentBattle(_) => Self::AnimFragmentBattle,
            RFileDecoded::AnimPack(_) => Self::AnimPack,
            RFileDecoded::AnimsTable(_) => Self::AnimsTable,
            RFileDecoded::Atlas(_) => Self::Atlas,
            RFileDecoded::Audio(_) => Self::Audio,
            RFileDecoded::BMD(_) => Self::BMD,
            RFileDecoded::BMDVegetation(_) => Self::BMDVegetation,
            RFileDecoded::Dat(_) => Self::Dat,
            RFileDecoded::DB(_) => Self::DB,
            RFileDecoded::ESF(_) => Self::ESF,
            RFileDecoded::Font(_) => Self::Font,
            RFileDecoded::HlslCompiled(_) => Self::HlslCompiled,
            RFileDecoded::GroupFormations(_) => Self::GroupFormations,
            RFileDecoded::Image(_) => Self::Image,
            RFileDecoded::Loc(_) => Self::Loc,
            RFileDecoded::MatchedCombat(_) => Self::MatchedCombat,
            RFileDecoded::Pack(_) => Self::Pack,
            RFileDecoded::PortraitSettings(_) => Self::PortraitSettings,
            RFileDecoded::RigidModel(_) => Self::RigidModel,
            RFileDecoded::SoundBank(_) => Self::SoundBank,
            RFileDecoded::Text(_) => Self::Text,
            RFileDecoded::TradeRoutes(_) => Self::TradeRoutes,
            RFileDecoded::UIC(_) => Self::UIC,
            RFileDecoded::UnitVariant(_) => Self::UnitVariant,
            RFileDecoded::Unknown(_) => Self::Unknown,
            RFileDecoded::Video(_) => Self::Video,
            RFileDecoded::VMD(_) => Self::VMD,
            RFileDecoded::WSModel(_) => Self::WSModel,
        }
    }
}

impl<'a> EncodeableExtraData<'a> {

    /// This functions generates an EncodeableExtraData for a specific game.
    pub fn new_from_game_info(game_info: &'a GameInfo) -> Self {
        let mut extra_data = Self::default();
        extra_data.set_game_info(Some(game_info));
        extra_data.set_table_has_guid(*game_info.db_tables_have_guid());
        extra_data
    }

    /// This functions generates an EncodeableExtraData for a specific game and settings.
    pub fn new_from_game_info_and_settings(game_info: &'a GameInfo, cf: CompressionFormat, disable_regen_table_guid: bool) -> EncodeableExtraData<'a> {
        let mut extra_data = EncodeableExtraData::new_from_game_info(game_info);
        extra_data.set_regenerate_table_guid(!disable_regen_table_guid);
        extra_data.set_compression_format(cf);
        extra_data
    }
}