From 008eae43813251a87bda72826a0e3843cfbafd2d Mon Sep 17 00:00:00 2001 From: Aaron Kimbrell Date: Tue, 23 Jun 2026 04:33:27 -0500 Subject: [PATCH 1/4] feat: parallel SD0 compression, finalize subcommand, and PKI improvements - Add parallel compression using rayon with configurable thread count (-j) and compression level (-c), defaulting to fastest (level 1) - Add `finalize` subcommand for post-processing cache: filters trunk manifest, generates frontend.txt, copies patcher.ini, and creates version/index/hotfix manifests - Improve PKI config parsing with support for dir spec format (directory=recursive=filter) and locale-aware pack generation - Normalize virtual paths to lowercase for case-insensitive consistency - Use local path dependency for assembly-pack to pick up compression changes - Improve error context in pack subcommand with per-file error messages Co-Authored-By: Claude Sonnet 4.6 --- Cargo.lock | 88 ++++++++++-- Cargo.toml | 3 +- src/cache/mod.rs | 220 +++++++++++++++++++++-------- src/cache/quickcheck.rs | 5 +- src/finalize.rs | 302 ++++++++++++++++++++++++++++++++++++++++ src/main.rs | 5 + src/pack.rs | 6 +- src/pki.rs | 126 ++++++++++++++--- 8 files changed, 663 insertions(+), 92 deletions(-) create mode 100644 src/finalize.rs diff --git a/Cargo.lock b/Cargo.lock index 28146cd..5841189 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" @@ -78,8 +78,6 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "assembly-pack" version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7516a74139653ba7d79db9a3139cdd4e5bcf6ee66d71ed8868362e598eb5f591" dependencies = [ "adler32", "crc", @@ -88,6 +86,7 @@ dependencies = [ "md5", "nom", "nom-supreme", + "rustc_version", "serde", "thiserror", ] @@ -126,9 +125,9 @@ dependencies = [ [[package]] name = "brownstone" -version = "1.1.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "030ea61398f34f1395ccbeb046fb68c87b631d1f34567fed0f0f11fa35d18d8d" +checksum = "c5839ee4f953e811bfdcf223f509cb2c6a3e1447959b0bff459405575bc17f22" dependencies = [ "arrayvec", ] @@ -184,18 +183,18 @@ dependencies = [ [[package]] name = "crc" -version = "2.1.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49fc9a695bca7f35f5f4c15cddc84415f66a74ea78eef08e90c5024f2b540e23" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" dependencies = [ "crc-catalog", ] [[package]] name = "crc-catalog" -version = "1.1.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccaeedb56da03b09f598226e25e80088cb4cd25f316e6e4df7d695f0feeb1403" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crc32fast" @@ -206,6 +205,37 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + [[package]] name = "env_logger" version = "0.9.3" @@ -343,6 +373,7 @@ dependencies = [ "globset", "indexmap", "log", + "rayon", "serde", "toml", ] @@ -395,9 +426,9 @@ dependencies = [ [[package]] name = "nom-supreme" -version = "0.6.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aadc66631948f6b65da03be4c4cd8bd104d481697ecbb9bbd65719b1ec60bc9f" +checksum = "2bd3ae6c901f1959588759ff51c95d24b491ecb9ff91aa9c2ef4acc5b1dcab27" dependencies = [ "brownstone", "indent_write", @@ -451,6 +482,26 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "regex" version = "1.8.4" @@ -474,6 +525,21 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + [[package]] name = "serde" version = "1.0.164" diff --git a/Cargo.toml b/Cargo.toml index cbdb111..5869a03 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" [dependencies] argh = "0.1.7" -assembly-pack = { version = "0.5.2", features = [ +assembly-pack = { path = "../Assembly/modules/pack", features = [ "pk", "pki", "sd0", @@ -17,6 +17,7 @@ assembly-pack = { version = "0.5.2", features = [ color-eyre = "0.5.11" env_logger = "0.9" globset = "0.4.8" +rayon = "1.7" indexmap = { version = "1.8.0", features = ["serde-1"] } log = "0.4" serde = "1.0" diff --git a/src/cache/mod.rs b/src/cache/mod.rs index c9f00f6..3fd1538 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -4,18 +4,20 @@ use assembly_pack::{ fs::{scan_dir, FileInfo, FsVisitor}, FileMetaPair, }, - crc::calculate_crc, + crc::{calculate_crc, CRC}, md5::{self, MD5Sum}, - sd0::fs::Converter, - txt::{FileLine, Manifest, VersionLine}, + sd0::fs::{Compression, Converter}, + txt::{FileLine, FileMeta, Manifest, VersionLine}, }; use color_eyre::eyre::Context; use globset::{Glob, GlobSet, GlobSetBuilder}; +use rayon::prelude::*; use std::{ collections::BTreeMap, fs::{File, Metadata}, io::{BufRead, BufReader, BufWriter, ErrorKind, Read, Seek, SeekFrom}, path::{Path, PathBuf}, + sync::atomic::{AtomicUsize, Ordering}, time::{Duration, UNIX_EPOCH}, }; @@ -51,6 +53,14 @@ pub struct Args { /// name of a file containing one path per line #[argh(option, short = 'F')] files: Option, + + /// compression level 0-9 (default: 1, fastest; 9 = smallest but slowest) + #[argh(option, short = 'c', default = "1")] + compression: u32, + + /// number of parallel compression threads (default: all cores) + #[argh(option, short = 'j')] + jobs: Option, } fn hash_to_path(hash: &MD5Sum) -> String { @@ -65,52 +75,47 @@ fn hash_to_path(hash: &MD5Sum) -> String { #[derive(Default, Debug)] struct Stats { quickcheck: usize, + cached_sd0: usize, + unchanged: usize, compress: usize, - updated: usize, total: usize, ignored: usize, } +/// A file that needs SD0 compression (queued during scan, processed in parallel) +struct PendingCompression { + path: String, + input: PathBuf, + outpath: PathBuf, + raw_meta: FileMeta, + mtime: Option, +} + +/// Result of a successful parallel compression +struct CompressResult { + path: String, + meta_pair: FileMetaPair, + linesum: MD5Sum, + mtime: Option, + raw_meta: FileMeta, +} + struct Visitor { stats: Stats, include_glob: GlobSet, exclude_glob: GlobSet, - quickcheck: BTreeMap, + quickcheck: BTreeMap, quickcheck_out: BufWriter, - conv: Converter, output: PathBuf, /// The previous manifest prev: BTreeMap, /// The new manifest manifest: Manifest, + /// Files queued for parallel compression + pending: Vec, } impl Visitor { - fn compress(&mut self, input: &Path, outpath: &Path) -> Option { - // Continue with conversion if it was just not found - let parent = outpath.parent().unwrap(); - if let Err(e) = std::fs::create_dir_all(parent) { - log::error!("Failed to create dir {}:\n\t{}", parent.display(), e); - return None; - } - log::info!("Converting {} to {}", input.display(), outpath.display()); - match self.conv.convert_file(input, &outpath) { - Err(e) => { - log::error!( - "Error converting {} to {}:\n\t{}", - input.display(), - outpath.display(), - e - ); - return None; - } - Ok(line) => { - self.stats.compress += 1; - Some(line) - } - } - } - fn visit(&mut self, path: String, input: &Path, meta: Option) { if !self.include_glob.is_match(&path) || self.exclude_glob.is_match(&path) { self.stats.ignored += 1; @@ -128,7 +133,6 @@ impl Visitor { let quickcheck = self.quickcheck.remove(&crc); let in_meta = match quickcheck { - // FIXME: size check Some(qc) if (mtime.is_some() && qc.mtime == mtime) => { self.stats.quickcheck += 1; qc.meta @@ -146,7 +150,6 @@ impl Visitor { let mut meta_pair = old_meta_pair.filter(|(p, _)| p.raw == in_meta); if let (Some(old), None) = (old_meta_pair.as_ref(), meta_pair.as_ref()) { - self.stats.updated += 1; log::debug!( "File {} was updated from {} to {}", path, @@ -158,25 +161,37 @@ impl Visitor { let outpath = self.output.join(hash_to_path(&in_meta.hash)); if meta_pair.is_none() { - let line = match md5::md5sum(&outpath) { - Ok(meta) => FileMetaPair { - raw: in_meta, - compressed: meta, - }, + match md5::md5sum(&outpath) { + Ok(compressed_meta) => { + // SD0 file already exists in cache, reuse it + self.stats.cached_sd0 += 1; + let line = FileMetaPair { + raw: in_meta, + compressed: compressed_meta, + }; + let linesum = md5::MD5Sum::compute(&format!("{path},{line}")); + meta_pair = Some((line, linesum)); + } Err(e) => { if e.kind() != ErrorKind::NotFound { log::error!("Failed to access {}:\n\t{}", outpath.display(), e); return; } - let Some(meta_pair) = self.compress(input, &outpath) else { - return - }; - meta_pair + // Queue for parallel compression + self.pending.push(PendingCompression { + path, + input: input.to_owned(), + outpath, + raw_meta: in_meta, + mtime, + }); + return; // will be handled after parallel compression } }; - let linesum = md5::MD5Sum::compute(&format!("{path},{line}")); - meta_pair = Some((line, linesum)); + } else { + self.stats.unchanged += 1; } + if let Some((meta_pair, linesum)) = meta_pair { let qc = QuickCheck { path: path.clone(), @@ -184,7 +199,6 @@ impl Visitor { meta: in_meta, }; qc.write(&mut self.quickcheck_out).unwrap(); - self.manifest.files.insert(path, (meta_pair, linesum)); } } @@ -206,7 +220,6 @@ impl Visitor { let meta = match std::fs::metadata(&real) { Ok(meta) => Some(meta), Err(e) if e.kind() == ErrorKind::NotFound => { - // If the file is explicitly listed but was no found, remove it log::warn!("File {:?} not found!", path); let crc = calculate_crc(path.as_bytes()); if let Some(_) = self.quickcheck.remove(&crc) { @@ -215,7 +228,7 @@ impl Visitor { if let Some(_) = self.prev.remove(&path) { log::info!("Removed {:?} from manifest", path); } - return; // don't visit this file + return; } Err(e) => { log::debug!("Failed to get file metadata: {}", e); @@ -225,7 +238,6 @@ impl Visitor { self.visit(path, &real, meta); } - /// Small generic function that calls [`do_scan_file`] on every line of its input fn do_scan_files( &mut self, file_list_reader: R, @@ -234,7 +246,7 @@ impl Visitor { ) -> color_eyre::Result<()> { let files = BufReader::new(file_list_reader); let strip_prefix = match relative { - true => "", // don't try to strip a prefix on relative paths + true => "", false => &paths.strip_prefix, }; for line in files.lines() { @@ -258,11 +270,82 @@ impl Visitor { self.do_scan_files(file_list_reader, paths, relative) } } + + /// Process all queued files in parallel, then merge results into the manifest. + fn flush_pending(&mut self, compression_level: u32) { + if self.pending.is_empty() { + return; + } + + let count = self.pending.len(); + log::info!( + "Compressing {} files in parallel (level {})", + count, + compression_level + ); + + let compressed_count = AtomicUsize::new(0); + + let results: Vec> = self + .pending + .par_iter() + .map(|p| { + let i = compressed_count.fetch_add(1, Ordering::Relaxed) + 1; + log::info!("[{}/{}] Compressing {}", i, count, p.input.display()); + + if let Some(parent) = p.outpath.parent() { + let _ = std::fs::create_dir_all(parent); + } + + let conv = Converter { + generate_segment_index: false, + compression: Some(Compression::new(compression_level)), + }; + let pair = conv + .convert_file(&p.input, &p.outpath) + .map_err(|e| { + log::error!("Failed to compress {}: {}", p.input.display(), e); + e + }) + .ok()?; + + let linesum = MD5Sum::compute(&format!("{},{}", p.path, pair)); + + Some(CompressResult { + path: p.path.clone(), + meta_pair: pair, + linesum, + mtime: p.mtime, + raw_meta: p.raw_meta, + }) + }) + .collect(); + + // Merge results back sequentially + let pending = std::mem::take(&mut self.pending); + let mut compress_count = 0usize; + for (_, result) in pending.into_iter().zip(results.into_iter()) { + if let Some(r) = result { + compress_count += 1; + let qc = QuickCheck { + path: r.path.clone(), + mtime: r.mtime, + meta: r.raw_meta, + }; + qc.write(&mut self.quickcheck_out).unwrap(); + self.manifest.files.insert(r.path, (r.meta_pair, r.linesum)); + } + } + self.stats.compress += compress_count; + log::info!("Compressed {} files", compress_count); + } } impl FsVisitor for Visitor { - fn visit_file(&mut self, info: FileInfo) { - self.visit(info.path(), info.real(), info.metadata().ok()) + fn visit_file(&mut self, info: F) { + // Normalize virtual paths to lowercase for case-insensitive consistency + // (LU is a Windows game; all paths should be case-insensitive) + self.visit(info.path().to_lowercase(), info.real(), info.metadata().ok()) } } @@ -287,6 +370,22 @@ fn exclude_glob(project: &ProjectConfig) -> Result { } pub fn run(args: ProjectArgs) -> color_eyre::Result<()> { + let compression_level = args.cmd.compression; + if compression_level > 9 { + return Err(color_eyre::eyre::eyre!( + "Compression level must be 0-9, got {}", + compression_level + )); + } + + // Configure rayon thread pool + if let Some(jobs) = args.cmd.jobs { + rayon::ThreadPoolBuilder::new() + .num_threads(jobs) + .build_global() + .ok(); // ignore if already initialized + } + let paths = args.paths(); let quickcheck_path = paths @@ -329,6 +428,12 @@ pub fn run(args: ProjectArgs) -> color_eyre::Result<()> { _ => BTreeMap::new(), }; + log::info!( + "Compression level: {}, threads: {}", + compression_level, + args.cmd.jobs.unwrap_or_else(|| rayon::current_num_threads()) + ); + let mut visitor = Visitor { include_glob, exclude_glob, @@ -340,10 +445,8 @@ pub fn run(args: ProjectArgs) -> color_eyre::Result<()> { version, files: BTreeMap::new(), }, - conv: Converter { - generate_segment_index: false, - }, output, + pending: Vec::new(), }; log::info!("Scanning {} as {}", proj_dir.display(), paths.prefix); @@ -351,20 +454,23 @@ pub fn run(args: ProjectArgs) -> color_eyre::Result<()> { if let Some(file_list_path) = args.cmd.files { visitor.scan_files(&file_list_path, &paths, args.cmd.relative)?; // Write out untouched manifest files - for (key, value) in visitor.prev { + for (key, value) in std::mem::take(&mut visitor.prev) { visitor.manifest.files.insert(key, value); } // Write out untouched quickcheck files - for (_key, value) in visitor.quickcheck { + for (_key, value) in std::mem::take(&mut visitor.quickcheck) { value.write(&mut visitor.quickcheck_out)?; } } else { scan_dir(&mut visitor, paths.prefix, &proj_dir, true); - for (k, _v) in visitor.prev { + for (k, _v) in &visitor.prev { log::info!("File {} was removed", k); } } + // Process all queued compressions in parallel + visitor.flush_pending(compression_level); + manifest::write_manifest(visitor.manifest, &manifest).context("Failed to write manifest")?; log::info!("{:?}", visitor.stats); diff --git a/src/cache/quickcheck.rs b/src/cache/quickcheck.rs index 1dd924c..b46c02d 100644 --- a/src/cache/quickcheck.rs +++ b/src/cache/quickcheck.rs @@ -4,8 +4,9 @@ use std::{ io::{self, BufRead, BufReader, BufWriter, Write}, }; -use assembly_pack::{crc::calculate_crc, md5::MD5Sum, txt::FileMeta}; +use assembly_pack::{crc::{calculate_crc, CRC}, md5::MD5Sum, txt::FileMeta}; +#[derive(Clone)] pub(super) struct QuickCheck { pub path: String, pub mtime: Option, @@ -27,7 +28,7 @@ impl QuickCheck { } } -pub(super) fn scan_quickcheck(reader: &mut R) -> BTreeMap { +pub(super) fn scan_quickcheck(reader: &mut R) -> BTreeMap { let mut quickcheck = BTreeMap::new(); let mut reader = BufReader::new(reader); let mut buffer = String::new(); diff --git a/src/finalize.rs b/src/finalize.rs new file mode 100644 index 0000000..8832a29 --- /dev/null +++ b/src/finalize.rs @@ -0,0 +1,302 @@ +use std::{ + fs::{self, File}, + io::{BufRead, BufReader, BufWriter, Write}, + path::{Path, PathBuf}, +}; + +use argh::FromArgs; +use assembly_pack::{ + md5::{self, MD5Sum}, + sd0::fs::{Compression, Converter}, +}; +use color_eyre::eyre::Context; +use globset::{Glob, GlobSet, GlobSetBuilder}; + +use crate::ProjectArgs; + +const CRLF: &str = "\r\n"; + +/// Default frontend file patterns. +/// The working cache includes all trunk entries with a file extension. +/// `**/*.*` matches any file containing a `.` at any directory depth. +const DEFAULT_FRONTEND_PATTERNS: &[&str] = &["**/*.*"]; + +const EXCLUDE_PATTERNS: &[&str] = &[ + ".git", + "generator_config.txt", + "generator_config_ci.txt", + "README.md", +]; + +#[derive(FromArgs, PartialEq, Debug)] +#[argh(subcommand, name = "finalize")] +/// post-process cache: filter trunk, generate frontend/version manifests, copy patcher +pub struct Args { + /// version number + #[argh(option, short = 'v', default = "1")] + version: u32, + + /// version name + #[argh(option, short = 'n')] + name: Option, + + /// path to patcher directory (default: "patcher") + #[argh(option, default = "PathBuf::from(\"patcher\")")] + patcher: PathBuf, + + /// path to file with frontend glob patterns (one per line) + #[argh(option)] + frontend_patterns: Option, + + /// compression level 0-9 (default: 1, fastest; 9 = smallest but slowest) + #[argh(option, short = 'c', default = "1")] + compression: u32, +} + +fn hash_to_path(hash: &MD5Sum) -> String { + let hash_str = format!("{}", hash); + let mut chars = hash_str.chars(); + let c1 = chars.next().unwrap(); + let c2 = chars.next().unwrap(); + format!("{}/{}/{}.sd0", c1, c2, hash_str) +} + +/// Write a manifest file with CRLF line endings (matching LU client expectations) +fn write_manifest_crlf( + path: &Path, + version_num: u32, + version_name: &str, + lines: &[String], +) -> color_eyre::Result<()> { + let file = File::create(path) + .wrap_err_with(|| format!("Failed to create {}", path.display()))?; + let mut w = BufWriter::new(file); + + let vnum_hash = MD5Sum::compute(&version_num.to_string()); + write!(w, "[version]{CRLF}")?; + write!(w, "{},{},{}{CRLF}", version_num, vnum_hash, version_name)?; + write!(w, "[files]")?; + for line in lines { + write!(w, "{CRLF}{line}")?; + } + + Ok(()) +} + +fn should_exclude(line: &str) -> bool { + EXCLUDE_PATTERNS.iter().any(|pat| line.contains(pat)) +} + +/// Read a manifest file, skipping the 3-line header, returning entry lines +fn read_manifest_entries(path: &Path) -> color_eyre::Result> { + let file = File::open(path) + .wrap_err_with(|| format!("Failed to open {}", path.display()))?; + let reader = BufReader::new(file); + let mut lines_iter = reader.lines(); + + // Skip header: [version], version line, [files] + for _ in 0..3 { + lines_iter.next(); + } + + let mut entries = Vec::new(); + for line_result in lines_iter { + let line = line_result?; + let trimmed = line.trim().to_string(); + if !trimmed.is_empty() { + entries.push(trimmed); + } + } + + Ok(entries) +} + +/// Filter trunk.txt: normalize paths, lowercase, remove excluded entries, recalculate line hashes +fn filter_trunk( + cache_dir: &Path, + manifest_name: &str, + version_num: u32, + version_name: &str, +) -> color_eyre::Result<()> { + let trunk_path = cache_dir.join(manifest_name).with_extension("txt"); + log::info!("Filtering {}", trunk_path.display()); + + let entries = read_manifest_entries(&trunk_path)?; + let mut filtered = Vec::new(); + + for entry in entries { + // Normalize: backslash to forward slash, lowercase + let entry = entry.replace('\\', "/").to_lowercase(); + + if should_exclude(&entry) { + continue; + } + + // Take first 5 fields, recalculate line hash + let fields: Vec<&str> = entry.split(',').collect(); + if fields.len() < 5 { + continue; + } + let first_five = fields[..5].join(","); + let line_hash = MD5Sum::compute(&first_five); + filtered.push(format!("{first_five},{line_hash}")); + } + + log::info!("Filtered trunk: {} entries", filtered.len()); + write_manifest_crlf(&trunk_path, version_num, version_name, &filtered)?; + + Ok(()) +} + +/// Generate frontend.txt by filtering trunk entries against frontend patterns +fn generate_frontend( + cache_dir: &Path, + manifest_name: &str, + version_num: u32, + version_name: &str, + patterns: &GlobSet, +) -> color_eyre::Result<()> { + let trunk_path = cache_dir.join(manifest_name).with_extension("txt"); + let frontend_path = cache_dir.join("frontend.txt"); + + log::info!("Generating {}", frontend_path.display()); + + let entries = read_manifest_entries(&trunk_path)?; + let mut frontend_lines = Vec::new(); + + for entry in entries { + // Extract file path (first field before comma) + if let Some(path) = entry.split(',').next() { + if patterns.is_match(path) { + frontend_lines.push(entry); + } + } + } + + log::info!("Frontend: {} entries", frontend_lines.len()); + write_manifest_crlf(&frontend_path, version_num, version_name, &frontend_lines)?; + + Ok(()) +} + +/// Copy patcher.ini to cache dir +fn copy_patcher(patcher_dir: &Path, cache_dir: &Path) -> color_eyre::Result<()> { + let src = patcher_dir.join("patcher.ini"); + let dst = cache_dir.join("patcher.ini"); + log::info!("Copying {} to {}", src.display(), dst.display()); + fs::copy(&src, &dst) + .wrap_err_with(|| format!("Failed to copy {} to {}", src.display(), dst.display()))?; + Ok(()) +} + +/// Hash and compress specific files, writing a version manifest +fn make_version( + cache_dir: &Path, + filename: &str, + version_num: u32, + version_name: &str, + files: &[&str], + compression_level: u32, +) -> color_eyre::Result<()> { + let output_path = cache_dir.join(filename); + log::info!("Generating {}", output_path.display()); + + let mut lines = Vec::new(); + + for &file in files { + let file_path = cache_dir.join(file); + + // Get raw hash to determine SD0 output path + let raw_meta = md5::md5sum(&file_path) + .wrap_err_with(|| format!("Failed to hash {}", file_path.display()))?; + + let compressed_path = cache_dir.join(hash_to_path(&raw_meta.hash)); + + log::info!("Compressing {}", file_path.display()); + let conv = Converter { + generate_segment_index: false, + compression: Some(Compression::new(compression_level)), + }; + let pair = conv + .convert_file(&file_path, &compressed_path) + .wrap_err_with(|| format!("Failed to compress {}", file_path.display()))?; + + // Build manifest line with forward slashes + let file_normalized = file.replace('\\', "/"); + let line_content = format!("{},{}", file_normalized, pair); + let line_hash = MD5Sum::compute(&line_content); + lines.push(format!("{line_content},{line_hash}")); + } + + write_manifest_crlf(&output_path, version_num, version_name, &lines)?; + Ok(()) +} + +fn build_frontend_patterns(patterns_file: Option<&Path>) -> color_eyre::Result { + let mut builder = GlobSetBuilder::new(); + + if let Some(path) = patterns_file { + let file = File::open(path) + .wrap_err_with(|| format!("Failed to open patterns file {}", path.display()))?; + let reader = BufReader::new(file); + for line in reader.lines() { + let line = line?; + let trimmed = line.trim(); + if !trimmed.is_empty() && !trimmed.starts_with('#') { + builder.add(Glob::new(trimmed)?); + } + } + } else { + for pattern in DEFAULT_FRONTEND_PATTERNS { + builder.add(Glob::new(pattern)?); + } + } + + Ok(builder.build()?) +} + +pub fn run(args: ProjectArgs) -> color_eyre::Result<()> { + let paths = args.paths(); + let cache_dir = &paths.cache_dir; + let manifest_name = args.project.manifest.to_str().unwrap_or("trunk"); + + let vnum = args.cmd.version; + let vname = args.cmd.name.unwrap_or_else(|| vnum.to_string()); + + let pki_name = format!( + "{}.pki", + args.project.pki.to_str().unwrap_or("primary") + ); + + let compression_level = args.cmd.compression; + let frontend_patterns = build_frontend_patterns(args.cmd.frontend_patterns.as_deref())?; + + log::info!("[Finalize] Filtering trunk"); + filter_trunk(cache_dir, manifest_name, vnum, &vname)?; + + log::info!("[Finalize] Generating frontend"); + generate_frontend(cache_dir, manifest_name, vnum, &vname, &frontend_patterns)?; + + log::info!("[Finalize] Copying patcher.ini"); + copy_patcher(&args.cmd.patcher, cache_dir)?; + + log::info!("[Finalize] Generating hotfix.txt"); + make_version(cache_dir, "hotfix.txt", vnum, &vname, &[], compression_level)?; + + log::info!("[Finalize] Generating index.txt"); + make_version( + cache_dir, + "index.txt", + vnum, + &vname, + &["frontend.txt", &pki_name, &format!("{manifest_name}.txt")], + compression_level, + )?; + + log::info!("[Finalize] Generating version.txt"); + make_version(cache_dir, "version.txt", vnum, &vname, &["index.txt"], compression_level)?; + + log::info!("[Finalize] Done"); + + Ok(()) +} diff --git a/src/main.rs b/src/main.rs index 2114860..b3c8bcb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,6 +7,7 @@ use log::LevelFilter; mod cache; mod config; +mod finalize; mod pack; mod pki; @@ -24,6 +25,7 @@ struct Args { #[argh(subcommand)] pub enum Commands { Cache(cache::Args), + Finalize(finalize::Args), Pack(pack::Args), PKI(pki::Args), } @@ -162,6 +164,9 @@ fn main() -> color_eyre::Result<()> { Commands::Cache(cmd) => { cache::run(ProjectArgs::new(dir, config.general, project, name, cmd)) } + Commands::Finalize(cmd) => { + finalize::run(ProjectArgs::new(dir, config.general, project, name, cmd)) + } Commands::Pack(cmd) => pack::run(ProjectArgs::new(dir, config.general, project, name, cmd)), Commands::PKI(cmd) => pki::run(ProjectArgs::new(dir, config.general, project, name, cmd)), } diff --git a/src/pack.rs b/src/pack.rs index e0c7a28..828ea22 100644 --- a/src/pack.rs +++ b/src/pack.rs @@ -124,8 +124,6 @@ pub fn run(args: ProjectArgs) -> color_eyre::Result<()> { }); let is_compressed = lookup.category & 0xFF > 0; - let raw = file.raw; - let compressed = file.compressed; let path = if is_compressed { output.join(file.to_path()) @@ -134,8 +132,10 @@ pub fn run(args: ProjectArgs) -> color_eyre::Result<()> { win_join(&paths.proj_dir, relative_name) }; + log::info!("Packing '{}' from '{}' (compressed={})", name, path.display(), is_compressed); let mut writer = Writer { path: &path }; - pk.put_file(crc, &mut writer, raw, compressed, is_compressed)?; + pk.put_file(crc, &mut writer, file, is_compressed) + .with_context(|| format!("Failed to pack '{}' from '{}'", name, path.display()))?; } } } diff --git a/src/pki.rs b/src/pki.rs index cce220b..3b35f7f 100644 --- a/src/pki.rs +++ b/src/pki.rs @@ -7,6 +7,7 @@ use color_eyre::eyre::Context; use indexmap::IndexMap; use serde::Deserialize; use std::{ + collections::BTreeMap, ffi::OsStr, fs::File, io::{BufRead, BufReader, BufWriter}, @@ -51,42 +52,108 @@ fn hidden_glob(filename: &str) -> Option { None } +/// Parse a dir spec string in the format `directory[=recursive[=filter]]`. +/// +/// Examples: +/// - `mesh\\env` → directory=mesh\env, recursive=true, no filter +/// - `mesh\\env=0=re_*` → directory=mesh\env, recursive=false, filter=re_* +/// - `brickmodels\\pettaming=1` → directory=brickmodels\pettaming, recursive=true +fn parse_dir_spec(spec: &str) -> DirSpec { + let parts: Vec<&str> = spec.splitn(3, '=').collect(); + let directory = parts[0].to_lowercase(); + let recurse = match parts.get(1) { + Some(r) => *r != "0", + None => true, + }; + let filter = parts + .get(2) + .map(|f| f.to_lowercase()) + .unwrap_or_default(); + + DirSpec { + directory, + recurse_subdirectories: recurse, + filter_wildcard: filter, + } +} + +/// Detect a locale directory pattern in a path (e.g. `_loc\en_gb`, `_loc\de_de`). +/// Returns the locale segment like `_loc\de_de` if found. +fn detect_locale(path: &str) -> Option { + let lower = path.to_lowercase(); + let idx = lower.find("_loc\\")?; + let after = &lower[idx + 5..]; + if after.len() < 5 { + return None; + } + let locale = &after[..5]; + let chars: Vec = locale.chars().collect(); + if chars[0].is_alphanumeric() + && chars[1].is_alphanumeric() + && chars[2] == '_' + && chars[3].is_alphanumeric() + && chars[4].is_alphanumeric() + { + Some(format!("_loc\\{}", locale)) + } else { + None + } +} + +/// Remap a pack name to include a locale segment after the first component. +/// `pack\front2_3.pk` + `_loc\de_de` → `pack\_loc\de_de\front2_3.pk` +fn localize_pack_name(pack_name: &str, locale: &str) -> String { + if let Some(idx) = pack_name.find('\\') { + format!("{}\\{}\\{}", &pack_name[..idx], locale, &pack_name[idx + 1..]) + } else { + format!("{}\\{}", locale, pack_name) + } +} + fn process_cfg(config: &mut Config, cfg: Cfg) { + // Collect locale-specific files to emit as separate packs at the end + let mut locale_packs: BTreeMap)> = BTreeMap::new(); + for (k, v) in cfg.pack { + let pack_name = format!("pack\\{}.pk", k).to_lowercase(); + let cmd = Command::Pack { - filename: format!("pack\\{}.pk", k), + filename: pack_name.clone(), force_compression: v.compress, }; push_command(config, cmd); for dir in v.dirs { - let cmd = Command::AddDir(DirSpec { - directory: dir, - recurse_subdirectories: true, - filter_wildcard: String::new(), - }); - push_command(config, cmd); + push_command(config, Command::AddDir(parse_dir_spec(&dir))); } for dir in v.exclude_dirs { - let cmd = Command::RemDir(DirSpec { - directory: dir, - recurse_subdirectories: true, - filter_wildcard: String::new(), - }); - push_command(config, cmd); + push_command(config, Command::RemDir(parse_dir_spec(&dir))); } for filename in v.files { - let cmd = if let Some(dir) = hidden_glob(&filename) { - Command::AddDir(dir) + let filename = filename.to_lowercase(); + if let Some(locale) = detect_locale(&filename) { + // Route locale-specific files to their own pack + let locale_pack = localize_pack_name(&pack_name, &locale); + log::debug!("Routing {} to locale pack {}", filename, locale_pack); + locale_packs + .entry(locale_pack) + .or_insert_with(|| (v.compress, Vec::new())) + .1 + .push(filename); } else { - Command::AddFile { filename } - }; - push_command(config, cmd); + let cmd = if let Some(dir) = hidden_glob(&filename) { + Command::AddDir(dir) + } else { + Command::AddFile { filename } + }; + push_command(config, cmd); + } } for filename in v.exclude_files { + let filename = filename.to_lowercase(); let cmd = if let Some(dir) = hidden_glob(&filename) { Command::RemDir(dir) } else { @@ -97,6 +164,29 @@ fn process_cfg(config: &mut Config, cfg: Cfg) { push_command(config, Command::EndPack); } + + // Emit locale-specific packs + for (locale_pack_name, (compress, files)) in locale_packs { + if files.is_empty() { + continue; + } + log::info!( + "Generating locale pack {} with {} files", + locale_pack_name, + files.len() + ); + push_command( + config, + Command::Pack { + filename: locale_pack_name, + force_compression: compress, + }, + ); + for filename in files { + push_command(config, Command::AddFile { filename }); + } + push_command(config, Command::EndPack); + } } pub fn run(args: ProjectArgs) -> color_eyre::Result<()> { From 77461be56e6f4f87da977354a955dcabc32b01cf Mon Sep 17 00:00:00 2001 From: Aaron Kimbrell Date: Tue, 23 Jun 2026 15:21:17 -0500 Subject: [PATCH 2/4] refactor: use u32 compression level to match updated assembly-pack API The compression field on Converter now takes Option instead of Option, removing the need to import flate2's type. Co-Authored-By: Claude Sonnet 4.6 --- src/cache/mod.rs | 4 ++-- src/finalize.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cache/mod.rs b/src/cache/mod.rs index 3fd1538..b5bc06d 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -6,7 +6,7 @@ use assembly_pack::{ }, crc::{calculate_crc, CRC}, md5::{self, MD5Sum}, - sd0::fs::{Compression, Converter}, + sd0::fs::Converter, txt::{FileLine, FileMeta, Manifest, VersionLine}, }; use color_eyre::eyre::Context; @@ -299,7 +299,7 @@ impl Visitor { let conv = Converter { generate_segment_index: false, - compression: Some(Compression::new(compression_level)), + compression: Some(compression_level), }; let pair = conv .convert_file(&p.input, &p.outpath) diff --git a/src/finalize.rs b/src/finalize.rs index 8832a29..4b485be 100644 --- a/src/finalize.rs +++ b/src/finalize.rs @@ -7,7 +7,7 @@ use std::{ use argh::FromArgs; use assembly_pack::{ md5::{self, MD5Sum}, - sd0::fs::{Compression, Converter}, + sd0::fs::Converter, }; use color_eyre::eyre::Context; use globset::{Glob, GlobSet, GlobSetBuilder}; @@ -215,7 +215,7 @@ fn make_version( log::info!("Compressing {}", file_path.display()); let conv = Converter { generate_segment_index: false, - compression: Some(Compression::new(compression_level)), + compression: Some(compression_level), }; let pair = conv .convert_file(&file_path, &compressed_path) From bb832e8324253cfc00ef0cc84e9948faa0269e1c Mon Sep 17 00:00:00 2001 From: Aaron Kimbrell Date: Tue, 23 Jun 2026 15:37:01 -0500 Subject: [PATCH 3/4] fix: address review feedback - rename flush_pending and fix detect_locale - Rename flush_pending to compress_pending for clarity since all compression happens there, not just flushing a buffer - Rewrite detect_locale to use iterator-based char extraction instead of vec indexing, avoiding potential panics on non-ASCII input Co-Authored-By: Claude Sonnet 4.6 --- src/cache/mod.rs | 6 +++--- src/pki.rs | 19 ++++++++----------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/src/cache/mod.rs b/src/cache/mod.rs index b5bc06d..591e52d 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -271,8 +271,8 @@ impl Visitor { } } - /// Process all queued files in parallel, then merge results into the manifest. - fn flush_pending(&mut self, compression_level: u32) { + /// Compress all queued files in parallel, then merge results into the manifest. + fn compress_pending(&mut self, compression_level: u32) { if self.pending.is_empty() { return; } @@ -469,7 +469,7 @@ pub fn run(args: ProjectArgs) -> color_eyre::Result<()> { } // Process all queued compressions in parallel - visitor.flush_pending(compression_level); + visitor.compress_pending(compression_level); manifest::write_manifest(visitor.manifest, &manifest).context("Failed to write manifest")?; diff --git a/src/pki.rs b/src/pki.rs index 3b35f7f..f3e9b16 100644 --- a/src/pki.rs +++ b/src/pki.rs @@ -83,17 +83,14 @@ fn detect_locale(path: &str) -> Option { let lower = path.to_lowercase(); let idx = lower.find("_loc\\")?; let after = &lower[idx + 5..]; - if after.len() < 5 { - return None; - } - let locale = &after[..5]; - let chars: Vec = locale.chars().collect(); - if chars[0].is_alphanumeric() - && chars[1].is_alphanumeric() - && chars[2] == '_' - && chars[3].is_alphanumeric() - && chars[4].is_alphanumeric() - { + let mut chars = after.chars(); + let c0 = chars.next().filter(|c| c.is_ascii_alphanumeric())?; + let c1 = chars.next().filter(|c| c.is_ascii_alphanumeric())?; + let c2 = chars.next().filter(|&c| c == '_')?; + let c3 = chars.next().filter(|c| c.is_ascii_alphanumeric())?; + let c4 = chars.next().filter(|c| c.is_ascii_alphanumeric())?; + let locale: String = [c0, c1, c2, c3, c4].iter().collect(); + if !locale.is_empty() { Some(format!("_loc\\{}", locale)) } else { None From 602bb6e0314c5c123e023e911f028ea08dfaae66 Mon Sep 17 00:00:00 2001 From: Aaron Kimbrell Date: Tue, 23 Jun 2026 18:49:43 -0500 Subject: [PATCH 4/4] feat: generate frontend.txt from PKI pack names instead of glob patterns Read the PKI file to find which packs have "front" in their name, collect the CRCs of files in those packs, and include only matching trunk entries in frontend.txt. This is more accurate than glob-matching file paths and automatically stays in sync with the pack definitions. Replaces --frontend-patterns with --frontend-pack-pattern (default: "front"). Co-Authored-By: Claude Sonnet 4.6 --- src/finalize.rs | 97 ++++++++++++++++++++++++++++--------------------- 1 file changed, 56 insertions(+), 41 deletions(-) diff --git a/src/finalize.rs b/src/finalize.rs index 4b485be..bf8b9e1 100644 --- a/src/finalize.rs +++ b/src/finalize.rs @@ -1,4 +1,5 @@ use std::{ + collections::HashSet, fs::{self, File}, io::{BufRead, BufReader, BufWriter, Write}, path::{Path, PathBuf}, @@ -6,20 +7,18 @@ use std::{ use argh::FromArgs; use assembly_pack::{ + crc::calculate_crc, md5::{self, MD5Sum}, + pki::parser::parse_pki_file, sd0::fs::Converter, }; use color_eyre::eyre::Context; -use globset::{Glob, GlobSet, GlobSetBuilder}; use crate::ProjectArgs; const CRLF: &str = "\r\n"; -/// Default frontend file patterns. -/// The working cache includes all trunk entries with a file extension. -/// `**/*.*` matches any file containing a `.` at any directory depth. -const DEFAULT_FRONTEND_PATTERNS: &[&str] = &["**/*.*"]; +const DEFAULT_FRONTEND_PACK_PATTERN: &str = "front"; const EXCLUDE_PATTERNS: &[&str] = &[ ".git", @@ -44,9 +43,9 @@ pub struct Args { #[argh(option, default = "PathBuf::from(\"patcher\")")] patcher: PathBuf, - /// path to file with frontend glob patterns (one per line) - #[argh(option)] - frontend_patterns: Option, + /// substring to match pack archive names for frontend (default: "front") + #[argh(option, default = "DEFAULT_FRONTEND_PACK_PATTERN.to_string()")] + frontend_pack_pattern: String, /// compression level 0-9 (default: 1, fastest; 9 = smallest but slowest) #[argh(option, short = 'c', default = "1")] @@ -148,32 +147,65 @@ fn filter_trunk( Ok(()) } -/// Generate frontend.txt by filtering trunk entries against frontend patterns +/// Generate frontend.txt by including only trunk entries whose files belong to +/// pack archives matching the given pattern (e.g. "front" matches front1.pk, +/// ui1_front_1.pk, physics_front.pk, etc.) fn generate_frontend( cache_dir: &Path, manifest_name: &str, + pki_name: &str, version_num: u32, version_name: &str, - patterns: &GlobSet, + pack_pattern: &str, ) -> color_eyre::Result<()> { + let pki_path = cache_dir.join(pki_name); let trunk_path = cache_dir.join(manifest_name).with_extension("txt"); let frontend_path = cache_dir.join("frontend.txt"); - log::info!("Generating {}", frontend_path.display()); + log::info!("Generating {} (packs matching '{}')", frontend_path.display(), pack_pattern); + + let pki_data = fs::read(&pki_path) + .wrap_err_with(|| format!("Failed to read PKI {}", pki_path.display()))?; + let (_, pki) = parse_pki_file(&pki_data) + .map_err(|e| color_eyre::eyre::eyre!("Failed to parse PKI: {}", e))?; + + let front_indices: HashSet = pki + .archives + .iter() + .enumerate() + .filter(|(_, a)| a.path.to_lowercase().contains(pack_pattern)) + .map(|(i, _)| i as u32) + .collect(); + + log::info!( + "Found {} frontend pack archives out of {}", + front_indices.len(), + pki.archives.len() + ); + + let front_crcs: HashSet = pki + .files + .iter() + .filter(|(_, file_ref)| front_indices.contains(&file_ref.pack_file)) + .map(|(crc, _)| crc.to_raw()) + .collect(); + + log::info!("Frontend packs contain {} files", front_crcs.len()); let entries = read_manifest_entries(&trunk_path)?; let mut frontend_lines = Vec::new(); - for entry in entries { - // Extract file path (first field before comma) + for entry in &entries { if let Some(path) = entry.split(',').next() { - if patterns.is_match(path) { - frontend_lines.push(entry); + let path_backslash = path.replace('/', "\\"); + let crc = calculate_crc(path_backslash.as_bytes()).to_raw(); + if front_crcs.contains(&crc) { + frontend_lines.push(entry.clone()); } } } - log::info!("Frontend: {} entries", frontend_lines.len()); + log::info!("Frontend: {} of {} trunk entries", frontend_lines.len(), entries.len()); write_manifest_crlf(&frontend_path, version_num, version_name, &frontend_lines)?; Ok(()) @@ -232,29 +264,6 @@ fn make_version( Ok(()) } -fn build_frontend_patterns(patterns_file: Option<&Path>) -> color_eyre::Result { - let mut builder = GlobSetBuilder::new(); - - if let Some(path) = patterns_file { - let file = File::open(path) - .wrap_err_with(|| format!("Failed to open patterns file {}", path.display()))?; - let reader = BufReader::new(file); - for line in reader.lines() { - let line = line?; - let trimmed = line.trim(); - if !trimmed.is_empty() && !trimmed.starts_with('#') { - builder.add(Glob::new(trimmed)?); - } - } - } else { - for pattern in DEFAULT_FRONTEND_PATTERNS { - builder.add(Glob::new(pattern)?); - } - } - - Ok(builder.build()?) -} - pub fn run(args: ProjectArgs) -> color_eyre::Result<()> { let paths = args.paths(); let cache_dir = &paths.cache_dir; @@ -269,13 +278,19 @@ pub fn run(args: ProjectArgs) -> color_eyre::Result<()> { ); let compression_level = args.cmd.compression; - let frontend_patterns = build_frontend_patterns(args.cmd.frontend_patterns.as_deref())?; log::info!("[Finalize] Filtering trunk"); filter_trunk(cache_dir, manifest_name, vnum, &vname)?; log::info!("[Finalize] Generating frontend"); - generate_frontend(cache_dir, manifest_name, vnum, &vname, &frontend_patterns)?; + generate_frontend( + cache_dir, + manifest_name, + &pki_name, + vnum, + &vname, + &args.cmd.frontend_pack_pattern, + )?; log::info!("[Finalize] Copying patcher.ini"); copy_patcher(&args.cmd.patcher, cache_dir)?;