diff --git a/Cargo.lock b/Cargo.lock index 28146cd..5841189 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" @@ -78,8 +78,6 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "assembly-pack" version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7516a74139653ba7d79db9a3139cdd4e5bcf6ee66d71ed8868362e598eb5f591" dependencies = [ "adler32", "crc", @@ -88,6 +86,7 @@ dependencies = [ "md5", "nom", "nom-supreme", + "rustc_version", "serde", "thiserror", ] @@ -126,9 +125,9 @@ dependencies = [ [[package]] name = "brownstone" -version = "1.1.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "030ea61398f34f1395ccbeb046fb68c87b631d1f34567fed0f0f11fa35d18d8d" +checksum = "c5839ee4f953e811bfdcf223f509cb2c6a3e1447959b0bff459405575bc17f22" dependencies = [ "arrayvec", ] @@ -184,18 +183,18 @@ dependencies = [ [[package]] name = "crc" -version = "2.1.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49fc9a695bca7f35f5f4c15cddc84415f66a74ea78eef08e90c5024f2b540e23" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" dependencies = [ "crc-catalog", ] [[package]] name = "crc-catalog" -version = "1.1.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccaeedb56da03b09f598226e25e80088cb4cd25f316e6e4df7d695f0feeb1403" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crc32fast" @@ -206,6 +205,37 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + [[package]] name = "env_logger" version = "0.9.3" @@ -343,6 +373,7 @@ dependencies = [ "globset", "indexmap", "log", + "rayon", "serde", "toml", ] @@ -395,9 +426,9 @@ dependencies = [ [[package]] name = "nom-supreme" -version = "0.6.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aadc66631948f6b65da03be4c4cd8bd104d481697ecbb9bbd65719b1ec60bc9f" +checksum = "2bd3ae6c901f1959588759ff51c95d24b491ecb9ff91aa9c2ef4acc5b1dcab27" dependencies = [ "brownstone", "indent_write", @@ -451,6 +482,26 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "regex" version = "1.8.4" @@ -474,6 +525,21 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + [[package]] name = "serde" version = "1.0.164" diff --git a/Cargo.toml b/Cargo.toml index cbdb111..5869a03 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" [dependencies] argh = "0.1.7" -assembly-pack = { version = "0.5.2", features = [ +assembly-pack = { path = "../Assembly/modules/pack", features = [ "pk", "pki", "sd0", @@ -17,6 +17,7 @@ assembly-pack = { version = "0.5.2", features = [ color-eyre = "0.5.11" env_logger = "0.9" globset = "0.4.8" +rayon = "1.7" indexmap = { version = "1.8.0", features = ["serde-1"] } log = "0.4" serde = "1.0" diff --git a/src/cache/mod.rs b/src/cache/mod.rs index c9f00f6..591e52d 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -4,18 +4,20 @@ use assembly_pack::{ fs::{scan_dir, FileInfo, FsVisitor}, FileMetaPair, }, - crc::calculate_crc, + crc::{calculate_crc, CRC}, md5::{self, MD5Sum}, sd0::fs::Converter, - txt::{FileLine, Manifest, VersionLine}, + txt::{FileLine, FileMeta, Manifest, VersionLine}, }; use color_eyre::eyre::Context; use globset::{Glob, GlobSet, GlobSetBuilder}; +use rayon::prelude::*; use std::{ collections::BTreeMap, fs::{File, Metadata}, io::{BufRead, BufReader, BufWriter, ErrorKind, Read, Seek, SeekFrom}, path::{Path, PathBuf}, + sync::atomic::{AtomicUsize, Ordering}, time::{Duration, UNIX_EPOCH}, }; @@ -51,6 +53,14 @@ pub struct Args { /// name of a file containing one path per line #[argh(option, short = 'F')] files: Option, + + /// compression level 0-9 (default: 1, fastest; 9 = smallest but slowest) + #[argh(option, short = 'c', default = "1")] + compression: u32, + + /// number of parallel compression threads (default: all cores) + #[argh(option, short = 'j')] + jobs: Option, } fn hash_to_path(hash: &MD5Sum) -> String { @@ -65,52 +75,47 @@ fn hash_to_path(hash: &MD5Sum) -> String { #[derive(Default, Debug)] struct Stats { quickcheck: usize, + cached_sd0: usize, + unchanged: usize, compress: usize, - updated: usize, total: usize, ignored: usize, } +/// A file that needs SD0 compression (queued during scan, processed in parallel) +struct PendingCompression { + path: String, + input: PathBuf, + outpath: PathBuf, + raw_meta: FileMeta, + mtime: Option, +} + +/// Result of a successful parallel compression +struct CompressResult { + path: String, + meta_pair: FileMetaPair, + linesum: MD5Sum, + mtime: Option, + raw_meta: FileMeta, +} + struct Visitor { stats: Stats, include_glob: GlobSet, exclude_glob: GlobSet, - quickcheck: BTreeMap, + quickcheck: BTreeMap, quickcheck_out: BufWriter, - conv: Converter, output: PathBuf, /// The previous manifest prev: BTreeMap, /// The new manifest manifest: Manifest, + /// Files queued for parallel compression + pending: Vec, } impl Visitor { - fn compress(&mut self, input: &Path, outpath: &Path) -> Option { - // Continue with conversion if it was just not found - let parent = outpath.parent().unwrap(); - if let Err(e) = std::fs::create_dir_all(parent) { - log::error!("Failed to create dir {}:\n\t{}", parent.display(), e); - return None; - } - log::info!("Converting {} to {}", input.display(), outpath.display()); - match self.conv.convert_file(input, &outpath) { - Err(e) => { - log::error!( - "Error converting {} to {}:\n\t{}", - input.display(), - outpath.display(), - e - ); - return None; - } - Ok(line) => { - self.stats.compress += 1; - Some(line) - } - } - } - fn visit(&mut self, path: String, input: &Path, meta: Option) { if !self.include_glob.is_match(&path) || self.exclude_glob.is_match(&path) { self.stats.ignored += 1; @@ -128,7 +133,6 @@ impl Visitor { let quickcheck = self.quickcheck.remove(&crc); let in_meta = match quickcheck { - // FIXME: size check Some(qc) if (mtime.is_some() && qc.mtime == mtime) => { self.stats.quickcheck += 1; qc.meta @@ -146,7 +150,6 @@ impl Visitor { let mut meta_pair = old_meta_pair.filter(|(p, _)| p.raw == in_meta); if let (Some(old), None) = (old_meta_pair.as_ref(), meta_pair.as_ref()) { - self.stats.updated += 1; log::debug!( "File {} was updated from {} to {}", path, @@ -158,25 +161,37 @@ impl Visitor { let outpath = self.output.join(hash_to_path(&in_meta.hash)); if meta_pair.is_none() { - let line = match md5::md5sum(&outpath) { - Ok(meta) => FileMetaPair { - raw: in_meta, - compressed: meta, - }, + match md5::md5sum(&outpath) { + Ok(compressed_meta) => { + // SD0 file already exists in cache, reuse it + self.stats.cached_sd0 += 1; + let line = FileMetaPair { + raw: in_meta, + compressed: compressed_meta, + }; + let linesum = md5::MD5Sum::compute(&format!("{path},{line}")); + meta_pair = Some((line, linesum)); + } Err(e) => { if e.kind() != ErrorKind::NotFound { log::error!("Failed to access {}:\n\t{}", outpath.display(), e); return; } - let Some(meta_pair) = self.compress(input, &outpath) else { - return - }; - meta_pair + // Queue for parallel compression + self.pending.push(PendingCompression { + path, + input: input.to_owned(), + outpath, + raw_meta: in_meta, + mtime, + }); + return; // will be handled after parallel compression } }; - let linesum = md5::MD5Sum::compute(&format!("{path},{line}")); - meta_pair = Some((line, linesum)); + } else { + self.stats.unchanged += 1; } + if let Some((meta_pair, linesum)) = meta_pair { let qc = QuickCheck { path: path.clone(), @@ -184,7 +199,6 @@ impl Visitor { meta: in_meta, }; qc.write(&mut self.quickcheck_out).unwrap(); - self.manifest.files.insert(path, (meta_pair, linesum)); } } @@ -206,7 +220,6 @@ impl Visitor { let meta = match std::fs::metadata(&real) { Ok(meta) => Some(meta), Err(e) if e.kind() == ErrorKind::NotFound => { - // If the file is explicitly listed but was no found, remove it log::warn!("File {:?} not found!", path); let crc = calculate_crc(path.as_bytes()); if let Some(_) = self.quickcheck.remove(&crc) { @@ -215,7 +228,7 @@ impl Visitor { if let Some(_) = self.prev.remove(&path) { log::info!("Removed {:?} from manifest", path); } - return; // don't visit this file + return; } Err(e) => { log::debug!("Failed to get file metadata: {}", e); @@ -225,7 +238,6 @@ impl Visitor { self.visit(path, &real, meta); } - /// Small generic function that calls [`do_scan_file`] on every line of its input fn do_scan_files( &mut self, file_list_reader: R, @@ -234,7 +246,7 @@ impl Visitor { ) -> color_eyre::Result<()> { let files = BufReader::new(file_list_reader); let strip_prefix = match relative { - true => "", // don't try to strip a prefix on relative paths + true => "", false => &paths.strip_prefix, }; for line in files.lines() { @@ -258,11 +270,82 @@ impl Visitor { self.do_scan_files(file_list_reader, paths, relative) } } + + /// Compress all queued files in parallel, then merge results into the manifest. + fn compress_pending(&mut self, compression_level: u32) { + if self.pending.is_empty() { + return; + } + + let count = self.pending.len(); + log::info!( + "Compressing {} files in parallel (level {})", + count, + compression_level + ); + + let compressed_count = AtomicUsize::new(0); + + let results: Vec> = self + .pending + .par_iter() + .map(|p| { + let i = compressed_count.fetch_add(1, Ordering::Relaxed) + 1; + log::info!("[{}/{}] Compressing {}", i, count, p.input.display()); + + if let Some(parent) = p.outpath.parent() { + let _ = std::fs::create_dir_all(parent); + } + + let conv = Converter { + generate_segment_index: false, + compression: Some(compression_level), + }; + let pair = conv + .convert_file(&p.input, &p.outpath) + .map_err(|e| { + log::error!("Failed to compress {}: {}", p.input.display(), e); + e + }) + .ok()?; + + let linesum = MD5Sum::compute(&format!("{},{}", p.path, pair)); + + Some(CompressResult { + path: p.path.clone(), + meta_pair: pair, + linesum, + mtime: p.mtime, + raw_meta: p.raw_meta, + }) + }) + .collect(); + + // Merge results back sequentially + let pending = std::mem::take(&mut self.pending); + let mut compress_count = 0usize; + for (_, result) in pending.into_iter().zip(results.into_iter()) { + if let Some(r) = result { + compress_count += 1; + let qc = QuickCheck { + path: r.path.clone(), + mtime: r.mtime, + meta: r.raw_meta, + }; + qc.write(&mut self.quickcheck_out).unwrap(); + self.manifest.files.insert(r.path, (r.meta_pair, r.linesum)); + } + } + self.stats.compress += compress_count; + log::info!("Compressed {} files", compress_count); + } } impl FsVisitor for Visitor { - fn visit_file(&mut self, info: FileInfo) { - self.visit(info.path(), info.real(), info.metadata().ok()) + fn visit_file(&mut self, info: F) { + // Normalize virtual paths to lowercase for case-insensitive consistency + // (LU is a Windows game; all paths should be case-insensitive) + self.visit(info.path().to_lowercase(), info.real(), info.metadata().ok()) } } @@ -287,6 +370,22 @@ fn exclude_glob(project: &ProjectConfig) -> Result { } pub fn run(args: ProjectArgs) -> color_eyre::Result<()> { + let compression_level = args.cmd.compression; + if compression_level > 9 { + return Err(color_eyre::eyre::eyre!( + "Compression level must be 0-9, got {}", + compression_level + )); + } + + // Configure rayon thread pool + if let Some(jobs) = args.cmd.jobs { + rayon::ThreadPoolBuilder::new() + .num_threads(jobs) + .build_global() + .ok(); // ignore if already initialized + } + let paths = args.paths(); let quickcheck_path = paths @@ -329,6 +428,12 @@ pub fn run(args: ProjectArgs) -> color_eyre::Result<()> { _ => BTreeMap::new(), }; + log::info!( + "Compression level: {}, threads: {}", + compression_level, + args.cmd.jobs.unwrap_or_else(|| rayon::current_num_threads()) + ); + let mut visitor = Visitor { include_glob, exclude_glob, @@ -340,10 +445,8 @@ pub fn run(args: ProjectArgs) -> color_eyre::Result<()> { version, files: BTreeMap::new(), }, - conv: Converter { - generate_segment_index: false, - }, output, + pending: Vec::new(), }; log::info!("Scanning {} as {}", proj_dir.display(), paths.prefix); @@ -351,20 +454,23 @@ pub fn run(args: ProjectArgs) -> color_eyre::Result<()> { if let Some(file_list_path) = args.cmd.files { visitor.scan_files(&file_list_path, &paths, args.cmd.relative)?; // Write out untouched manifest files - for (key, value) in visitor.prev { + for (key, value) in std::mem::take(&mut visitor.prev) { visitor.manifest.files.insert(key, value); } // Write out untouched quickcheck files - for (_key, value) in visitor.quickcheck { + for (_key, value) in std::mem::take(&mut visitor.quickcheck) { value.write(&mut visitor.quickcheck_out)?; } } else { scan_dir(&mut visitor, paths.prefix, &proj_dir, true); - for (k, _v) in visitor.prev { + for (k, _v) in &visitor.prev { log::info!("File {} was removed", k); } } + // Process all queued compressions in parallel + visitor.compress_pending(compression_level); + manifest::write_manifest(visitor.manifest, &manifest).context("Failed to write manifest")?; log::info!("{:?}", visitor.stats); diff --git a/src/cache/quickcheck.rs b/src/cache/quickcheck.rs index 1dd924c..b46c02d 100644 --- a/src/cache/quickcheck.rs +++ b/src/cache/quickcheck.rs @@ -4,8 +4,9 @@ use std::{ io::{self, BufRead, BufReader, BufWriter, Write}, }; -use assembly_pack::{crc::calculate_crc, md5::MD5Sum, txt::FileMeta}; +use assembly_pack::{crc::{calculate_crc, CRC}, md5::MD5Sum, txt::FileMeta}; +#[derive(Clone)] pub(super) struct QuickCheck { pub path: String, pub mtime: Option, @@ -27,7 +28,7 @@ impl QuickCheck { } } -pub(super) fn scan_quickcheck(reader: &mut R) -> BTreeMap { +pub(super) fn scan_quickcheck(reader: &mut R) -> BTreeMap { let mut quickcheck = BTreeMap::new(); let mut reader = BufReader::new(reader); let mut buffer = String::new(); diff --git a/src/finalize.rs b/src/finalize.rs new file mode 100644 index 0000000..bf8b9e1 --- /dev/null +++ b/src/finalize.rs @@ -0,0 +1,317 @@ +use std::{ + collections::HashSet, + fs::{self, File}, + io::{BufRead, BufReader, BufWriter, Write}, + path::{Path, PathBuf}, +}; + +use argh::FromArgs; +use assembly_pack::{ + crc::calculate_crc, + md5::{self, MD5Sum}, + pki::parser::parse_pki_file, + sd0::fs::Converter, +}; +use color_eyre::eyre::Context; + +use crate::ProjectArgs; + +const CRLF: &str = "\r\n"; + +const DEFAULT_FRONTEND_PACK_PATTERN: &str = "front"; + +const EXCLUDE_PATTERNS: &[&str] = &[ + ".git", + "generator_config.txt", + "generator_config_ci.txt", + "README.md", +]; + +#[derive(FromArgs, PartialEq, Debug)] +#[argh(subcommand, name = "finalize")] +/// post-process cache: filter trunk, generate frontend/version manifests, copy patcher +pub struct Args { + /// version number + #[argh(option, short = 'v', default = "1")] + version: u32, + + /// version name + #[argh(option, short = 'n')] + name: Option, + + /// path to patcher directory (default: "patcher") + #[argh(option, default = "PathBuf::from(\"patcher\")")] + patcher: PathBuf, + + /// substring to match pack archive names for frontend (default: "front") + #[argh(option, default = "DEFAULT_FRONTEND_PACK_PATTERN.to_string()")] + frontend_pack_pattern: String, + + /// compression level 0-9 (default: 1, fastest; 9 = smallest but slowest) + #[argh(option, short = 'c', default = "1")] + compression: u32, +} + +fn hash_to_path(hash: &MD5Sum) -> String { + let hash_str = format!("{}", hash); + let mut chars = hash_str.chars(); + let c1 = chars.next().unwrap(); + let c2 = chars.next().unwrap(); + format!("{}/{}/{}.sd0", c1, c2, hash_str) +} + +/// Write a manifest file with CRLF line endings (matching LU client expectations) +fn write_manifest_crlf( + path: &Path, + version_num: u32, + version_name: &str, + lines: &[String], +) -> color_eyre::Result<()> { + let file = File::create(path) + .wrap_err_with(|| format!("Failed to create {}", path.display()))?; + let mut w = BufWriter::new(file); + + let vnum_hash = MD5Sum::compute(&version_num.to_string()); + write!(w, "[version]{CRLF}")?; + write!(w, "{},{},{}{CRLF}", version_num, vnum_hash, version_name)?; + write!(w, "[files]")?; + for line in lines { + write!(w, "{CRLF}{line}")?; + } + + Ok(()) +} + +fn should_exclude(line: &str) -> bool { + EXCLUDE_PATTERNS.iter().any(|pat| line.contains(pat)) +} + +/// Read a manifest file, skipping the 3-line header, returning entry lines +fn read_manifest_entries(path: &Path) -> color_eyre::Result> { + let file = File::open(path) + .wrap_err_with(|| format!("Failed to open {}", path.display()))?; + let reader = BufReader::new(file); + let mut lines_iter = reader.lines(); + + // Skip header: [version], version line, [files] + for _ in 0..3 { + lines_iter.next(); + } + + let mut entries = Vec::new(); + for line_result in lines_iter { + let line = line_result?; + let trimmed = line.trim().to_string(); + if !trimmed.is_empty() { + entries.push(trimmed); + } + } + + Ok(entries) +} + +/// Filter trunk.txt: normalize paths, lowercase, remove excluded entries, recalculate line hashes +fn filter_trunk( + cache_dir: &Path, + manifest_name: &str, + version_num: u32, + version_name: &str, +) -> color_eyre::Result<()> { + let trunk_path = cache_dir.join(manifest_name).with_extension("txt"); + log::info!("Filtering {}", trunk_path.display()); + + let entries = read_manifest_entries(&trunk_path)?; + let mut filtered = Vec::new(); + + for entry in entries { + // Normalize: backslash to forward slash, lowercase + let entry = entry.replace('\\', "/").to_lowercase(); + + if should_exclude(&entry) { + continue; + } + + // Take first 5 fields, recalculate line hash + let fields: Vec<&str> = entry.split(',').collect(); + if fields.len() < 5 { + continue; + } + let first_five = fields[..5].join(","); + let line_hash = MD5Sum::compute(&first_five); + filtered.push(format!("{first_five},{line_hash}")); + } + + log::info!("Filtered trunk: {} entries", filtered.len()); + write_manifest_crlf(&trunk_path, version_num, version_name, &filtered)?; + + Ok(()) +} + +/// Generate frontend.txt by including only trunk entries whose files belong to +/// pack archives matching the given pattern (e.g. "front" matches front1.pk, +/// ui1_front_1.pk, physics_front.pk, etc.) +fn generate_frontend( + cache_dir: &Path, + manifest_name: &str, + pki_name: &str, + version_num: u32, + version_name: &str, + pack_pattern: &str, +) -> color_eyre::Result<()> { + let pki_path = cache_dir.join(pki_name); + let trunk_path = cache_dir.join(manifest_name).with_extension("txt"); + let frontend_path = cache_dir.join("frontend.txt"); + + log::info!("Generating {} (packs matching '{}')", frontend_path.display(), pack_pattern); + + let pki_data = fs::read(&pki_path) + .wrap_err_with(|| format!("Failed to read PKI {}", pki_path.display()))?; + let (_, pki) = parse_pki_file(&pki_data) + .map_err(|e| color_eyre::eyre::eyre!("Failed to parse PKI: {}", e))?; + + let front_indices: HashSet = pki + .archives + .iter() + .enumerate() + .filter(|(_, a)| a.path.to_lowercase().contains(pack_pattern)) + .map(|(i, _)| i as u32) + .collect(); + + log::info!( + "Found {} frontend pack archives out of {}", + front_indices.len(), + pki.archives.len() + ); + + let front_crcs: HashSet = pki + .files + .iter() + .filter(|(_, file_ref)| front_indices.contains(&file_ref.pack_file)) + .map(|(crc, _)| crc.to_raw()) + .collect(); + + log::info!("Frontend packs contain {} files", front_crcs.len()); + + let entries = read_manifest_entries(&trunk_path)?; + let mut frontend_lines = Vec::new(); + + for entry in &entries { + if let Some(path) = entry.split(',').next() { + let path_backslash = path.replace('/', "\\"); + let crc = calculate_crc(path_backslash.as_bytes()).to_raw(); + if front_crcs.contains(&crc) { + frontend_lines.push(entry.clone()); + } + } + } + + log::info!("Frontend: {} of {} trunk entries", frontend_lines.len(), entries.len()); + write_manifest_crlf(&frontend_path, version_num, version_name, &frontend_lines)?; + + Ok(()) +} + +/// Copy patcher.ini to cache dir +fn copy_patcher(patcher_dir: &Path, cache_dir: &Path) -> color_eyre::Result<()> { + let src = patcher_dir.join("patcher.ini"); + let dst = cache_dir.join("patcher.ini"); + log::info!("Copying {} to {}", src.display(), dst.display()); + fs::copy(&src, &dst) + .wrap_err_with(|| format!("Failed to copy {} to {}", src.display(), dst.display()))?; + Ok(()) +} + +/// Hash and compress specific files, writing a version manifest +fn make_version( + cache_dir: &Path, + filename: &str, + version_num: u32, + version_name: &str, + files: &[&str], + compression_level: u32, +) -> color_eyre::Result<()> { + let output_path = cache_dir.join(filename); + log::info!("Generating {}", output_path.display()); + + let mut lines = Vec::new(); + + for &file in files { + let file_path = cache_dir.join(file); + + // Get raw hash to determine SD0 output path + let raw_meta = md5::md5sum(&file_path) + .wrap_err_with(|| format!("Failed to hash {}", file_path.display()))?; + + let compressed_path = cache_dir.join(hash_to_path(&raw_meta.hash)); + + log::info!("Compressing {}", file_path.display()); + let conv = Converter { + generate_segment_index: false, + compression: Some(compression_level), + }; + let pair = conv + .convert_file(&file_path, &compressed_path) + .wrap_err_with(|| format!("Failed to compress {}", file_path.display()))?; + + // Build manifest line with forward slashes + let file_normalized = file.replace('\\', "/"); + let line_content = format!("{},{}", file_normalized, pair); + let line_hash = MD5Sum::compute(&line_content); + lines.push(format!("{line_content},{line_hash}")); + } + + write_manifest_crlf(&output_path, version_num, version_name, &lines)?; + Ok(()) +} + +pub fn run(args: ProjectArgs) -> color_eyre::Result<()> { + let paths = args.paths(); + let cache_dir = &paths.cache_dir; + let manifest_name = args.project.manifest.to_str().unwrap_or("trunk"); + + let vnum = args.cmd.version; + let vname = args.cmd.name.unwrap_or_else(|| vnum.to_string()); + + let pki_name = format!( + "{}.pki", + args.project.pki.to_str().unwrap_or("primary") + ); + + let compression_level = args.cmd.compression; + + log::info!("[Finalize] Filtering trunk"); + filter_trunk(cache_dir, manifest_name, vnum, &vname)?; + + log::info!("[Finalize] Generating frontend"); + generate_frontend( + cache_dir, + manifest_name, + &pki_name, + vnum, + &vname, + &args.cmd.frontend_pack_pattern, + )?; + + log::info!("[Finalize] Copying patcher.ini"); + copy_patcher(&args.cmd.patcher, cache_dir)?; + + log::info!("[Finalize] Generating hotfix.txt"); + make_version(cache_dir, "hotfix.txt", vnum, &vname, &[], compression_level)?; + + log::info!("[Finalize] Generating index.txt"); + make_version( + cache_dir, + "index.txt", + vnum, + &vname, + &["frontend.txt", &pki_name, &format!("{manifest_name}.txt")], + compression_level, + )?; + + log::info!("[Finalize] Generating version.txt"); + make_version(cache_dir, "version.txt", vnum, &vname, &["index.txt"], compression_level)?; + + log::info!("[Finalize] Done"); + + Ok(()) +} diff --git a/src/main.rs b/src/main.rs index 2114860..b3c8bcb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,6 +7,7 @@ use log::LevelFilter; mod cache; mod config; +mod finalize; mod pack; mod pki; @@ -24,6 +25,7 @@ struct Args { #[argh(subcommand)] pub enum Commands { Cache(cache::Args), + Finalize(finalize::Args), Pack(pack::Args), PKI(pki::Args), } @@ -162,6 +164,9 @@ fn main() -> color_eyre::Result<()> { Commands::Cache(cmd) => { cache::run(ProjectArgs::new(dir, config.general, project, name, cmd)) } + Commands::Finalize(cmd) => { + finalize::run(ProjectArgs::new(dir, config.general, project, name, cmd)) + } Commands::Pack(cmd) => pack::run(ProjectArgs::new(dir, config.general, project, name, cmd)), Commands::PKI(cmd) => pki::run(ProjectArgs::new(dir, config.general, project, name, cmd)), } diff --git a/src/pack.rs b/src/pack.rs index e0c7a28..828ea22 100644 --- a/src/pack.rs +++ b/src/pack.rs @@ -124,8 +124,6 @@ pub fn run(args: ProjectArgs) -> color_eyre::Result<()> { }); let is_compressed = lookup.category & 0xFF > 0; - let raw = file.raw; - let compressed = file.compressed; let path = if is_compressed { output.join(file.to_path()) @@ -134,8 +132,10 @@ pub fn run(args: ProjectArgs) -> color_eyre::Result<()> { win_join(&paths.proj_dir, relative_name) }; + log::info!("Packing '{}' from '{}' (compressed={})", name, path.display(), is_compressed); let mut writer = Writer { path: &path }; - pk.put_file(crc, &mut writer, raw, compressed, is_compressed)?; + pk.put_file(crc, &mut writer, file, is_compressed) + .with_context(|| format!("Failed to pack '{}' from '{}'", name, path.display()))?; } } } diff --git a/src/pki.rs b/src/pki.rs index cce220b..f3e9b16 100644 --- a/src/pki.rs +++ b/src/pki.rs @@ -7,6 +7,7 @@ use color_eyre::eyre::Context; use indexmap::IndexMap; use serde::Deserialize; use std::{ + collections::BTreeMap, ffi::OsStr, fs::File, io::{BufRead, BufReader, BufWriter}, @@ -51,42 +52,105 @@ fn hidden_glob(filename: &str) -> Option { None } +/// Parse a dir spec string in the format `directory[=recursive[=filter]]`. +/// +/// Examples: +/// - `mesh\\env` → directory=mesh\env, recursive=true, no filter +/// - `mesh\\env=0=re_*` → directory=mesh\env, recursive=false, filter=re_* +/// - `brickmodels\\pettaming=1` → directory=brickmodels\pettaming, recursive=true +fn parse_dir_spec(spec: &str) -> DirSpec { + let parts: Vec<&str> = spec.splitn(3, '=').collect(); + let directory = parts[0].to_lowercase(); + let recurse = match parts.get(1) { + Some(r) => *r != "0", + None => true, + }; + let filter = parts + .get(2) + .map(|f| f.to_lowercase()) + .unwrap_or_default(); + + DirSpec { + directory, + recurse_subdirectories: recurse, + filter_wildcard: filter, + } +} + +/// Detect a locale directory pattern in a path (e.g. `_loc\en_gb`, `_loc\de_de`). +/// Returns the locale segment like `_loc\de_de` if found. +fn detect_locale(path: &str) -> Option { + let lower = path.to_lowercase(); + let idx = lower.find("_loc\\")?; + let after = &lower[idx + 5..]; + let mut chars = after.chars(); + let c0 = chars.next().filter(|c| c.is_ascii_alphanumeric())?; + let c1 = chars.next().filter(|c| c.is_ascii_alphanumeric())?; + let c2 = chars.next().filter(|&c| c == '_')?; + let c3 = chars.next().filter(|c| c.is_ascii_alphanumeric())?; + let c4 = chars.next().filter(|c| c.is_ascii_alphanumeric())?; + let locale: String = [c0, c1, c2, c3, c4].iter().collect(); + if !locale.is_empty() { + Some(format!("_loc\\{}", locale)) + } else { + None + } +} + +/// Remap a pack name to include a locale segment after the first component. +/// `pack\front2_3.pk` + `_loc\de_de` → `pack\_loc\de_de\front2_3.pk` +fn localize_pack_name(pack_name: &str, locale: &str) -> String { + if let Some(idx) = pack_name.find('\\') { + format!("{}\\{}\\{}", &pack_name[..idx], locale, &pack_name[idx + 1..]) + } else { + format!("{}\\{}", locale, pack_name) + } +} + fn process_cfg(config: &mut Config, cfg: Cfg) { + // Collect locale-specific files to emit as separate packs at the end + let mut locale_packs: BTreeMap)> = BTreeMap::new(); + for (k, v) in cfg.pack { + let pack_name = format!("pack\\{}.pk", k).to_lowercase(); + let cmd = Command::Pack { - filename: format!("pack\\{}.pk", k), + filename: pack_name.clone(), force_compression: v.compress, }; push_command(config, cmd); for dir in v.dirs { - let cmd = Command::AddDir(DirSpec { - directory: dir, - recurse_subdirectories: true, - filter_wildcard: String::new(), - }); - push_command(config, cmd); + push_command(config, Command::AddDir(parse_dir_spec(&dir))); } for dir in v.exclude_dirs { - let cmd = Command::RemDir(DirSpec { - directory: dir, - recurse_subdirectories: true, - filter_wildcard: String::new(), - }); - push_command(config, cmd); + push_command(config, Command::RemDir(parse_dir_spec(&dir))); } for filename in v.files { - let cmd = if let Some(dir) = hidden_glob(&filename) { - Command::AddDir(dir) + let filename = filename.to_lowercase(); + if let Some(locale) = detect_locale(&filename) { + // Route locale-specific files to their own pack + let locale_pack = localize_pack_name(&pack_name, &locale); + log::debug!("Routing {} to locale pack {}", filename, locale_pack); + locale_packs + .entry(locale_pack) + .or_insert_with(|| (v.compress, Vec::new())) + .1 + .push(filename); } else { - Command::AddFile { filename } - }; - push_command(config, cmd); + let cmd = if let Some(dir) = hidden_glob(&filename) { + Command::AddDir(dir) + } else { + Command::AddFile { filename } + }; + push_command(config, cmd); + } } for filename in v.exclude_files { + let filename = filename.to_lowercase(); let cmd = if let Some(dir) = hidden_glob(&filename) { Command::RemDir(dir) } else { @@ -97,6 +161,29 @@ fn process_cfg(config: &mut Config, cfg: Cfg) { push_command(config, Command::EndPack); } + + // Emit locale-specific packs + for (locale_pack_name, (compress, files)) in locale_packs { + if files.is_empty() { + continue; + } + log::info!( + "Generating locale pack {} with {} files", + locale_pack_name, + files.len() + ); + push_command( + config, + Command::Pack { + filename: locale_pack_name, + force_compression: compress, + }, + ); + for filename in files { + push_command(config, Command::AddFile { filename }); + } + push_command(config, Command::EndPack); + } } pub fn run(args: ProjectArgs) -> color_eyre::Result<()> {