From c95ae5f952695de2232accb80ebbeaa25775255c Mon Sep 17 00:00:00 2001 From: hungerdeng Date: Tue, 15 Sep 2026 21:58:05 +0800 Subject: [PATCH 1/4] feat: add source-aligned FST TN API Expose source spans and semantic classes alongside compiled NeMo TN output across Rust, C, Swift, and WASM. Preserve selected FST input progress so callers can map normalized tokens without string diffing. --- Cargo.lock | 48 +++++ Cargo.toml | 3 +- README.md | 34 ++++ src/ffi.rs | 143 +++++++++++++++ src/fst/de.rs | 7 +- src/fst/driver.rs | 165 +++++++++++++++++- src/fst/en.rs | 7 +- src/fst/engine.rs | 89 ++++++++-- src/fst/es.rs | 7 +- src/fst/fr.rs | 7 +- src/fst/hi.rs | 7 +- src/fst/ja.rs | 7 +- src/fst/mod.rs | 91 ++++++++++ src/fst/zh.rs | 7 +- src/wasm.rs | 50 ++++++ .../include/nemo_text_processing.h | 18 ++ swift/NemoTextProcessing.swift | 71 ++++++++ .../nemo_text_processing.h | 34 ++++ 18 files changed, 762 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9502b59..5c109bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,6 +116,30 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + [[package]] name = "generic-array" version = "1.4.3" @@ -158,6 +182,17 @@ dependencies = [ "either", ] +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -232,6 +267,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -427,6 +468,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "superslice" version = "1.0.0" @@ -463,6 +510,7 @@ version = "0.3.0" dependencies = [ "console_error_panic_hook", "flate2", + "js-sys", "lazy_static", "proptest", "rustfst", diff --git a/Cargo.toml b/Cargo.toml index 2a6825f..e225683 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ crate-type = ["lib", "staticlib", "cdylib"] [dependencies] lazy_static = "1" wasm-bindgen = { version = "0.2", optional = true } +js-sys = { version = "0.3", optional = true } console_error_panic_hook = { version = "0.1", optional = true } # Optional: byte-exact NeMo parity via compiled weighted FST grammars. # Off by default — the pure-Rust rule path carries no rustfst/OpenFST weight. @@ -26,7 +27,7 @@ proptest = "1" [features] default = [] ffi = [] # Enable C FFI bindings -wasm = ["dep:wasm-bindgen", "dep:console_error_panic_hook"] +wasm = ["dep:wasm-bindgen", "dep:js-sys", "dep:console_error_panic_hook"] # Byte-exact NeMo parity engine (runs NeMo's compiled grammars via rustfst). # Adds rustfst + flate2 and ~7 MB of gzipped grammars (all languages combined). fst-engine = ["dep:rustfst", "dep:flate2"] diff --git a/README.md b/README.md index 7466c80..14a9a35 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,29 @@ let result = tn_normalize_sentence("I paid $5 for 23 items"); assert_eq!(result, "I paid five dollars for twenty three items"); ``` +Compiled-FST TN can also retain the source range and semantic class for every +normalized span. Offsets are half-open UTF-8 byte offsets: + +```rust +use text_processing_rs::fst; + +let result = fst::normalize_aligned("The price is $1,234.56.", "en").unwrap(); +assert_eq!( + result.normalized, + "The price is one thousand two hundred and thirty four dollars fifty six cents." +); + +let money = &result.spans[3]; +assert_eq!(money.input_start, 13); +assert_eq!(money.input_end, 22); +assert_eq!(money.original, "$1,234.56"); +assert_eq!(money.kind.as_str(), "money"); +``` + +Build this API with `--features fst-engine`. It uses the same compiled NeMo +classifier and verbalizer as `fst::::normalize`, rather than recovering +alignment by diffing the final strings. + ### Swift ```swift @@ -149,6 +172,16 @@ let itnFr = NemoTextProcessing.normalizeSentence("j'ai vingt et un ans", languag let tn = NemoTextProcessing.tnNormalizeSentence("I paid $5 for 23 items") // "I paid five dollars for twenty three items" + +if let aligned = NemoTextProcessing.tnNormalizeAligned( + "The price is $1,234.56.", + language: "en" +) { + let money = aligned.spans[3] + // money.original == "$1,234.56" + // money.normalized == "one thousand ... dollars fifty six cents" + // money.inputRange == 13..<22, money.kind == "money" +} ``` ### CLI @@ -215,6 +248,7 @@ echo "2:30 PM" | nemo-tn # → two thirty p m - Phone numbers, IP addresses, SSN - Case preservation for proper nouns and abbreviations - Sentence-level normalization with sliding window span matching +- Source-to-normalized span alignment with semantic classes (compiled FST) - Custom rules for domain-specific terms - C FFI for integration with Swift, Python, and other languages diff --git a/src/ffi.rs b/src/ffi.rs index b35926e..f1c54c1 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -10,6 +10,27 @@ use crate::{ tn_normalize_sentence_with_max_span_lang, NormalizeOptions, }; +/// One source-to-normalized TN span returned through the C ABI. +/// +/// All strings are owned by the containing [`NemoTnAlignment`] and remain valid +/// until [`nemo_tn_alignment_free`] is called. +#[repr(C)] +pub struct NemoTnAlignedSpan { + pub input_start: usize, + pub input_end: usize, + pub original: *mut c_char, + pub normalized: *mut c_char, + pub kind: *mut c_char, +} + +/// Sentence-level compiled-FST TN output and its source-span mapping. +#[repr(C)] +pub struct NemoTnAlignment { + pub normalized: *mut c_char, + pub spans: *mut NemoTnAlignedSpan, + pub span_count: usize, +} + /// Build [`NormalizeOptions`] from FFI primitives. /// /// `concat_compound_numbers`: any non-zero value enables concat behavior @@ -494,6 +515,128 @@ pub unsafe extern "C" fn nemo_tn_fst(input: *const c_char, lang: *const c_char) } } +/// Normalize with the compiled-FST engine and retain every source span. +/// +/// Offsets are half-open UTF-8 byte offsets into `input`. The returned object +/// and every pointer it owns must be released with +/// [`nemo_tn_alignment_free`]. +/// +/// Returns null when the library was built without `fst-engine`, the language +/// is unsupported, either input string is invalid UTF-8, or allocation of a C +/// string fails. +/// +/// # Safety +/// - `input` and `lang` must be valid null-terminated UTF-8 strings. +/// - The result must be freed exactly once with [`nemo_tn_alignment_free`]. +#[no_mangle] +#[cfg(feature = "fst-engine")] +pub unsafe extern "C" fn nemo_tn_fst_aligned( + input: *const c_char, + lang: *const c_char, +) -> *mut NemoTnAlignment { + if input.is_null() || lang.is_null() { + return ptr::null_mut(); + } + let input_str = match CStr::from_ptr(input).to_str() { + Ok(s) => s, + Err(_) => return ptr::null_mut(), + }; + let lang_str = match CStr::from_ptr(lang).to_str() { + Ok(s) => s, + Err(_) => return ptr::null_mut(), + }; + + let Some(alignment) = crate::fst::normalize_aligned(input_str, lang_str) else { + return ptr::null_mut(); + }; + let Ok(normalized) = CString::new(alignment.normalized) else { + return ptr::null_mut(); + }; + + let converted = alignment + .spans + .into_iter() + .map(|span| { + Some(( + span.input_start, + span.input_end, + CString::new(span.original).ok()?, + CString::new(span.normalized).ok()?, + CString::new(span.kind.as_str()).ok()?, + )) + }) + .collect::>>(); + let Some(converted) = converted else { + return ptr::null_mut(); + }; + + let spans = converted + .into_iter() + .map( + |(input_start, input_end, original, normalized, kind)| NemoTnAlignedSpan { + input_start, + input_end, + original: original.into_raw(), + normalized: normalized.into_raw(), + kind: kind.into_raw(), + }, + ) + .collect::>() + .into_boxed_slice(); + let span_count = spans.len(); + let spans = Box::into_raw(spans) as *mut NemoTnAlignedSpan; + + Box::into_raw(Box::new(NemoTnAlignment { + normalized: normalized.into_raw(), + spans, + span_count, + })) +} + +#[no_mangle] +#[cfg(not(feature = "fst-engine"))] +pub unsafe extern "C" fn nemo_tn_fst_aligned( + _input: *const c_char, + _lang: *const c_char, +) -> *mut NemoTnAlignment { + ptr::null_mut() +} + +/// Free a result returned by [`nemo_tn_fst_aligned`]. +/// +/// Passing null is allowed. +/// +/// # Safety +/// - `alignment` must be null or a pointer returned by +/// [`nemo_tn_fst_aligned`]. +/// - The pointer must not be freed more than once. +#[no_mangle] +pub unsafe extern "C" fn nemo_tn_alignment_free(alignment: *mut NemoTnAlignment) { + if alignment.is_null() { + return; + } + + let alignment = Box::from_raw(alignment); + if !alignment.normalized.is_null() { + drop(CString::from_raw(alignment.normalized)); + } + if !alignment.spans.is_null() { + let slice = ptr::slice_from_raw_parts_mut(alignment.spans, alignment.span_count); + let mut spans = Box::from_raw(slice); + for span in spans.iter_mut() { + if !span.original.is_null() { + drop(CString::from_raw(span.original)); + } + if !span.normalized.is_null() { + drop(CString::from_raw(span.normalized)); + } + if !span.kind.is_null() { + drop(CString::from_raw(span.kind)); + } + } + } +} + #[cfg(feature = "fst-engine")] fn fst_normalize(input: &str, lang: &str) -> Option { use crate::fst; diff --git a/src/fst/de.rs b/src/fst/de.rs index b6e3b0c..a33bda1 100644 --- a/src/fst/de.rs +++ b/src/fst/de.rs @@ -6,7 +6,7 @@ //! Grammars in `grammars/de/` are exported from NeMo-text-processing //! (Apache-2.0, pinned commit `1f1263579fe57ba7ed783cad3dddee710fcc5064`). -use super::{driver, load_gz}; +use super::{driver, load_gz, AlignedNormalization}; use lazy_static::lazy_static; use rustfst::prelude::*; @@ -30,3 +30,8 @@ lazy_static! { pub fn normalize(input: &str) -> String { driver::normalize(&CLASSIFY, &VERBALIZE, None, input, " ") } + +/// Normalize German text and retain the source span for every output token. +pub fn normalize_aligned(input: &str) -> Option { + driver::normalize_aligned(&CLASSIFY, &VERBALIZE, None, input, " ") +} diff --git a/src/fst/driver.rs b/src/fst/driver.rs index 40a9332..fda7f60 100644 --- a/src/fst/driver.rs +++ b/src/fst/driver.rs @@ -6,7 +6,8 @@ //! verbalizer only accepts fields in a specific order, so the driver tries //! every field permutation of each token until one verbalizes. -use super::engine::apply; +use super::engine::{apply, apply_with_alignment}; +use super::{AlignedNormalization, AlignedSpan, TokenKind}; use rustfst::prelude::*; /// A parsed tag value: a leaf string, a nested tag, or a bare boolean flag. @@ -20,6 +21,8 @@ enum Val { /// Recursive-descent parser over the classifier's tagged output. struct TagParser { chars: Vec, + byte_offsets: Vec, + byte_len: usize, pos: usize, } @@ -27,10 +30,19 @@ impl TagParser { fn new(s: &str) -> Self { TagParser { chars: s.chars().collect(), + byte_offsets: s.char_indices().map(|(offset, _)| offset).collect(), + byte_len: s.len(), pos: 0, } } + fn byte_position(&self) -> usize { + self.byte_offsets + .get(self.pos) + .copied() + .unwrap_or(self.byte_len) + } + fn skip_ws(&mut self) { while self.pos < self.chars.len() && self.chars[self.pos].is_whitespace() { self.pos += 1; @@ -79,6 +91,26 @@ impl TagParser { out } + /// Parse root fields while retaining the output byte boundary at which + /// each top-level `tokens { ... }` record ends. The selected FST path maps + /// that boundary back to an input byte position. + fn aligned_fields(&mut self) -> Vec<(String, Val, usize)> { + let mut out = Vec::new(); + loop { + self.skip_ws(); + if self.pos >= self.chars.len() || self.chars[self.pos] == '}' { + break; + } + let key = self.key(); + if key.is_empty() { + break; + } + let value = self.value(); + out.push((key, value, self.byte_position())); + } + out + } + /// Parse either a `: "quoted string"` value or a nested `{ ... }` tag. /// /// A quoted value ends at a `"` *followed by a space* (or end of input), so @@ -203,17 +235,87 @@ pub fn normalize( let mut parts = Vec::with_capacity(tokens.len()); for (k, v) in tokens { - let single = vec![(k, v)]; - let mut verbalized = None; - for candidate in permute(&single) { - if let Some(out) = apply(verbalize, &candidate) { - verbalized = Some(out); - break; - } + parts.push(verbalize_token(verbalize, k, v).unwrap_or_default()); + } + + finish_normalization(parts, post, input, sep) +} + +/// Run TN while retaining the input span consumed by every top-level NeMo +/// classifier token. +pub(super) fn normalize_aligned( + classify: &VectorFst, + verbalize: &VectorFst, + post: Option<&VectorFst>, + input: &str, + sep: &str, +) -> Option { + let classified = apply_with_alignment(classify, input)?; + let tokens = TagParser::new(&classified.output).aligned_fields(); + let mut parts = Vec::with_capacity(tokens.len()); + let mut spans = Vec::with_capacity(tokens.len()); + let mut previous_input_end = 0usize; + + for (key, value, output_end) in tokens { + let raw_input_end = source_char_boundary( + input, + classified + .input_at_output_boundary + .get(output_end) + .copied() + .unwrap_or(input.len()) + .min(input.len()), + ); + let (input_start, input_end) = + trim_source_span(input, previous_input_end.min(raw_input_end), raw_input_end); + previous_input_end = raw_input_end; + + let original = input[input_start..input_end].to_string(); + let kind = token_kind(&value, &original); + let normalized = verbalize_token(verbalize, key, value).unwrap_or_default(); + parts.push(normalized.clone()); + spans.push(AlignedSpan { + input_start, + input_end, + original, + normalized, + kind, + }); + } + + Some(AlignedNormalization { + normalized: finish_normalization(parts, post, input, sep), + spans, + }) +} + +fn source_char_boundary(input: &str, mut position: usize) -> usize { + while !input.is_char_boundary(position) { + position -= 1; + } + position +} + +fn verbalize_token( + verbalize: &VectorFst, + key: String, + value: Val, +) -> Option { + let single = vec![(key, value)]; + for candidate in permute(&single) { + if let Some(output) = apply(verbalize, &candidate) { + return Some(output); } - parts.push(verbalized.unwrap_or_default()); } + None +} +fn finish_normalization( + parts: Vec, + post: Option<&VectorFst>, + input: &str, + sep: &str, +) -> String { // NeMo's post-verbalization steps (normalize.py): collapse spaces, apply the // post-processing FST, Moses-detokenize, then re-align punctuation spacing to // the original input. @@ -225,6 +327,51 @@ pub fn normalize( post_process_punct(input, &moses_despace(&processed)) } +fn trim_source_span(input: &str, start: usize, end: usize) -> (usize, usize) { + let raw = &input[start..end]; + let trimmed_start = raw.trim_start_matches(char::is_whitespace); + let leading_bytes = raw.len() - trimmed_start.len(); + let trimmed = trimmed_start.trim_end_matches(char::is_whitespace); + (start + leading_bytes, start + leading_bytes + trimmed.len()) +} + +fn token_kind(value: &Val, original: &str) -> TokenKind { + let class = match value { + Val::Map(fields) => fields + .iter() + .map(|(key, _)| key.as_str()) + .find(|key| *key != "preserve_order"), + _ => None, + }; + + match class { + Some("cardinal") => TokenKind::Cardinal, + Some("ordinal") => TokenKind::Ordinal, + Some("decimal") => TokenKind::Decimal, + Some("fraction") => TokenKind::Fraction, + Some("time") => TokenKind::Time, + Some("measure") => TokenKind::Measure, + Some("percent") => TokenKind::Percent, + Some("date") => TokenKind::Date, + Some("telephone") => TokenKind::Telephone, + Some("money") => TokenKind::Money, + Some("electronic") => TokenKind::Electronic, + Some("verbatim") => TokenKind::Verbatim, + Some("letters") => TokenKind::Letters, + Some("abbreviation") => TokenKind::Abbreviation, + Some("name") | None + if !original.is_empty() + && original + .chars() + .all(|c| !c.is_alphanumeric() && !c.is_whitespace()) => + { + TokenKind::Punctuation + } + Some("name") | None => TokenKind::Word, + Some(other) => TokenKind::Other(other.to_string()), + } +} + /// Collapse runs of spaces to one and trim (NeMo's `SPACE_DUP` + strip). fn collapse_spaces(s: String) -> String { let mut out = String::with_capacity(s.len()); diff --git a/src/fst/en.rs b/src/fst/en.rs index 5aeaa19..b158aec 100644 --- a/src/fst/en.rs +++ b/src/fst/en.rs @@ -6,7 +6,7 @@ //! Grammars in `grammars/en/` are exported from NeMo-text-processing //! (Apache-2.0, pinned commit `1f1263579fe57ba7ed783cad3dddee710fcc5064`). -use super::{driver, load_gz}; +use super::{driver, load_gz, AlignedNormalization}; use lazy_static::lazy_static; use rustfst::prelude::*; @@ -32,3 +32,8 @@ lazy_static! { pub fn normalize(input: &str) -> String { driver::normalize(&CLASSIFY, &VERBALIZE, Some(&POST), input, " ") } + +/// Normalize English text and retain the source span for every output token. +pub fn normalize_aligned(input: &str) -> Option { + driver::normalize_aligned(&CLASSIFY, &VERBALIZE, Some(&POST), input, " ") +} diff --git a/src/fst/engine.rs b/src/fst/engine.rs index e5bb028..bf11027 100644 --- a/src/fst/engine.rs +++ b/src/fst/engine.rs @@ -17,6 +17,14 @@ use rustfst::algorithms::compose::compose; use rustfst::algorithms::rm_epsilon::rm_epsilon; use rustfst::prelude::*; +/// The selected transducer path, including the input position reached at each +/// output-byte boundary. The boundary table lets the higher-level token parser +/// recover which source bytes produced each top-level classifier token. +pub(super) struct AppliedPath { + pub output: String, + pub input_at_output_boundary: Vec, +} + /// Build a linear FST that accepts exactly the bytes of `s` (input == output). /// /// Working at the byte level keeps the engine encoding-agnostic: Chinese, @@ -39,14 +47,14 @@ fn byte_acceptor(s: &str) -> VectorFst { fst } -/// Return the output-label string of the lowest-weight path through an acyclic -/// FST, or `None` if it has no accepting path. +/// Return the input/output label pairs on the lowest-weight path through an +/// acyclic FST, or `None` if it has no accepting path. /// /// Iterative-DFS topological order, then a single relaxation pass /// (`dist[next] = min(dist[next], dist[s] + w)`) followed by a backtrack over /// the recorded predecessors. Matches OpenFST's tropical shortest-path on the /// acyclic FSTs this engine produces. -fn shortest_output(fst: &VectorFst) -> Option { +fn shortest_labels(fst: &VectorFst) -> Option> { let start = fst.start()?; let n = fst.num_states(); @@ -76,7 +84,7 @@ fn shortest_output(fst: &VectorFst) -> Option { // Relax edges in topological order. let inf = f32::INFINITY; let mut dist = vec![inf; n]; - let mut pred: Vec> = vec![None; n]; + let mut pred: Vec> = vec![None; n]; dist[start as usize] = 0.0; for &s in &order { if dist[s] == inf { @@ -87,7 +95,7 @@ fn shortest_output(fst: &VectorFst) -> Option { let ns = tr.nextstate as usize; if w < dist[ns] { dist[ns] = w; - pred[ns] = Some((s, tr.olabel)); + pred[ns] = Some((s, tr.ilabel, tr.olabel)); } } } @@ -108,30 +116,79 @@ fn shortest_output(fst: &VectorFst) -> Option { // Backtrack, collecting non-epsilon output labels. let mut f = best_final?; let mut labels = Vec::new(); - while let Some((p, olabel)) = pred[f] { - if olabel != 0 { - labels.push(olabel as u8); - } + while let Some((p, ilabel, olabel)) = pred[f] { + labels.push((ilabel, olabel)); f = p; } labels.reverse(); - Some(String::from_utf8_lossy(&labels).to_string()) + Some(labels) +} + +fn applied_path(labels: Vec<(u32, u32)>) -> Option { + let output_len = labels.iter().filter(|(_, output)| *output != 0).count(); + let mut output = Vec::with_capacity(output_len); + let mut input_at_output_boundary = vec![0usize; output_len + 1]; + let mut input_position = 0usize; + let mut output_position = 0usize; + + for (input, emitted) in labels { + if input != 0 { + input_position += 1; + } + if emitted != 0 { + output.push(emitted as u8); + output_position += 1; + } + // Input-only arcs between emitted bytes belong to the boundary that + // has most recently been reached. Updating it is important for spaces + // and deleted characters between adjacent classifier tokens. + input_at_output_boundary[output_position] = input_position; + } + + Some(AppliedPath { + output: String::from_utf8(output).ok()?, + input_at_output_boundary, + }) } /// Apply a transducer to `input`: compose, remove epsilons, take the tropical /// shortest path's output. Returns `None` if the input is not in the domain /// (empty composition) or the shortest path emits nothing. pub fn apply(fst: &VectorFst, input: &str) -> Option { + let labels = apply_labels(fst, input)?; + let output = labels + .into_iter() + .filter_map(|(_, output)| (output != 0).then_some(output as u8)) + .collect::>(); + if output.is_empty() { + None + } else { + String::from_utf8(output).ok() + } +} + +/// Apply a transducer while retaining input progress along the selected path. +/// +/// This is used only for classifier alignment. Verbalization callers that need +/// just the output should continue to use [`apply`]. +pub(super) fn apply_with_alignment( + fst: &VectorFst, + input: &str, +) -> Option { + let path = applied_path(apply_labels(fst, input)?)?; + if path.output.is_empty() { + None + } else { + Some(path) + } +} + +fn apply_labels(fst: &VectorFst, input: &str) -> Option> { let mut composed: VectorFst = compose(byte_acceptor(input), fst.clone()).ok()?; if composed.num_states() == 0 || composed.start().is_none() { return None; } rm_epsilon(&mut composed).ok()?; - let out = shortest_output(&composed)?; - if out.is_empty() { - None - } else { - Some(out) - } + shortest_labels(&composed) } diff --git a/src/fst/es.rs b/src/fst/es.rs index 3240af4..84a6191 100644 --- a/src/fst/es.rs +++ b/src/fst/es.rs @@ -6,7 +6,7 @@ //! Grammars in `grammars/es/` are exported from NeMo-text-processing //! (Apache-2.0, pinned commit `1f1263579fe57ba7ed783cad3dddee710fcc5064`). -use super::{driver, load_gz}; +use super::{driver, load_gz, AlignedNormalization}; use lazy_static::lazy_static; use rustfst::prelude::*; @@ -30,3 +30,8 @@ lazy_static! { pub fn normalize(input: &str) -> String { driver::normalize(&CLASSIFY, &VERBALIZE, None, input, " ") } + +/// Normalize Spanish text and retain the source span for every output token. +pub fn normalize_aligned(input: &str) -> Option { + driver::normalize_aligned(&CLASSIFY, &VERBALIZE, None, input, " ") +} diff --git a/src/fst/fr.rs b/src/fst/fr.rs index f787e22..8a5e908 100644 --- a/src/fst/fr.rs +++ b/src/fst/fr.rs @@ -6,7 +6,7 @@ //! Grammars in `grammars/fr/` are exported from NeMo-text-processing //! (Apache-2.0, pinned commit `1f1263579fe57ba7ed783cad3dddee710fcc5064`). -use super::{driver, load_gz}; +use super::{driver, load_gz, AlignedNormalization}; use lazy_static::lazy_static; use rustfst::prelude::*; @@ -30,3 +30,8 @@ lazy_static! { pub fn normalize(input: &str) -> String { driver::normalize(&CLASSIFY, &VERBALIZE, None, input, " ") } + +/// Normalize French text and retain the source span for every output token. +pub fn normalize_aligned(input: &str) -> Option { + driver::normalize_aligned(&CLASSIFY, &VERBALIZE, None, input, " ") +} diff --git a/src/fst/hi.rs b/src/fst/hi.rs index 9812471..3595de3 100644 --- a/src/fst/hi.rs +++ b/src/fst/hi.rs @@ -6,7 +6,7 @@ //! Grammars in `grammars/hi/` are exported from NeMo-text-processing //! (Apache-2.0, pinned commit `1f1263579fe57ba7ed783cad3dddee710fcc5064`). -use super::{driver, load_gz}; +use super::{driver, load_gz, AlignedNormalization}; use lazy_static::lazy_static; use rustfst::prelude::*; @@ -31,3 +31,8 @@ lazy_static! { pub fn normalize(input: &str) -> String { driver::normalize(&CLASSIFY, &VERBALIZE, Some(&POST), input, " ") } + +/// Normalize Hindi text and retain the source span for every output token. +pub fn normalize_aligned(input: &str) -> Option { + driver::normalize_aligned(&CLASSIFY, &VERBALIZE, Some(&POST), input, " ") +} diff --git a/src/fst/ja.rs b/src/fst/ja.rs index 7f6bef0..89f066d 100644 --- a/src/fst/ja.rs +++ b/src/fst/ja.rs @@ -6,7 +6,7 @@ //! Grammars in `grammars/ja/` are exported from NeMo-text-processing //! (Apache-2.0, pinned commit `1f1263579fe57ba7ed783cad3dddee710fcc5064`). -use super::{driver, load_gz}; +use super::{driver, load_gz, AlignedNormalization}; use lazy_static::lazy_static; use rustfst::prelude::*; @@ -30,3 +30,8 @@ lazy_static! { pub fn normalize(input: &str) -> String { driver::normalize(&CLASSIFY, &VERBALIZE, None, input, "") } + +/// Normalize Japanese text and retain the source span for every output token. +pub fn normalize_aligned(input: &str) -> Option { + driver::normalize_aligned(&CLASSIFY, &VERBALIZE, None, input, "") +} diff --git a/src/fst/mod.rs b/src/fst/mod.rs index 3a4881f..b30d913 100644 --- a/src/fst/mod.rs +++ b/src/fst/mod.rs @@ -35,6 +35,97 @@ use flate2::read::GzDecoder; use rustfst::prelude::*; use std::io::Read; +/// A semantic class assigned by the NeMo TN classifier. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum TokenKind { + Word, + Punctuation, + Cardinal, + Ordinal, + Decimal, + Fraction, + Time, + Measure, + Percent, + Date, + Telephone, + Money, + Electronic, + Verbatim, + Letters, + Abbreviation, + Other(String), +} + +impl TokenKind { + /// Stable lower-case name suitable for serialization and foreign-language + /// bindings. + pub fn as_str(&self) -> &str { + match self { + Self::Word => "word", + Self::Punctuation => "punctuation", + Self::Cardinal => "cardinal", + Self::Ordinal => "ordinal", + Self::Decimal => "decimal", + Self::Fraction => "fraction", + Self::Time => "time", + Self::Measure => "measure", + Self::Percent => "percent", + Self::Date => "date", + Self::Telephone => "telephone", + Self::Money => "money", + Self::Electronic => "electronic", + Self::Verbatim => "verbatim", + Self::Letters => "letters", + Self::Abbreviation => "abbreviation", + Self::Other(name) => name, + } + } +} + +/// One source span and the words produced for it by text normalization. +/// +/// Offsets are half-open UTF-8 byte offsets into the original input. Whitespace +/// between spans is deliberately excluded, so callers can recover it from the +/// gaps between adjacent ranges without losing the original formatting. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AlignedSpan { + pub input_start: usize, + pub input_end: usize, + pub original: String, + /// Direct verbalizer output for this classifier token. Sentence-level + /// punctuation and spacing cleanup is reflected in + /// [`AlignedNormalization::normalized`]. + pub normalized: String, + pub kind: TokenKind, +} + +/// Sentence-level TN output together with its source-to-normalized mapping. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AlignedNormalization { + /// The same sentence-level result returned by the corresponding + /// language's [`normalize`](crate::fst::en::normalize) function. + pub normalized: String, + pub spans: Vec, +} + +/// Normalize with NeMo's compiled FST grammar and preserve source spans. +/// +/// Supported language codes are `en`, `fr`, `es`, `de`, `zh`, `hi`, and `ja`. +/// Returns `None` for unsupported languages or when classification fails. +pub fn normalize_aligned(input: &str, lang: &str) -> Option { + match lang { + "en" => en::normalize_aligned(input), + "fr" => fr::normalize_aligned(input), + "es" => es::normalize_aligned(input), + "de" => de::normalize_aligned(input), + "zh" => zh::normalize_aligned(input), + "hi" => hi::normalize_aligned(input), + "ja" => ja::normalize_aligned(input), + _ => None, + } +} + /// Decompress a bundled `*.fst.gz` grammar and load it as an FST. fn load_gz(gz: &[u8]) -> VectorFst { let mut bytes = Vec::new(); diff --git a/src/fst/zh.rs b/src/fst/zh.rs index 63c5b09..ba0804a 100644 --- a/src/fst/zh.rs +++ b/src/fst/zh.rs @@ -6,7 +6,7 @@ //! Grammars in `grammars/zh/` are exported from NeMo-text-processing //! (Apache-2.0, pinned commit `1f1263579fe57ba7ed783cad3dddee710fcc5064`). -use super::{driver, load_gz}; +use super::{driver, load_gz, AlignedNormalization}; use lazy_static::lazy_static; use rustfst::prelude::*; @@ -31,3 +31,8 @@ lazy_static! { pub fn normalize(input: &str) -> String { driver::normalize(&CLASSIFY, &VERBALIZE, None, input, "") } + +/// Normalize Mandarin text and retain the source span for every output token. +pub fn normalize_aligned(input: &str) -> Option { + driver::normalize_aligned(&CLASSIFY, &VERBALIZE, None, input, "") +} diff --git a/src/wasm.rs b/src/wasm.rs index 7763abb..c397a0d 100644 --- a/src/wasm.rs +++ b/src/wasm.rs @@ -2,6 +2,9 @@ use wasm_bindgen::prelude::*; +#[cfg(feature = "fst-engine")] +use js_sys::{Array, Object, Reflect}; + use crate::{ custom_rules, normalize, normalize_sentence, normalize_sentence_lang, normalize_sentence_with_options, normalize_with_lang, normalize_with_options, tn_normalize, @@ -128,6 +131,53 @@ pub fn tn_normalize_sentence_with_max_span_lang_js( tn_normalize_sentence_with_max_span_lang(input, lang, max_span_tokens as usize) } +/// Compiled-FST TN with source-span alignment. +/// +/// Returns `null` unless the build enables both `wasm` and `fst-engine`, or +/// when `lang` is unsupported. Offsets are half-open UTF-8 byte offsets. +#[wasm_bindgen(js_name = tnFstNormalizeAligned)] +pub fn tn_fst_normalize_aligned_js(input: &str, lang: &str) -> JsValue { + #[cfg(not(feature = "fst-engine"))] + { + let _ = (input, lang); + JsValue::NULL + } + + #[cfg(feature = "fst-engine")] + { + let Some(alignment) = crate::fst::normalize_aligned(input, lang) else { + return JsValue::NULL; + }; + let result = Object::new(); + let spans = Array::new(); + for span in alignment.spans { + let item = Object::new(); + set_js_property( + &item, + "inputStart", + &JsValue::from_f64(span.input_start as f64), + ); + set_js_property(&item, "inputEnd", &JsValue::from_f64(span.input_end as f64)); + set_js_property(&item, "original", &JsValue::from_str(&span.original)); + set_js_property(&item, "normalized", &JsValue::from_str(&span.normalized)); + set_js_property(&item, "kind", &JsValue::from_str(span.kind.as_str())); + spans.push(&item); + } + set_js_property( + &result, + "normalized", + &JsValue::from_str(&alignment.normalized), + ); + set_js_property(&result, "spans", &spans); + result.into() + } +} + +#[cfg(feature = "fst-engine")] +fn set_js_property(object: &Object, name: &str, value: &JsValue) { + let _ = Reflect::set(object, &JsValue::from_str(name), value); +} + #[wasm_bindgen(js_name = addRule)] pub fn add_rule_js(spoken: &str, written: &str) { custom_rules::add_rule(spoken, written); diff --git a/swift-test/Sources/CNemoTextProcessing/include/nemo_text_processing.h b/swift-test/Sources/CNemoTextProcessing/include/nemo_text_processing.h index 3318749..99591c0 100644 --- a/swift-test/Sources/CNemoTextProcessing/include/nemo_text_processing.h +++ b/swift-test/Sources/CNemoTextProcessing/include/nemo_text_processing.h @@ -1,6 +1,7 @@ #ifndef NEMO_TEXT_PROCESSING_H #define NEMO_TEXT_PROCESSING_H +#include #include #ifdef __cplusplus @@ -35,6 +36,23 @@ char* nemo_tn_normalize_sentence_with_max_span(const char* input, uint32_t max_s /* Byte-exact NeMo TN via the compiled-FST engine (NULL if unavailable) */ char* nemo_tn_fst(const char* input, const char* lang); +typedef struct NemoTnAlignedSpan { + size_t input_start; + size_t input_end; + char* original; + char* normalized; + char* kind; +} NemoTnAlignedSpan; + +typedef struct NemoTnAlignment { + char* normalized; + NemoTnAlignedSpan* spans; + size_t span_count; +} NemoTnAlignment; + +NemoTnAlignment* nemo_tn_fst_aligned(const char* input, const char* lang); +void nemo_tn_alignment_free(NemoTnAlignment* alignment); + #ifdef __cplusplus } #endif diff --git a/swift/NemoTextProcessing.swift b/swift/NemoTextProcessing.swift index 1768010..2013640 100644 --- a/swift/NemoTextProcessing.swift +++ b/swift/NemoTextProcessing.swift @@ -1,5 +1,23 @@ import Foundation +/// One source span and the spoken-form words produced for it by TN. +public struct TNAlignedSpan: Sendable, Equatable { + /// Half-open UTF-8 byte range in the original input. + public let inputRange: Range + public let original: String + /// Direct verbalizer output for this classifier token. Use the containing + /// `TNAlignment.normalized` for sentence-level punctuation and spacing. + public let normalized: String + /// Stable NeMo class such as `word`, `money`, `date`, or `telephone`. + public let kind: String +} + +/// Sentence-level normalized text and its source-to-output span mapping. +public struct TNAlignment: Sendable, Equatable { + public let normalized: String + public let spans: [TNAlignedSpan] +} + /// Swift wrapper for NeMo Text Processing (Inverse Text Normalization). /// /// Converts spoken-form ASR output to written form: @@ -271,6 +289,59 @@ public enum NemoTextProcessing { return String(cString: resultPtr) } + /// Normalize through the compiled NeMo FST and retain the source span for + /// every normalized token. + /// + /// This API is available when the native library is built with the + /// `fst-engine` feature. Offsets are UTF-8 byte offsets, so slice + /// `input.utf8` rather than using them as `String.Index` values directly. + /// + /// - Parameters: + /// - input: Complete written-form sentence. + /// - language: One of `en`, `fr`, `es`, `de`, `zh`, `hi`, or `ja`. + /// - Returns: Normalized text and aligned spans, or `nil` when FST TN is + /// unavailable or the language is unsupported. + public static func tnNormalizeAligned(_ input: String, language: String) -> TNAlignment? { + guard let inputC = input.cString(using: .utf8), + let langC = language.cString(using: .utf8), + let resultPtr = nemo_tn_fst_aligned(inputC, langC) else { + return nil + } + defer { nemo_tn_alignment_free(resultPtr) } + + let result = resultPtr.pointee + guard let normalizedPtr = result.normalized else { + return nil + } + + let count = Int(result.span_count) + if count > 0 && result.spans == nil { + return nil + } + + let nativeSpans = UnsafeBufferPointer(start: result.spans, count: count) + var spans: [TNAlignedSpan] = [] + spans.reserveCapacity(count) + for native in nativeSpans { + guard let original = native.original, + let normalized = native.normalized, + let kind = native.kind else { + return nil + } + spans.append(TNAlignedSpan( + inputRange: Int(native.input_start).. #include #ifdef __cplusplus @@ -187,6 +188,39 @@ char* nemo_tn_normalize_sentence_with_max_span_lang(const char* input, const cha */ char* nemo_tn_fst(const char* input, const char* lang); +/** One aligned source span produced by compiled-FST text normalization. */ +typedef struct NemoTnAlignedSpan { + /** Half-open UTF-8 byte range in the original input. */ + size_t input_start; + size_t input_end; + /** Original source substring. Owned by the containing alignment. */ + char* original; + /** Direct verbalizer output for this classifier token. */ + char* normalized; + /** Stable semantic class, for example "word", "money", or "date". */ + char* kind; +} NemoTnAlignedSpan; + +/** Sentence-level normalized output and source-to-output span mapping. */ +typedef struct NemoTnAlignment { + char* normalized; + NemoTnAlignedSpan* spans; + size_t span_count; +} NemoTnAlignment; + +/** + * Normalize via the compiled-FST engine and retain source spans. + * + * Supported langs: "en", "zh", "ja", "fr", "es", "de", "hi". + * Returns NULL when fst-engine is unavailable, the language is unsupported, + * or the input is invalid. The result must be released exactly once with + * nemo_tn_alignment_free(). + */ +NemoTnAlignment* nemo_tn_fst_aligned(const char* input, const char* lang); + +/** Free a result returned by nemo_tn_fst_aligned(). NULL is allowed. */ +void nemo_tn_alignment_free(NemoTnAlignment* alignment); + /** * Free a string allocated by nemo_normalize or nemo_normalize_sentence. * From 594b84696d7bd110dcba7addf4be5406b2e90336 Mon Sep 17 00:00:00 2001 From: hungerdeng Date: Tue, 15 Sep 2026 21:58:26 +0800 Subject: [PATCH 2/4] feat: add Swift aligned TN CLI Provide a macOS SwiftPM executable that emits aligned TN results as JSON or JSONL. Select the host Rust archive and link it statically so the existing and aligned CLIs build on both Apple Silicon and Intel Macs. --- README.md | 27 ++- swift-test/Package.swift | 40 ++-- swift-test/Sources/nemo-tn-aligned/main.swift | 180 ++++++++++++++++++ 3 files changed, 226 insertions(+), 21 deletions(-) create mode 100644 swift-test/Sources/nemo-tn-aligned/main.swift diff --git a/README.md b/README.md index 14a9a35..06f4970 100644 --- a/README.md +++ b/README.md @@ -277,14 +277,35 @@ npm run wasm:publish ### CLI Tools ```bash -# Build the Rust library (release, with FFI) -cargo build --release --target aarch64-apple-darwin --features ffi +# Build the Rust library for this Mac's architecture. +RUST_TARGET="$(rustc -vV | sed -n 's/^host: //p')" +cargo build --release --target "$RUST_TARGET" --features "ffi,fst-engine" # Build Swift CLI tools cd swift-test && swift build ``` -Binaries are at `swift-test/.build/debug/nemo-itn` and `swift-test/.build/debug/nemo-tn`. +Binaries are at `swift-test/.build/debug/nemo-itn`, +`swift-test/.build/debug/nemo-tn`, and +`swift-test/.build/debug/nemo-tn-aligned`. + +#### nemo-tn +```bash +swift-test/.build/debug/nemo-tn -s 'The price is $1,234.56.' +# output: The price is one thousand two hundred and thirty four point five six dollars +``` + +#### nemo-tn-aligned +The aligned CLI emits compact JSON for argument input and JSON Lines for stdin: + +```bash +swift-test/.build/debug/nemo-tn-aligned --lang en 'The price is $1,234.56.' +# output: {"input":"The price is $1,234.56.","language":"en","normalized":"The price is one thousand two hundred and thirty four dollars fifty six cents.","spans":[{"input_end":3,"input_start":0,"kind":"word","normalized":"The","original":"The"},{"input_end":9,"input_start":4,"kind":"word","normalized":"price","original":"price"},{"input_end":12,"input_start":10,"kind":"word","normalized":"is","original":"is"},{"input_end":22,"input_start":13,"kind":"money","normalized":"one thousand two hundred and thirty four dollars fifty six cents","original":"$1,234.56"},{"input_end":23,"input_start":22,"kind":"punctuation","normalized":".","original":"."}]} +``` + +Each result contains `input`, `language`, the complete `normalized` text, and +`spans` with `input_start`, `input_end`, `original`, `normalized`, and `kind`. +Offsets are half-open UTF-8 byte offsets. ### Swift (XCFramework) diff --git a/swift-test/Package.swift b/swift-test/Package.swift index fb66d44..aa7d9db 100644 --- a/swift-test/Package.swift +++ b/swift-test/Package.swift @@ -2,6 +2,20 @@ import PackageDescription +#if arch(arm64) +let rustTarget = "aarch64-apple-darwin" +#elseif arch(x86_64) +let rustTarget = "x86_64-apple-darwin" +#else +#error("Unsupported macOS architecture") +#endif + +let rustLinkerSettings: [LinkerSetting] = [ + .unsafeFlags([ + "../target/\(rustTarget)/release/libtext_processing_rs.a" + ]) +] + let package = Package( name: "NemoTest", platforms: [.macOS(.v14)], @@ -13,32 +27,22 @@ let package = Package( .executableTarget( name: "NemoTest", dependencies: ["CNemoTextProcessing"], - linkerSettings: [ - .unsafeFlags([ - "-L../target/aarch64-apple-darwin/release", - "-ltext_processing_rs" - ]) - ] + linkerSettings: rustLinkerSettings ), .executableTarget( name: "nemo-itn", dependencies: ["CNemoTextProcessing"], - linkerSettings: [ - .unsafeFlags([ - "-L../target/aarch64-apple-darwin/release", - "-ltext_processing_rs" - ]) - ] + linkerSettings: rustLinkerSettings ), .executableTarget( name: "nemo-tn", dependencies: ["CNemoTextProcessing"], - linkerSettings: [ - .unsafeFlags([ - "-L../target/aarch64-apple-darwin/release", - "-ltext_processing_rs" - ]) - ] + linkerSettings: rustLinkerSettings + ), + .executableTarget( + name: "nemo-tn-aligned", + dependencies: ["CNemoTextProcessing"], + linkerSettings: rustLinkerSettings ), ] ) diff --git a/swift-test/Sources/nemo-tn-aligned/main.swift b/swift-test/Sources/nemo-tn-aligned/main.swift new file mode 100644 index 0000000..7a78009 --- /dev/null +++ b/swift-test/Sources/nemo-tn-aligned/main.swift @@ -0,0 +1,180 @@ +import Foundation +import CNemoTextProcessing + +// MARK: - Wrapper + +struct AlignedSpan: Encodable { + let inputStart: Int + let inputEnd: Int + let original: String + let normalized: String + let kind: String + + enum CodingKeys: String, CodingKey { + case inputStart = "input_start" + case inputEnd = "input_end" + case original + case normalized + case kind + } +} + +struct Alignment: Encodable { + let input: String + let language: String + let normalized: String + let spans: [AlignedSpan] +} + +enum Nemo { + static func tnNormalizeAligned(_ input: String, language: String) -> Alignment? { + guard let resultPtr = nemo_tn_fst_aligned(input, language) else { + return nil + } + defer { nemo_tn_alignment_free(resultPtr) } + + let result = resultPtr.pointee + guard let normalized = result.normalized else { + return nil + } + + let count = Int(result.span_count) + if count > 0 && result.spans == nil { + return nil + } + + let nativeSpans = UnsafeBufferPointer(start: result.spans, count: count) + var spans: [AlignedSpan] = [] + spans.reserveCapacity(count) + for span in nativeSpans { + guard let original = span.original, + let normalized = span.normalized, + let kind = span.kind else { + return nil + } + spans.append(AlignedSpan( + inputStart: Int(span.input_start), + inputEnd: Int(span.input_end), + original: String(cString: original), + normalized: String(cString: normalized), + kind: String(cString: kind) + )) + } + + return Alignment( + input: input, + language: language, + normalized: String(cString: normalized), + spans: spans + ) + } + + static var version: String { + guard let pointer = nemo_version() else { return "unknown" } + return String(cString: pointer) + } +} + +// MARK: - CLI + +let usage = """ + nemo-tn-aligned - FST Text Normalization with source alignment + + USAGE: + nemo-tn-aligned [-l ] + echo "text" | nemo-tn-aligned [-l ] + nemo-tn-aligned --version + nemo-tn-aligned --help + + OPTIONS: + -l, --lang en, fr, es, de, zh, hi, or ja (default: en) + + OUTPUT: + One compact JSON object per input. Stdin mode emits JSON Lines. + Source offsets are half-open UTF-8 byte offsets. + + EXAMPLE: + nemo-tn-aligned --lang en 'The price is $1,234.56.' + """ + +let args = Array(CommandLine.arguments.dropFirst()) + +if args.contains("--help") || args.contains("-h") { + print(usage) + exit(0) +} + +if args.contains("--version") || args.contains("-V") { + print("nemo-tn-aligned \(Nemo.version)") + exit(0) +} + +var language = "en" +var inputArgs: [String] = [] +var index = 0 +var optionsEnded = false +while index < args.count { + let argument = args[index] + if !optionsEnded && argument == "--" { + optionsEnded = true + } else if !optionsEnded && (argument == "-l" || argument == "--lang") { + index += 1 + guard index < args.count else { + fputs("nemo-tn-aligned: --lang requires a language code\n", stderr) + exit(1) + } + language = args[index] + } else if !optionsEnded && argument.hasPrefix("--lang=") { + language = String(argument.dropFirst("--lang=".count)) + } else { + inputArgs.append(argument) + } + index += 1 +} + +let supportedLanguages = ["en", "fr", "es", "de", "zh", "hi", "ja"] +guard supportedLanguages.contains(language) else { + fputs( + "nemo-tn-aligned: unsupported language '\(language)'; expected one of \(supportedLanguages.joined(separator: ", "))\n", + stderr + ) + exit(1) +} + +if inputArgs.isEmpty && isatty(fileno(stdin)) != 0 { + fputs(usage, stderr) + exit(1) +} + +let encoder = JSONEncoder() +encoder.outputFormatting = [.sortedKeys, .withoutEscapingSlashes] + +func emit(_ input: String) -> Bool { + guard let alignment = Nemo.tnNormalizeAligned(input, language: language) else { + fputs("nemo-tn-aligned: could not classify input as \(language) text\n", stderr) + return false + } + + do { + let data = try encoder.encode(alignment) + FileHandle.standardOutput.write(data) + FileHandle.standardOutput.write(Data([0x0A])) + return true + } catch { + fputs("nemo-tn-aligned: failed to encode JSON: \(error)\n", stderr) + return false + } +} + +if !inputArgs.isEmpty { + exit(emit(inputArgs.joined(separator: " ")) ? 0 : 1) +} + +var failed = false +while let line = readLine() { + guard !line.trimmingCharacters(in: .whitespaces).isEmpty else { continue } + if !emit(line) { + failed = true + } +} +exit(failed ? 1 : 0) From 1dd01c19a8922db947a29364db77543637d600a1 Mon Sep 17 00:00:00 2001 From: hungerdeng Date: Tue, 15 Sep 2026 22:37:08 +0800 Subject: [PATCH 3/4] test(fst): cover aligned text normalization Verify full-sentence mappings preserve semantic token kinds and UTF-8 byte ranges across English and Spanish inputs. Exercise currency, email, date, phone, unsupported-language behavior, and C FFI cleanup. --- tests/fst_alignment.rs | 220 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 tests/fst_alignment.rs diff --git a/tests/fst_alignment.rs b/tests/fst_alignment.rs new file mode 100644 index 0000000..a435d3f --- /dev/null +++ b/tests/fst_alignment.rs @@ -0,0 +1,220 @@ +//! Source-alignment coverage for the compiled-FST text normalizer. +//! +//! Requires the `fst-engine` feature: +//! `cargo test --features ffi,fst-engine --test fst_alignment`. +#![cfg(feature = "fst-engine")] + +use text_processing_rs::fst::{self, AlignedNormalization, AlignedSpan, TokenKind}; + +fn assert_source_ranges(input: &str, alignment: &AlignedNormalization) { + let mut previous_end = 0; + + for span in &alignment.spans { + assert!(span.input_start >= previous_end, "source spans overlap"); + assert!( + span.input_start <= span.input_end, + "source span is reversed" + ); + assert!(input.is_char_boundary(span.input_start)); + assert!(input.is_char_boundary(span.input_end)); + assert_eq!(&input[span.input_start..span.input_end], span.original); + previous_end = span.input_end; + } +} + +#[test] +fn aligns_a_money_span_within_a_complete_sentence() { + let input = "The price is $1,234.56."; + let alignment = fst::normalize_aligned(input, "en").expect("English input should normalize"); + + assert_eq!( + alignment.normalized, + "The price is one thousand two hundred and thirty four dollars fifty six cents." + ); + assert_eq!( + alignment.spans, + vec![ + AlignedSpan { + input_start: 0, + input_end: 3, + original: "The".into(), + normalized: "The".into(), + kind: TokenKind::Word, + }, + AlignedSpan { + input_start: 4, + input_end: 9, + original: "price".into(), + normalized: "price".into(), + kind: TokenKind::Word, + }, + AlignedSpan { + input_start: 10, + input_end: 12, + original: "is".into(), + normalized: "is".into(), + kind: TokenKind::Word, + }, + AlignedSpan { + input_start: 13, + input_end: 22, + original: "$1,234.56".into(), + normalized: "one thousand two hundred and thirty four dollars fifty six cents" + .into(), + kind: TokenKind::Money, + }, + AlignedSpan { + input_start: 22, + input_end: 23, + original: ".".into(), + normalized: ".".into(), + kind: TokenKind::Punctuation, + }, + ] + ); + assert_source_ranges(input, &alignment); + assert_eq!(alignment.normalized, fst::en::normalize(input)); +} + +#[test] +fn reports_utf8_byte_offsets() { + let input = "Café costs €12.50."; + let alignment = fst::normalize_aligned(input, "en").expect("English input should normalize"); + + assert_eq!(alignment.spans[0].original, "Café"); + assert_eq!( + (alignment.spans[0].input_start, alignment.spans[0].input_end), + (0, 5) + ); + assert_eq!(alignment.spans[2].original, "€12.50"); + assert_eq!( + (alignment.spans[2].input_start, alignment.spans[2].input_end), + (12, 20) + ); + assert_eq!(alignment.spans[2].kind, TokenKind::Money); + assert_source_ranges(input, &alignment); +} + +#[test] +fn distinguishes_electronic_date_and_punctuation_spans() { + let input = "Email jane.doe@example.com on 12/31/2025."; + let alignment = fst::normalize_aligned(input, "en").expect("English input should normalize"); + + let mapped = alignment + .spans + .iter() + .map(|span| { + ( + span.original.as_str(), + span.normalized.as_str(), + span.kind.as_str(), + ) + }) + .collect::>(); + assert_eq!( + mapped, + vec![ + ("Email", "Email", "word"), + ( + "jane.doe@example.com", + "jane dot doe at example dot com", + "electronic", + ), + ("on", "on", "word"), + ( + "12/31/2025", + "december thirty first twenty twenty five", + "date", + ), + (".", ".", "punctuation"), + ] + ); + assert_source_ranges(input, &alignment); +} + +#[test] +fn classifies_a_phone_number_inside_a_sentence() { + let input = "Call 415-555-0123."; + let alignment = fst::normalize_aligned(input, "en").expect("English input should normalize"); + + assert_eq!( + alignment.normalized, + "Call four one five, five five five, zero one two three." + ); + assert_eq!(alignment.spans[1].original, "415-555-0123"); + assert_eq!( + alignment.spans[1].normalized, + "four one five, five five five, zero one two three" + ); + assert_eq!(alignment.spans[1].kind, TokenKind::Telephone); + assert_eq!( + (alignment.spans[1].input_start, alignment.spans[1].input_end), + (5, 17) + ); + assert_source_ranges(input, &alignment); +} + +#[test] +fn aligns_spanish_currency_with_non_ascii_offsets() { + let input = "El precio es 1.234,56 €."; + let alignment = fst::normalize_aligned(input, "es").expect("Spanish input should normalize"); + + assert_eq!( + alignment.normalized, + "El precio es mil doscientos treinta y cuatro coma cincuenta y seis euros." + ); + assert_eq!(alignment.spans[3].original, "1.234,56 €"); + assert_eq!( + alignment.spans[3].normalized, + "mil doscientos treinta y cuatro coma cincuenta y seis euros" + ); + assert_eq!(alignment.spans[3].kind, TokenKind::Money); + assert_eq!( + (alignment.spans[3].input_start, alignment.spans[3].input_end), + (13, 25) + ); + assert_source_ranges(input, &alignment); + assert_eq!(alignment.normalized, fst::es::normalize(input)); +} + +#[cfg(feature = "ffi")] +#[test] +fn exposes_alignment_through_the_c_ffi() { + use std::ffi::{CStr, CString}; + + use text_processing_rs::ffi::{nemo_tn_alignment_free, nemo_tn_fst_aligned}; + + unsafe { + let input = CString::new("The price is $1,234.56.").unwrap(); + let en = CString::new("en").unwrap(); + let result = nemo_tn_fst_aligned(input.as_ptr(), en.as_ptr()); + + assert!(!result.is_null()); + let alignment = &*result; + assert_eq!(alignment.span_count, 5); + assert_eq!( + CStr::from_ptr(alignment.normalized).to_str().unwrap(), + "The price is one thousand two hundred and thirty four dollars fifty six cents." + ); + + let spans = std::slice::from_raw_parts(alignment.spans, alignment.span_count); + let money = &spans[3]; + assert_eq!((money.input_start, money.input_end), (13, 22)); + assert_eq!( + CStr::from_ptr(money.original).to_str().unwrap(), + "$1,234.56" + ); + assert_eq!(CStr::from_ptr(money.kind).to_str().unwrap(), "money"); + + nemo_tn_alignment_free(result); + nemo_tn_alignment_free(std::ptr::null_mut()); + + let unsupported = CString::new("xx").unwrap(); + assert!(nemo_tn_fst_aligned(input.as_ptr(), unsupported.as_ptr()).is_null()); + } +} + +#[test] +fn rejects_an_unsupported_language() { + assert!(fst::normalize_aligned("$12.50", "xx").is_none()); +} From 0c9c23e415702edda97fa0776dbccb50549b0d5d Mon Sep 17 00:00:00 2001 From: hungerdeng Date: Fri, 18 Sep 2026 16:18:50 +0800 Subject: [PATCH 4/4] docs: document compiled-FST architecture and alignment contract Record the optional fst-engine surface and its UTF-8 byte-range alignment contract so future Rust, FFI, WASM, and Swift changes follow the same feature boundaries. --- AGENTS.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6dcc783..4cf4d18 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,6 +7,7 @@ cargo build # Build project cargo build --release # Release build cargo build --features ffi # Build with C FFI bindings cargo build --features wasm --target wasm32-unknown-unknown # Build for WebAssembly +cargo build --features fst-engine # Build the optional compiled NeMo FST TN engine cargo test # Run all tests cargo test --test en_tests # Run a single integration test file cargo test en_tn_tests::test_name # Run a single test by name @@ -18,6 +19,7 @@ cargo clippy --all-targets # Lint - **src/itn/**: ITN taggers (spoken → written, Inverse Text Normalization) per language (en, de, es, fr, hi, ja, zh) - **src/tn/**: TN taggers (written → spoken, Text Normalization) +- **src/fst/**: Optional compiled NeMo FST TN engine and source-aligned TN spans, gated by `fst-engine` - **src/custom_rules.rs**: User-defined custom normalization rules (highest priority) - **src/ffi.rs**: C FFI bindings for Swift/Python integration (gated by `ffi` feature) - **src/wasm.rs**: JavaScript-callable wasm bindings (gated by `wasm` feature) @@ -43,8 +45,8 @@ Taggers are tried in order of specificity (most specific first). If no tagger ma - **NEVER** create simplified or stub versions — implement full solutions or consult first - **NEVER** introduce mock data or fabricated test cases — use realistic spoken/written forms - Add unit/integration tests when adding new taggers or rules -- Keep all changes local to the appropriate language module under `src/itn//` or `src/tn//` -- Maintain feature-flag isolation: code behind `ffi` and `wasm` must not leak into the default build +- Keep language-specific rule changes local to `src/itn//` or `src/tn//`; keep compiled-FST behavior in `src/fst/` and its bindings in the corresponding consumer surfaces +- Maintain feature-flag isolation: `ffi`, `wasm`, and `fst-engine` code and dependencies must not leak into the default build ## Code Style (rustfmt + clippy) @@ -67,6 +69,7 @@ Taggers are tried in order of specificity (most specific first). If no tagger ma - C FFI lives in `src/ffi.rs` behind the `ffi` feature; consumed by the Swift package in `swift/` - Wasm bindings live in `src/wasm.rs` behind the `wasm` feature; consumed by `wasm-tests/` +- Compiled-FST alignment uses `fst::normalize_aligned` and half-open UTF-8 byte offsets; its Rust API is gated by `fst-engine`, and its C/WASM exports must preserve the same contract - Run `cargo build --features ffi` and the Swift tests under `swift-test/` after touching FFI signatures - Run the wasm test suite after touching `wasm.rs` or anything affecting the wasm-exposed surface