diff --git a/AGENTS.md b/AGENTS.md index 6dcc783..4cf4d18 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,6 +7,7 @@ cargo build # Build project cargo build --release # Release build cargo build --features ffi # Build with C FFI bindings cargo build --features wasm --target wasm32-unknown-unknown # Build for WebAssembly +cargo build --features fst-engine # Build the optional compiled NeMo FST TN engine cargo test # Run all tests cargo test --test en_tests # Run a single integration test file cargo test en_tn_tests::test_name # Run a single test by name @@ -18,6 +19,7 @@ cargo clippy --all-targets # Lint - **src/itn/**: ITN taggers (spoken → written, Inverse Text Normalization) per language (en, de, es, fr, hi, ja, zh) - **src/tn/**: TN taggers (written → spoken, Text Normalization) +- **src/fst/**: Optional compiled NeMo FST TN engine and source-aligned TN spans, gated by `fst-engine` - **src/custom_rules.rs**: User-defined custom normalization rules (highest priority) - **src/ffi.rs**: C FFI bindings for Swift/Python integration (gated by `ffi` feature) - **src/wasm.rs**: JavaScript-callable wasm bindings (gated by `wasm` feature) @@ -43,8 +45,8 @@ Taggers are tried in order of specificity (most specific first). If no tagger ma - **NEVER** create simplified or stub versions — implement full solutions or consult first - **NEVER** introduce mock data or fabricated test cases — use realistic spoken/written forms - Add unit/integration tests when adding new taggers or rules -- Keep all changes local to the appropriate language module under `src/itn//` or `src/tn//` -- Maintain feature-flag isolation: code behind `ffi` and `wasm` must not leak into the default build +- Keep language-specific rule changes local to `src/itn//` or `src/tn//`; keep compiled-FST behavior in `src/fst/` and its bindings in the corresponding consumer surfaces +- Maintain feature-flag isolation: `ffi`, `wasm`, and `fst-engine` code and dependencies must not leak into the default build ## Code Style (rustfmt + clippy) @@ -67,6 +69,7 @@ Taggers are tried in order of specificity (most specific first). If no tagger ma - C FFI lives in `src/ffi.rs` behind the `ffi` feature; consumed by the Swift package in `swift/` - Wasm bindings live in `src/wasm.rs` behind the `wasm` feature; consumed by `wasm-tests/` +- Compiled-FST alignment uses `fst::normalize_aligned` and half-open UTF-8 byte offsets; its Rust API is gated by `fst-engine`, and its C/WASM exports must preserve the same contract - Run `cargo build --features ffi` and the Swift tests under `swift-test/` after touching FFI signatures - Run the wasm test suite after touching `wasm.rs` or anything affecting the wasm-exposed surface diff --git a/Cargo.lock b/Cargo.lock index 9502b59..5c109bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,6 +116,30 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + [[package]] name = "generic-array" version = "1.4.3" @@ -158,6 +182,17 @@ dependencies = [ "either", ] +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -232,6 +267,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -427,6 +468,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "superslice" version = "1.0.0" @@ -463,6 +510,7 @@ version = "0.3.0" dependencies = [ "console_error_panic_hook", "flate2", + "js-sys", "lazy_static", "proptest", "rustfst", diff --git a/Cargo.toml b/Cargo.toml index 2a6825f..e225683 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ crate-type = ["lib", "staticlib", "cdylib"] [dependencies] lazy_static = "1" wasm-bindgen = { version = "0.2", optional = true } +js-sys = { version = "0.3", optional = true } console_error_panic_hook = { version = "0.1", optional = true } # Optional: byte-exact NeMo parity via compiled weighted FST grammars. # Off by default — the pure-Rust rule path carries no rustfst/OpenFST weight. @@ -26,7 +27,7 @@ proptest = "1" [features] default = [] ffi = [] # Enable C FFI bindings -wasm = ["dep:wasm-bindgen", "dep:console_error_panic_hook"] +wasm = ["dep:wasm-bindgen", "dep:js-sys", "dep:console_error_panic_hook"] # Byte-exact NeMo parity engine (runs NeMo's compiled grammars via rustfst). # Adds rustfst + flate2 and ~7 MB of gzipped grammars (all languages combined). fst-engine = ["dep:rustfst", "dep:flate2"] diff --git a/README.md b/README.md index 7466c80..06f4970 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,29 @@ let result = tn_normalize_sentence("I paid $5 for 23 items"); assert_eq!(result, "I paid five dollars for twenty three items"); ``` +Compiled-FST TN can also retain the source range and semantic class for every +normalized span. Offsets are half-open UTF-8 byte offsets: + +```rust +use text_processing_rs::fst; + +let result = fst::normalize_aligned("The price is $1,234.56.", "en").unwrap(); +assert_eq!( + result.normalized, + "The price is one thousand two hundred and thirty four dollars fifty six cents." +); + +let money = &result.spans[3]; +assert_eq!(money.input_start, 13); +assert_eq!(money.input_end, 22); +assert_eq!(money.original, "$1,234.56"); +assert_eq!(money.kind.as_str(), "money"); +``` + +Build this API with `--features fst-engine`. It uses the same compiled NeMo +classifier and verbalizer as `fst::::normalize`, rather than recovering +alignment by diffing the final strings. + ### Swift ```swift @@ -149,6 +172,16 @@ let itnFr = NemoTextProcessing.normalizeSentence("j'ai vingt et un ans", languag let tn = NemoTextProcessing.tnNormalizeSentence("I paid $5 for 23 items") // "I paid five dollars for twenty three items" + +if let aligned = NemoTextProcessing.tnNormalizeAligned( + "The price is $1,234.56.", + language: "en" +) { + let money = aligned.spans[3] + // money.original == "$1,234.56" + // money.normalized == "one thousand ... dollars fifty six cents" + // money.inputRange == 13..<22, money.kind == "money" +} ``` ### CLI @@ -215,6 +248,7 @@ echo "2:30 PM" | nemo-tn # → two thirty p m - Phone numbers, IP addresses, SSN - Case preservation for proper nouns and abbreviations - Sentence-level normalization with sliding window span matching +- Source-to-normalized span alignment with semantic classes (compiled FST) - Custom rules for domain-specific terms - C FFI for integration with Swift, Python, and other languages @@ -243,14 +277,35 @@ npm run wasm:publish ### CLI Tools ```bash -# Build the Rust library (release, with FFI) -cargo build --release --target aarch64-apple-darwin --features ffi +# Build the Rust library for this Mac's architecture. +RUST_TARGET="$(rustc -vV | sed -n 's/^host: //p')" +cargo build --release --target "$RUST_TARGET" --features "ffi,fst-engine" # Build Swift CLI tools cd swift-test && swift build ``` -Binaries are at `swift-test/.build/debug/nemo-itn` and `swift-test/.build/debug/nemo-tn`. +Binaries are at `swift-test/.build/debug/nemo-itn`, +`swift-test/.build/debug/nemo-tn`, and +`swift-test/.build/debug/nemo-tn-aligned`. + +#### nemo-tn +```bash +swift-test/.build/debug/nemo-tn -s 'The price is $1,234.56.' +# output: The price is one thousand two hundred and thirty four point five six dollars +``` + +#### nemo-tn-aligned +The aligned CLI emits compact JSON for argument input and JSON Lines for stdin: + +```bash +swift-test/.build/debug/nemo-tn-aligned --lang en 'The price is $1,234.56.' +# output: {"input":"The price is $1,234.56.","language":"en","normalized":"The price is one thousand two hundred and thirty four dollars fifty six cents.","spans":[{"input_end":3,"input_start":0,"kind":"word","normalized":"The","original":"The"},{"input_end":9,"input_start":4,"kind":"word","normalized":"price","original":"price"},{"input_end":12,"input_start":10,"kind":"word","normalized":"is","original":"is"},{"input_end":22,"input_start":13,"kind":"money","normalized":"one thousand two hundred and thirty four dollars fifty six cents","original":"$1,234.56"},{"input_end":23,"input_start":22,"kind":"punctuation","normalized":".","original":"."}]} +``` + +Each result contains `input`, `language`, the complete `normalized` text, and +`spans` with `input_start`, `input_end`, `original`, `normalized`, and `kind`. +Offsets are half-open UTF-8 byte offsets. ### Swift (XCFramework) diff --git a/src/ffi.rs b/src/ffi.rs index b35926e..f1c54c1 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -10,6 +10,27 @@ use crate::{ tn_normalize_sentence_with_max_span_lang, NormalizeOptions, }; +/// One source-to-normalized TN span returned through the C ABI. +/// +/// All strings are owned by the containing [`NemoTnAlignment`] and remain valid +/// until [`nemo_tn_alignment_free`] is called. +#[repr(C)] +pub struct NemoTnAlignedSpan { + pub input_start: usize, + pub input_end: usize, + pub original: *mut c_char, + pub normalized: *mut c_char, + pub kind: *mut c_char, +} + +/// Sentence-level compiled-FST TN output and its source-span mapping. +#[repr(C)] +pub struct NemoTnAlignment { + pub normalized: *mut c_char, + pub spans: *mut NemoTnAlignedSpan, + pub span_count: usize, +} + /// Build [`NormalizeOptions`] from FFI primitives. /// /// `concat_compound_numbers`: any non-zero value enables concat behavior @@ -494,6 +515,128 @@ pub unsafe extern "C" fn nemo_tn_fst(input: *const c_char, lang: *const c_char) } } +/// Normalize with the compiled-FST engine and retain every source span. +/// +/// Offsets are half-open UTF-8 byte offsets into `input`. The returned object +/// and every pointer it owns must be released with +/// [`nemo_tn_alignment_free`]. +/// +/// Returns null when the library was built without `fst-engine`, the language +/// is unsupported, either input string is invalid UTF-8, or allocation of a C +/// string fails. +/// +/// # Safety +/// - `input` and `lang` must be valid null-terminated UTF-8 strings. +/// - The result must be freed exactly once with [`nemo_tn_alignment_free`]. +#[no_mangle] +#[cfg(feature = "fst-engine")] +pub unsafe extern "C" fn nemo_tn_fst_aligned( + input: *const c_char, + lang: *const c_char, +) -> *mut NemoTnAlignment { + if input.is_null() || lang.is_null() { + return ptr::null_mut(); + } + let input_str = match CStr::from_ptr(input).to_str() { + Ok(s) => s, + Err(_) => return ptr::null_mut(), + }; + let lang_str = match CStr::from_ptr(lang).to_str() { + Ok(s) => s, + Err(_) => return ptr::null_mut(), + }; + + let Some(alignment) = crate::fst::normalize_aligned(input_str, lang_str) else { + return ptr::null_mut(); + }; + let Ok(normalized) = CString::new(alignment.normalized) else { + return ptr::null_mut(); + }; + + let converted = alignment + .spans + .into_iter() + .map(|span| { + Some(( + span.input_start, + span.input_end, + CString::new(span.original).ok()?, + CString::new(span.normalized).ok()?, + CString::new(span.kind.as_str()).ok()?, + )) + }) + .collect::>>(); + let Some(converted) = converted else { + return ptr::null_mut(); + }; + + let spans = converted + .into_iter() + .map( + |(input_start, input_end, original, normalized, kind)| NemoTnAlignedSpan { + input_start, + input_end, + original: original.into_raw(), + normalized: normalized.into_raw(), + kind: kind.into_raw(), + }, + ) + .collect::>() + .into_boxed_slice(); + let span_count = spans.len(); + let spans = Box::into_raw(spans) as *mut NemoTnAlignedSpan; + + Box::into_raw(Box::new(NemoTnAlignment { + normalized: normalized.into_raw(), + spans, + span_count, + })) +} + +#[no_mangle] +#[cfg(not(feature = "fst-engine"))] +pub unsafe extern "C" fn nemo_tn_fst_aligned( + _input: *const c_char, + _lang: *const c_char, +) -> *mut NemoTnAlignment { + ptr::null_mut() +} + +/// Free a result returned by [`nemo_tn_fst_aligned`]. +/// +/// Passing null is allowed. +/// +/// # Safety +/// - `alignment` must be null or a pointer returned by +/// [`nemo_tn_fst_aligned`]. +/// - The pointer must not be freed more than once. +#[no_mangle] +pub unsafe extern "C" fn nemo_tn_alignment_free(alignment: *mut NemoTnAlignment) { + if alignment.is_null() { + return; + } + + let alignment = Box::from_raw(alignment); + if !alignment.normalized.is_null() { + drop(CString::from_raw(alignment.normalized)); + } + if !alignment.spans.is_null() { + let slice = ptr::slice_from_raw_parts_mut(alignment.spans, alignment.span_count); + let mut spans = Box::from_raw(slice); + for span in spans.iter_mut() { + if !span.original.is_null() { + drop(CString::from_raw(span.original)); + } + if !span.normalized.is_null() { + drop(CString::from_raw(span.normalized)); + } + if !span.kind.is_null() { + drop(CString::from_raw(span.kind)); + } + } + } +} + #[cfg(feature = "fst-engine")] fn fst_normalize(input: &str, lang: &str) -> Option { use crate::fst; diff --git a/src/fst/de.rs b/src/fst/de.rs index b6e3b0c..a33bda1 100644 --- a/src/fst/de.rs +++ b/src/fst/de.rs @@ -6,7 +6,7 @@ //! Grammars in `grammars/de/` are exported from NeMo-text-processing //! (Apache-2.0, pinned commit `1f1263579fe57ba7ed783cad3dddee710fcc5064`). -use super::{driver, load_gz}; +use super::{driver, load_gz, AlignedNormalization}; use lazy_static::lazy_static; use rustfst::prelude::*; @@ -30,3 +30,8 @@ lazy_static! { pub fn normalize(input: &str) -> String { driver::normalize(&CLASSIFY, &VERBALIZE, None, input, " ") } + +/// Normalize German text and retain the source span for every output token. +pub fn normalize_aligned(input: &str) -> Option { + driver::normalize_aligned(&CLASSIFY, &VERBALIZE, None, input, " ") +} diff --git a/src/fst/driver.rs b/src/fst/driver.rs index 40a9332..fda7f60 100644 --- a/src/fst/driver.rs +++ b/src/fst/driver.rs @@ -6,7 +6,8 @@ //! verbalizer only accepts fields in a specific order, so the driver tries //! every field permutation of each token until one verbalizes. -use super::engine::apply; +use super::engine::{apply, apply_with_alignment}; +use super::{AlignedNormalization, AlignedSpan, TokenKind}; use rustfst::prelude::*; /// A parsed tag value: a leaf string, a nested tag, or a bare boolean flag. @@ -20,6 +21,8 @@ enum Val { /// Recursive-descent parser over the classifier's tagged output. struct TagParser { chars: Vec, + byte_offsets: Vec, + byte_len: usize, pos: usize, } @@ -27,10 +30,19 @@ impl TagParser { fn new(s: &str) -> Self { TagParser { chars: s.chars().collect(), + byte_offsets: s.char_indices().map(|(offset, _)| offset).collect(), + byte_len: s.len(), pos: 0, } } + fn byte_position(&self) -> usize { + self.byte_offsets + .get(self.pos) + .copied() + .unwrap_or(self.byte_len) + } + fn skip_ws(&mut self) { while self.pos < self.chars.len() && self.chars[self.pos].is_whitespace() { self.pos += 1; @@ -79,6 +91,26 @@ impl TagParser { out } + /// Parse root fields while retaining the output byte boundary at which + /// each top-level `tokens { ... }` record ends. The selected FST path maps + /// that boundary back to an input byte position. + fn aligned_fields(&mut self) -> Vec<(String, Val, usize)> { + let mut out = Vec::new(); + loop { + self.skip_ws(); + if self.pos >= self.chars.len() || self.chars[self.pos] == '}' { + break; + } + let key = self.key(); + if key.is_empty() { + break; + } + let value = self.value(); + out.push((key, value, self.byte_position())); + } + out + } + /// Parse either a `: "quoted string"` value or a nested `{ ... }` tag. /// /// A quoted value ends at a `"` *followed by a space* (or end of input), so @@ -203,17 +235,87 @@ pub fn normalize( let mut parts = Vec::with_capacity(tokens.len()); for (k, v) in tokens { - let single = vec![(k, v)]; - let mut verbalized = None; - for candidate in permute(&single) { - if let Some(out) = apply(verbalize, &candidate) { - verbalized = Some(out); - break; - } + parts.push(verbalize_token(verbalize, k, v).unwrap_or_default()); + } + + finish_normalization(parts, post, input, sep) +} + +/// Run TN while retaining the input span consumed by every top-level NeMo +/// classifier token. +pub(super) fn normalize_aligned( + classify: &VectorFst, + verbalize: &VectorFst, + post: Option<&VectorFst>, + input: &str, + sep: &str, +) -> Option { + let classified = apply_with_alignment(classify, input)?; + let tokens = TagParser::new(&classified.output).aligned_fields(); + let mut parts = Vec::with_capacity(tokens.len()); + let mut spans = Vec::with_capacity(tokens.len()); + let mut previous_input_end = 0usize; + + for (key, value, output_end) in tokens { + let raw_input_end = source_char_boundary( + input, + classified + .input_at_output_boundary + .get(output_end) + .copied() + .unwrap_or(input.len()) + .min(input.len()), + ); + let (input_start, input_end) = + trim_source_span(input, previous_input_end.min(raw_input_end), raw_input_end); + previous_input_end = raw_input_end; + + let original = input[input_start..input_end].to_string(); + let kind = token_kind(&value, &original); + let normalized = verbalize_token(verbalize, key, value).unwrap_or_default(); + parts.push(normalized.clone()); + spans.push(AlignedSpan { + input_start, + input_end, + original, + normalized, + kind, + }); + } + + Some(AlignedNormalization { + normalized: finish_normalization(parts, post, input, sep), + spans, + }) +} + +fn source_char_boundary(input: &str, mut position: usize) -> usize { + while !input.is_char_boundary(position) { + position -= 1; + } + position +} + +fn verbalize_token( + verbalize: &VectorFst, + key: String, + value: Val, +) -> Option { + let single = vec![(key, value)]; + for candidate in permute(&single) { + if let Some(output) = apply(verbalize, &candidate) { + return Some(output); } - parts.push(verbalized.unwrap_or_default()); } + None +} +fn finish_normalization( + parts: Vec, + post: Option<&VectorFst>, + input: &str, + sep: &str, +) -> String { // NeMo's post-verbalization steps (normalize.py): collapse spaces, apply the // post-processing FST, Moses-detokenize, then re-align punctuation spacing to // the original input. @@ -225,6 +327,51 @@ pub fn normalize( post_process_punct(input, &moses_despace(&processed)) } +fn trim_source_span(input: &str, start: usize, end: usize) -> (usize, usize) { + let raw = &input[start..end]; + let trimmed_start = raw.trim_start_matches(char::is_whitespace); + let leading_bytes = raw.len() - trimmed_start.len(); + let trimmed = trimmed_start.trim_end_matches(char::is_whitespace); + (start + leading_bytes, start + leading_bytes + trimmed.len()) +} + +fn token_kind(value: &Val, original: &str) -> TokenKind { + let class = match value { + Val::Map(fields) => fields + .iter() + .map(|(key, _)| key.as_str()) + .find(|key| *key != "preserve_order"), + _ => None, + }; + + match class { + Some("cardinal") => TokenKind::Cardinal, + Some("ordinal") => TokenKind::Ordinal, + Some("decimal") => TokenKind::Decimal, + Some("fraction") => TokenKind::Fraction, + Some("time") => TokenKind::Time, + Some("measure") => TokenKind::Measure, + Some("percent") => TokenKind::Percent, + Some("date") => TokenKind::Date, + Some("telephone") => TokenKind::Telephone, + Some("money") => TokenKind::Money, + Some("electronic") => TokenKind::Electronic, + Some("verbatim") => TokenKind::Verbatim, + Some("letters") => TokenKind::Letters, + Some("abbreviation") => TokenKind::Abbreviation, + Some("name") | None + if !original.is_empty() + && original + .chars() + .all(|c| !c.is_alphanumeric() && !c.is_whitespace()) => + { + TokenKind::Punctuation + } + Some("name") | None => TokenKind::Word, + Some(other) => TokenKind::Other(other.to_string()), + } +} + /// Collapse runs of spaces to one and trim (NeMo's `SPACE_DUP` + strip). fn collapse_spaces(s: String) -> String { let mut out = String::with_capacity(s.len()); diff --git a/src/fst/en.rs b/src/fst/en.rs index 5aeaa19..b158aec 100644 --- a/src/fst/en.rs +++ b/src/fst/en.rs @@ -6,7 +6,7 @@ //! Grammars in `grammars/en/` are exported from NeMo-text-processing //! (Apache-2.0, pinned commit `1f1263579fe57ba7ed783cad3dddee710fcc5064`). -use super::{driver, load_gz}; +use super::{driver, load_gz, AlignedNormalization}; use lazy_static::lazy_static; use rustfst::prelude::*; @@ -32,3 +32,8 @@ lazy_static! { pub fn normalize(input: &str) -> String { driver::normalize(&CLASSIFY, &VERBALIZE, Some(&POST), input, " ") } + +/// Normalize English text and retain the source span for every output token. +pub fn normalize_aligned(input: &str) -> Option { + driver::normalize_aligned(&CLASSIFY, &VERBALIZE, Some(&POST), input, " ") +} diff --git a/src/fst/engine.rs b/src/fst/engine.rs index e5bb028..bf11027 100644 --- a/src/fst/engine.rs +++ b/src/fst/engine.rs @@ -17,6 +17,14 @@ use rustfst::algorithms::compose::compose; use rustfst::algorithms::rm_epsilon::rm_epsilon; use rustfst::prelude::*; +/// The selected transducer path, including the input position reached at each +/// output-byte boundary. The boundary table lets the higher-level token parser +/// recover which source bytes produced each top-level classifier token. +pub(super) struct AppliedPath { + pub output: String, + pub input_at_output_boundary: Vec, +} + /// Build a linear FST that accepts exactly the bytes of `s` (input == output). /// /// Working at the byte level keeps the engine encoding-agnostic: Chinese, @@ -39,14 +47,14 @@ fn byte_acceptor(s: &str) -> VectorFst { fst } -/// Return the output-label string of the lowest-weight path through an acyclic -/// FST, or `None` if it has no accepting path. +/// Return the input/output label pairs on the lowest-weight path through an +/// acyclic FST, or `None` if it has no accepting path. /// /// Iterative-DFS topological order, then a single relaxation pass /// (`dist[next] = min(dist[next], dist[s] + w)`) followed by a backtrack over /// the recorded predecessors. Matches OpenFST's tropical shortest-path on the /// acyclic FSTs this engine produces. -fn shortest_output(fst: &VectorFst) -> Option { +fn shortest_labels(fst: &VectorFst) -> Option> { let start = fst.start()?; let n = fst.num_states(); @@ -76,7 +84,7 @@ fn shortest_output(fst: &VectorFst) -> Option { // Relax edges in topological order. let inf = f32::INFINITY; let mut dist = vec![inf; n]; - let mut pred: Vec> = vec![None; n]; + let mut pred: Vec> = vec![None; n]; dist[start as usize] = 0.0; for &s in &order { if dist[s] == inf { @@ -87,7 +95,7 @@ fn shortest_output(fst: &VectorFst) -> Option { let ns = tr.nextstate as usize; if w < dist[ns] { dist[ns] = w; - pred[ns] = Some((s, tr.olabel)); + pred[ns] = Some((s, tr.ilabel, tr.olabel)); } } } @@ -108,30 +116,79 @@ fn shortest_output(fst: &VectorFst) -> Option { // Backtrack, collecting non-epsilon output labels. let mut f = best_final?; let mut labels = Vec::new(); - while let Some((p, olabel)) = pred[f] { - if olabel != 0 { - labels.push(olabel as u8); - } + while let Some((p, ilabel, olabel)) = pred[f] { + labels.push((ilabel, olabel)); f = p; } labels.reverse(); - Some(String::from_utf8_lossy(&labels).to_string()) + Some(labels) +} + +fn applied_path(labels: Vec<(u32, u32)>) -> Option { + let output_len = labels.iter().filter(|(_, output)| *output != 0).count(); + let mut output = Vec::with_capacity(output_len); + let mut input_at_output_boundary = vec![0usize; output_len + 1]; + let mut input_position = 0usize; + let mut output_position = 0usize; + + for (input, emitted) in labels { + if input != 0 { + input_position += 1; + } + if emitted != 0 { + output.push(emitted as u8); + output_position += 1; + } + // Input-only arcs between emitted bytes belong to the boundary that + // has most recently been reached. Updating it is important for spaces + // and deleted characters between adjacent classifier tokens. + input_at_output_boundary[output_position] = input_position; + } + + Some(AppliedPath { + output: String::from_utf8(output).ok()?, + input_at_output_boundary, + }) } /// Apply a transducer to `input`: compose, remove epsilons, take the tropical /// shortest path's output. Returns `None` if the input is not in the domain /// (empty composition) or the shortest path emits nothing. pub fn apply(fst: &VectorFst, input: &str) -> Option { + let labels = apply_labels(fst, input)?; + let output = labels + .into_iter() + .filter_map(|(_, output)| (output != 0).then_some(output as u8)) + .collect::>(); + if output.is_empty() { + None + } else { + String::from_utf8(output).ok() + } +} + +/// Apply a transducer while retaining input progress along the selected path. +/// +/// This is used only for classifier alignment. Verbalization callers that need +/// just the output should continue to use [`apply`]. +pub(super) fn apply_with_alignment( + fst: &VectorFst, + input: &str, +) -> Option { + let path = applied_path(apply_labels(fst, input)?)?; + if path.output.is_empty() { + None + } else { + Some(path) + } +} + +fn apply_labels(fst: &VectorFst, input: &str) -> Option> { let mut composed: VectorFst = compose(byte_acceptor(input), fst.clone()).ok()?; if composed.num_states() == 0 || composed.start().is_none() { return None; } rm_epsilon(&mut composed).ok()?; - let out = shortest_output(&composed)?; - if out.is_empty() { - None - } else { - Some(out) - } + shortest_labels(&composed) } diff --git a/src/fst/es.rs b/src/fst/es.rs index 3240af4..84a6191 100644 --- a/src/fst/es.rs +++ b/src/fst/es.rs @@ -6,7 +6,7 @@ //! Grammars in `grammars/es/` are exported from NeMo-text-processing //! (Apache-2.0, pinned commit `1f1263579fe57ba7ed783cad3dddee710fcc5064`). -use super::{driver, load_gz}; +use super::{driver, load_gz, AlignedNormalization}; use lazy_static::lazy_static; use rustfst::prelude::*; @@ -30,3 +30,8 @@ lazy_static! { pub fn normalize(input: &str) -> String { driver::normalize(&CLASSIFY, &VERBALIZE, None, input, " ") } + +/// Normalize Spanish text and retain the source span for every output token. +pub fn normalize_aligned(input: &str) -> Option { + driver::normalize_aligned(&CLASSIFY, &VERBALIZE, None, input, " ") +} diff --git a/src/fst/fr.rs b/src/fst/fr.rs index f787e22..8a5e908 100644 --- a/src/fst/fr.rs +++ b/src/fst/fr.rs @@ -6,7 +6,7 @@ //! Grammars in `grammars/fr/` are exported from NeMo-text-processing //! (Apache-2.0, pinned commit `1f1263579fe57ba7ed783cad3dddee710fcc5064`). -use super::{driver, load_gz}; +use super::{driver, load_gz, AlignedNormalization}; use lazy_static::lazy_static; use rustfst::prelude::*; @@ -30,3 +30,8 @@ lazy_static! { pub fn normalize(input: &str) -> String { driver::normalize(&CLASSIFY, &VERBALIZE, None, input, " ") } + +/// Normalize French text and retain the source span for every output token. +pub fn normalize_aligned(input: &str) -> Option { + driver::normalize_aligned(&CLASSIFY, &VERBALIZE, None, input, " ") +} diff --git a/src/fst/hi.rs b/src/fst/hi.rs index 9812471..3595de3 100644 --- a/src/fst/hi.rs +++ b/src/fst/hi.rs @@ -6,7 +6,7 @@ //! Grammars in `grammars/hi/` are exported from NeMo-text-processing //! (Apache-2.0, pinned commit `1f1263579fe57ba7ed783cad3dddee710fcc5064`). -use super::{driver, load_gz}; +use super::{driver, load_gz, AlignedNormalization}; use lazy_static::lazy_static; use rustfst::prelude::*; @@ -31,3 +31,8 @@ lazy_static! { pub fn normalize(input: &str) -> String { driver::normalize(&CLASSIFY, &VERBALIZE, Some(&POST), input, " ") } + +/// Normalize Hindi text and retain the source span for every output token. +pub fn normalize_aligned(input: &str) -> Option { + driver::normalize_aligned(&CLASSIFY, &VERBALIZE, Some(&POST), input, " ") +} diff --git a/src/fst/ja.rs b/src/fst/ja.rs index 7f6bef0..89f066d 100644 --- a/src/fst/ja.rs +++ b/src/fst/ja.rs @@ -6,7 +6,7 @@ //! Grammars in `grammars/ja/` are exported from NeMo-text-processing //! (Apache-2.0, pinned commit `1f1263579fe57ba7ed783cad3dddee710fcc5064`). -use super::{driver, load_gz}; +use super::{driver, load_gz, AlignedNormalization}; use lazy_static::lazy_static; use rustfst::prelude::*; @@ -30,3 +30,8 @@ lazy_static! { pub fn normalize(input: &str) -> String { driver::normalize(&CLASSIFY, &VERBALIZE, None, input, "") } + +/// Normalize Japanese text and retain the source span for every output token. +pub fn normalize_aligned(input: &str) -> Option { + driver::normalize_aligned(&CLASSIFY, &VERBALIZE, None, input, "") +} diff --git a/src/fst/mod.rs b/src/fst/mod.rs index 3a4881f..b30d913 100644 --- a/src/fst/mod.rs +++ b/src/fst/mod.rs @@ -35,6 +35,97 @@ use flate2::read::GzDecoder; use rustfst::prelude::*; use std::io::Read; +/// A semantic class assigned by the NeMo TN classifier. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum TokenKind { + Word, + Punctuation, + Cardinal, + Ordinal, + Decimal, + Fraction, + Time, + Measure, + Percent, + Date, + Telephone, + Money, + Electronic, + Verbatim, + Letters, + Abbreviation, + Other(String), +} + +impl TokenKind { + /// Stable lower-case name suitable for serialization and foreign-language + /// bindings. + pub fn as_str(&self) -> &str { + match self { + Self::Word => "word", + Self::Punctuation => "punctuation", + Self::Cardinal => "cardinal", + Self::Ordinal => "ordinal", + Self::Decimal => "decimal", + Self::Fraction => "fraction", + Self::Time => "time", + Self::Measure => "measure", + Self::Percent => "percent", + Self::Date => "date", + Self::Telephone => "telephone", + Self::Money => "money", + Self::Electronic => "electronic", + Self::Verbatim => "verbatim", + Self::Letters => "letters", + Self::Abbreviation => "abbreviation", + Self::Other(name) => name, + } + } +} + +/// One source span and the words produced for it by text normalization. +/// +/// Offsets are half-open UTF-8 byte offsets into the original input. Whitespace +/// between spans is deliberately excluded, so callers can recover it from the +/// gaps between adjacent ranges without losing the original formatting. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AlignedSpan { + pub input_start: usize, + pub input_end: usize, + pub original: String, + /// Direct verbalizer output for this classifier token. Sentence-level + /// punctuation and spacing cleanup is reflected in + /// [`AlignedNormalization::normalized`]. + pub normalized: String, + pub kind: TokenKind, +} + +/// Sentence-level TN output together with its source-to-normalized mapping. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AlignedNormalization { + /// The same sentence-level result returned by the corresponding + /// language's [`normalize`](crate::fst::en::normalize) function. + pub normalized: String, + pub spans: Vec, +} + +/// Normalize with NeMo's compiled FST grammar and preserve source spans. +/// +/// Supported language codes are `en`, `fr`, `es`, `de`, `zh`, `hi`, and `ja`. +/// Returns `None` for unsupported languages or when classification fails. +pub fn normalize_aligned(input: &str, lang: &str) -> Option { + match lang { + "en" => en::normalize_aligned(input), + "fr" => fr::normalize_aligned(input), + "es" => es::normalize_aligned(input), + "de" => de::normalize_aligned(input), + "zh" => zh::normalize_aligned(input), + "hi" => hi::normalize_aligned(input), + "ja" => ja::normalize_aligned(input), + _ => None, + } +} + /// Decompress a bundled `*.fst.gz` grammar and load it as an FST. fn load_gz(gz: &[u8]) -> VectorFst { let mut bytes = Vec::new(); diff --git a/src/fst/zh.rs b/src/fst/zh.rs index 63c5b09..ba0804a 100644 --- a/src/fst/zh.rs +++ b/src/fst/zh.rs @@ -6,7 +6,7 @@ //! Grammars in `grammars/zh/` are exported from NeMo-text-processing //! (Apache-2.0, pinned commit `1f1263579fe57ba7ed783cad3dddee710fcc5064`). -use super::{driver, load_gz}; +use super::{driver, load_gz, AlignedNormalization}; use lazy_static::lazy_static; use rustfst::prelude::*; @@ -31,3 +31,8 @@ lazy_static! { pub fn normalize(input: &str) -> String { driver::normalize(&CLASSIFY, &VERBALIZE, None, input, "") } + +/// Normalize Mandarin text and retain the source span for every output token. +pub fn normalize_aligned(input: &str) -> Option { + driver::normalize_aligned(&CLASSIFY, &VERBALIZE, None, input, "") +} diff --git a/src/wasm.rs b/src/wasm.rs index 7763abb..c397a0d 100644 --- a/src/wasm.rs +++ b/src/wasm.rs @@ -2,6 +2,9 @@ use wasm_bindgen::prelude::*; +#[cfg(feature = "fst-engine")] +use js_sys::{Array, Object, Reflect}; + use crate::{ custom_rules, normalize, normalize_sentence, normalize_sentence_lang, normalize_sentence_with_options, normalize_with_lang, normalize_with_options, tn_normalize, @@ -128,6 +131,53 @@ pub fn tn_normalize_sentence_with_max_span_lang_js( tn_normalize_sentence_with_max_span_lang(input, lang, max_span_tokens as usize) } +/// Compiled-FST TN with source-span alignment. +/// +/// Returns `null` unless the build enables both `wasm` and `fst-engine`, or +/// when `lang` is unsupported. Offsets are half-open UTF-8 byte offsets. +#[wasm_bindgen(js_name = tnFstNormalizeAligned)] +pub fn tn_fst_normalize_aligned_js(input: &str, lang: &str) -> JsValue { + #[cfg(not(feature = "fst-engine"))] + { + let _ = (input, lang); + JsValue::NULL + } + + #[cfg(feature = "fst-engine")] + { + let Some(alignment) = crate::fst::normalize_aligned(input, lang) else { + return JsValue::NULL; + }; + let result = Object::new(); + let spans = Array::new(); + for span in alignment.spans { + let item = Object::new(); + set_js_property( + &item, + "inputStart", + &JsValue::from_f64(span.input_start as f64), + ); + set_js_property(&item, "inputEnd", &JsValue::from_f64(span.input_end as f64)); + set_js_property(&item, "original", &JsValue::from_str(&span.original)); + set_js_property(&item, "normalized", &JsValue::from_str(&span.normalized)); + set_js_property(&item, "kind", &JsValue::from_str(span.kind.as_str())); + spans.push(&item); + } + set_js_property( + &result, + "normalized", + &JsValue::from_str(&alignment.normalized), + ); + set_js_property(&result, "spans", &spans); + result.into() + } +} + +#[cfg(feature = "fst-engine")] +fn set_js_property(object: &Object, name: &str, value: &JsValue) { + let _ = Reflect::set(object, &JsValue::from_str(name), value); +} + #[wasm_bindgen(js_name = addRule)] pub fn add_rule_js(spoken: &str, written: &str) { custom_rules::add_rule(spoken, written); diff --git a/swift-test/Package.swift b/swift-test/Package.swift index fb66d44..aa7d9db 100644 --- a/swift-test/Package.swift +++ b/swift-test/Package.swift @@ -2,6 +2,20 @@ import PackageDescription +#if arch(arm64) +let rustTarget = "aarch64-apple-darwin" +#elseif arch(x86_64) +let rustTarget = "x86_64-apple-darwin" +#else +#error("Unsupported macOS architecture") +#endif + +let rustLinkerSettings: [LinkerSetting] = [ + .unsafeFlags([ + "../target/\(rustTarget)/release/libtext_processing_rs.a" + ]) +] + let package = Package( name: "NemoTest", platforms: [.macOS(.v14)], @@ -13,32 +27,22 @@ let package = Package( .executableTarget( name: "NemoTest", dependencies: ["CNemoTextProcessing"], - linkerSettings: [ - .unsafeFlags([ - "-L../target/aarch64-apple-darwin/release", - "-ltext_processing_rs" - ]) - ] + linkerSettings: rustLinkerSettings ), .executableTarget( name: "nemo-itn", dependencies: ["CNemoTextProcessing"], - linkerSettings: [ - .unsafeFlags([ - "-L../target/aarch64-apple-darwin/release", - "-ltext_processing_rs" - ]) - ] + linkerSettings: rustLinkerSettings ), .executableTarget( name: "nemo-tn", dependencies: ["CNemoTextProcessing"], - linkerSettings: [ - .unsafeFlags([ - "-L../target/aarch64-apple-darwin/release", - "-ltext_processing_rs" - ]) - ] + linkerSettings: rustLinkerSettings + ), + .executableTarget( + name: "nemo-tn-aligned", + dependencies: ["CNemoTextProcessing"], + linkerSettings: rustLinkerSettings ), ] ) diff --git a/swift-test/Sources/CNemoTextProcessing/include/nemo_text_processing.h b/swift-test/Sources/CNemoTextProcessing/include/nemo_text_processing.h index 3318749..99591c0 100644 --- a/swift-test/Sources/CNemoTextProcessing/include/nemo_text_processing.h +++ b/swift-test/Sources/CNemoTextProcessing/include/nemo_text_processing.h @@ -1,6 +1,7 @@ #ifndef NEMO_TEXT_PROCESSING_H #define NEMO_TEXT_PROCESSING_H +#include #include #ifdef __cplusplus @@ -35,6 +36,23 @@ char* nemo_tn_normalize_sentence_with_max_span(const char* input, uint32_t max_s /* Byte-exact NeMo TN via the compiled-FST engine (NULL if unavailable) */ char* nemo_tn_fst(const char* input, const char* lang); +typedef struct NemoTnAlignedSpan { + size_t input_start; + size_t input_end; + char* original; + char* normalized; + char* kind; +} NemoTnAlignedSpan; + +typedef struct NemoTnAlignment { + char* normalized; + NemoTnAlignedSpan* spans; + size_t span_count; +} NemoTnAlignment; + +NemoTnAlignment* nemo_tn_fst_aligned(const char* input, const char* lang); +void nemo_tn_alignment_free(NemoTnAlignment* alignment); + #ifdef __cplusplus } #endif diff --git a/swift-test/Sources/nemo-tn-aligned/main.swift b/swift-test/Sources/nemo-tn-aligned/main.swift new file mode 100644 index 0000000..7a78009 --- /dev/null +++ b/swift-test/Sources/nemo-tn-aligned/main.swift @@ -0,0 +1,180 @@ +import Foundation +import CNemoTextProcessing + +// MARK: - Wrapper + +struct AlignedSpan: Encodable { + let inputStart: Int + let inputEnd: Int + let original: String + let normalized: String + let kind: String + + enum CodingKeys: String, CodingKey { + case inputStart = "input_start" + case inputEnd = "input_end" + case original + case normalized + case kind + } +} + +struct Alignment: Encodable { + let input: String + let language: String + let normalized: String + let spans: [AlignedSpan] +} + +enum Nemo { + static func tnNormalizeAligned(_ input: String, language: String) -> Alignment? { + guard let resultPtr = nemo_tn_fst_aligned(input, language) else { + return nil + } + defer { nemo_tn_alignment_free(resultPtr) } + + let result = resultPtr.pointee + guard let normalized = result.normalized else { + return nil + } + + let count = Int(result.span_count) + if count > 0 && result.spans == nil { + return nil + } + + let nativeSpans = UnsafeBufferPointer(start: result.spans, count: count) + var spans: [AlignedSpan] = [] + spans.reserveCapacity(count) + for span in nativeSpans { + guard let original = span.original, + let normalized = span.normalized, + let kind = span.kind else { + return nil + } + spans.append(AlignedSpan( + inputStart: Int(span.input_start), + inputEnd: Int(span.input_end), + original: String(cString: original), + normalized: String(cString: normalized), + kind: String(cString: kind) + )) + } + + return Alignment( + input: input, + language: language, + normalized: String(cString: normalized), + spans: spans + ) + } + + static var version: String { + guard let pointer = nemo_version() else { return "unknown" } + return String(cString: pointer) + } +} + +// MARK: - CLI + +let usage = """ + nemo-tn-aligned - FST Text Normalization with source alignment + + USAGE: + nemo-tn-aligned [-l ] + echo "text" | nemo-tn-aligned [-l ] + nemo-tn-aligned --version + nemo-tn-aligned --help + + OPTIONS: + -l, --lang en, fr, es, de, zh, hi, or ja (default: en) + + OUTPUT: + One compact JSON object per input. Stdin mode emits JSON Lines. + Source offsets are half-open UTF-8 byte offsets. + + EXAMPLE: + nemo-tn-aligned --lang en 'The price is $1,234.56.' + """ + +let args = Array(CommandLine.arguments.dropFirst()) + +if args.contains("--help") || args.contains("-h") { + print(usage) + exit(0) +} + +if args.contains("--version") || args.contains("-V") { + print("nemo-tn-aligned \(Nemo.version)") + exit(0) +} + +var language = "en" +var inputArgs: [String] = [] +var index = 0 +var optionsEnded = false +while index < args.count { + let argument = args[index] + if !optionsEnded && argument == "--" { + optionsEnded = true + } else if !optionsEnded && (argument == "-l" || argument == "--lang") { + index += 1 + guard index < args.count else { + fputs("nemo-tn-aligned: --lang requires a language code\n", stderr) + exit(1) + } + language = args[index] + } else if !optionsEnded && argument.hasPrefix("--lang=") { + language = String(argument.dropFirst("--lang=".count)) + } else { + inputArgs.append(argument) + } + index += 1 +} + +let supportedLanguages = ["en", "fr", "es", "de", "zh", "hi", "ja"] +guard supportedLanguages.contains(language) else { + fputs( + "nemo-tn-aligned: unsupported language '\(language)'; expected one of \(supportedLanguages.joined(separator: ", "))\n", + stderr + ) + exit(1) +} + +if inputArgs.isEmpty && isatty(fileno(stdin)) != 0 { + fputs(usage, stderr) + exit(1) +} + +let encoder = JSONEncoder() +encoder.outputFormatting = [.sortedKeys, .withoutEscapingSlashes] + +func emit(_ input: String) -> Bool { + guard let alignment = Nemo.tnNormalizeAligned(input, language: language) else { + fputs("nemo-tn-aligned: could not classify input as \(language) text\n", stderr) + return false + } + + do { + let data = try encoder.encode(alignment) + FileHandle.standardOutput.write(data) + FileHandle.standardOutput.write(Data([0x0A])) + return true + } catch { + fputs("nemo-tn-aligned: failed to encode JSON: \(error)\n", stderr) + return false + } +} + +if !inputArgs.isEmpty { + exit(emit(inputArgs.joined(separator: " ")) ? 0 : 1) +} + +var failed = false +while let line = readLine() { + guard !line.trimmingCharacters(in: .whitespaces).isEmpty else { continue } + if !emit(line) { + failed = true + } +} +exit(failed ? 1 : 0) diff --git a/swift/NemoTextProcessing.swift b/swift/NemoTextProcessing.swift index 1768010..2013640 100644 --- a/swift/NemoTextProcessing.swift +++ b/swift/NemoTextProcessing.swift @@ -1,5 +1,23 @@ import Foundation +/// One source span and the spoken-form words produced for it by TN. +public struct TNAlignedSpan: Sendable, Equatable { + /// Half-open UTF-8 byte range in the original input. + public let inputRange: Range + public let original: String + /// Direct verbalizer output for this classifier token. Use the containing + /// `TNAlignment.normalized` for sentence-level punctuation and spacing. + public let normalized: String + /// Stable NeMo class such as `word`, `money`, `date`, or `telephone`. + public let kind: String +} + +/// Sentence-level normalized text and its source-to-output span mapping. +public struct TNAlignment: Sendable, Equatable { + public let normalized: String + public let spans: [TNAlignedSpan] +} + /// Swift wrapper for NeMo Text Processing (Inverse Text Normalization). /// /// Converts spoken-form ASR output to written form: @@ -271,6 +289,59 @@ public enum NemoTextProcessing { return String(cString: resultPtr) } + /// Normalize through the compiled NeMo FST and retain the source span for + /// every normalized token. + /// + /// This API is available when the native library is built with the + /// `fst-engine` feature. Offsets are UTF-8 byte offsets, so slice + /// `input.utf8` rather than using them as `String.Index` values directly. + /// + /// - Parameters: + /// - input: Complete written-form sentence. + /// - language: One of `en`, `fr`, `es`, `de`, `zh`, `hi`, or `ja`. + /// - Returns: Normalized text and aligned spans, or `nil` when FST TN is + /// unavailable or the language is unsupported. + public static func tnNormalizeAligned(_ input: String, language: String) -> TNAlignment? { + guard let inputC = input.cString(using: .utf8), + let langC = language.cString(using: .utf8), + let resultPtr = nemo_tn_fst_aligned(inputC, langC) else { + return nil + } + defer { nemo_tn_alignment_free(resultPtr) } + + let result = resultPtr.pointee + guard let normalizedPtr = result.normalized else { + return nil + } + + let count = Int(result.span_count) + if count > 0 && result.spans == nil { + return nil + } + + let nativeSpans = UnsafeBufferPointer(start: result.spans, count: count) + var spans: [TNAlignedSpan] = [] + spans.reserveCapacity(count) + for native in nativeSpans { + guard let original = native.original, + let normalized = native.normalized, + let kind = native.kind else { + return nil + } + spans.append(TNAlignedSpan( + inputRange: Int(native.input_start).. #include #ifdef __cplusplus @@ -187,6 +188,39 @@ char* nemo_tn_normalize_sentence_with_max_span_lang(const char* input, const cha */ char* nemo_tn_fst(const char* input, const char* lang); +/** One aligned source span produced by compiled-FST text normalization. */ +typedef struct NemoTnAlignedSpan { + /** Half-open UTF-8 byte range in the original input. */ + size_t input_start; + size_t input_end; + /** Original source substring. Owned by the containing alignment. */ + char* original; + /** Direct verbalizer output for this classifier token. */ + char* normalized; + /** Stable semantic class, for example "word", "money", or "date". */ + char* kind; +} NemoTnAlignedSpan; + +/** Sentence-level normalized output and source-to-output span mapping. */ +typedef struct NemoTnAlignment { + char* normalized; + NemoTnAlignedSpan* spans; + size_t span_count; +} NemoTnAlignment; + +/** + * Normalize via the compiled-FST engine and retain source spans. + * + * Supported langs: "en", "zh", "ja", "fr", "es", "de", "hi". + * Returns NULL when fst-engine is unavailable, the language is unsupported, + * or the input is invalid. The result must be released exactly once with + * nemo_tn_alignment_free(). + */ +NemoTnAlignment* nemo_tn_fst_aligned(const char* input, const char* lang); + +/** Free a result returned by nemo_tn_fst_aligned(). NULL is allowed. */ +void nemo_tn_alignment_free(NemoTnAlignment* alignment); + /** * Free a string allocated by nemo_normalize or nemo_normalize_sentence. * diff --git a/tests/fst_alignment.rs b/tests/fst_alignment.rs new file mode 100644 index 0000000..a435d3f --- /dev/null +++ b/tests/fst_alignment.rs @@ -0,0 +1,220 @@ +//! Source-alignment coverage for the compiled-FST text normalizer. +//! +//! Requires the `fst-engine` feature: +//! `cargo test --features ffi,fst-engine --test fst_alignment`. +#![cfg(feature = "fst-engine")] + +use text_processing_rs::fst::{self, AlignedNormalization, AlignedSpan, TokenKind}; + +fn assert_source_ranges(input: &str, alignment: &AlignedNormalization) { + let mut previous_end = 0; + + for span in &alignment.spans { + assert!(span.input_start >= previous_end, "source spans overlap"); + assert!( + span.input_start <= span.input_end, + "source span is reversed" + ); + assert!(input.is_char_boundary(span.input_start)); + assert!(input.is_char_boundary(span.input_end)); + assert_eq!(&input[span.input_start..span.input_end], span.original); + previous_end = span.input_end; + } +} + +#[test] +fn aligns_a_money_span_within_a_complete_sentence() { + let input = "The price is $1,234.56."; + let alignment = fst::normalize_aligned(input, "en").expect("English input should normalize"); + + assert_eq!( + alignment.normalized, + "The price is one thousand two hundred and thirty four dollars fifty six cents." + ); + assert_eq!( + alignment.spans, + vec![ + AlignedSpan { + input_start: 0, + input_end: 3, + original: "The".into(), + normalized: "The".into(), + kind: TokenKind::Word, + }, + AlignedSpan { + input_start: 4, + input_end: 9, + original: "price".into(), + normalized: "price".into(), + kind: TokenKind::Word, + }, + AlignedSpan { + input_start: 10, + input_end: 12, + original: "is".into(), + normalized: "is".into(), + kind: TokenKind::Word, + }, + AlignedSpan { + input_start: 13, + input_end: 22, + original: "$1,234.56".into(), + normalized: "one thousand two hundred and thirty four dollars fifty six cents" + .into(), + kind: TokenKind::Money, + }, + AlignedSpan { + input_start: 22, + input_end: 23, + original: ".".into(), + normalized: ".".into(), + kind: TokenKind::Punctuation, + }, + ] + ); + assert_source_ranges(input, &alignment); + assert_eq!(alignment.normalized, fst::en::normalize(input)); +} + +#[test] +fn reports_utf8_byte_offsets() { + let input = "Café costs €12.50."; + let alignment = fst::normalize_aligned(input, "en").expect("English input should normalize"); + + assert_eq!(alignment.spans[0].original, "Café"); + assert_eq!( + (alignment.spans[0].input_start, alignment.spans[0].input_end), + (0, 5) + ); + assert_eq!(alignment.spans[2].original, "€12.50"); + assert_eq!( + (alignment.spans[2].input_start, alignment.spans[2].input_end), + (12, 20) + ); + assert_eq!(alignment.spans[2].kind, TokenKind::Money); + assert_source_ranges(input, &alignment); +} + +#[test] +fn distinguishes_electronic_date_and_punctuation_spans() { + let input = "Email jane.doe@example.com on 12/31/2025."; + let alignment = fst::normalize_aligned(input, "en").expect("English input should normalize"); + + let mapped = alignment + .spans + .iter() + .map(|span| { + ( + span.original.as_str(), + span.normalized.as_str(), + span.kind.as_str(), + ) + }) + .collect::>(); + assert_eq!( + mapped, + vec![ + ("Email", "Email", "word"), + ( + "jane.doe@example.com", + "jane dot doe at example dot com", + "electronic", + ), + ("on", "on", "word"), + ( + "12/31/2025", + "december thirty first twenty twenty five", + "date", + ), + (".", ".", "punctuation"), + ] + ); + assert_source_ranges(input, &alignment); +} + +#[test] +fn classifies_a_phone_number_inside_a_sentence() { + let input = "Call 415-555-0123."; + let alignment = fst::normalize_aligned(input, "en").expect("English input should normalize"); + + assert_eq!( + alignment.normalized, + "Call four one five, five five five, zero one two three." + ); + assert_eq!(alignment.spans[1].original, "415-555-0123"); + assert_eq!( + alignment.spans[1].normalized, + "four one five, five five five, zero one two three" + ); + assert_eq!(alignment.spans[1].kind, TokenKind::Telephone); + assert_eq!( + (alignment.spans[1].input_start, alignment.spans[1].input_end), + (5, 17) + ); + assert_source_ranges(input, &alignment); +} + +#[test] +fn aligns_spanish_currency_with_non_ascii_offsets() { + let input = "El precio es 1.234,56 €."; + let alignment = fst::normalize_aligned(input, "es").expect("Spanish input should normalize"); + + assert_eq!( + alignment.normalized, + "El precio es mil doscientos treinta y cuatro coma cincuenta y seis euros." + ); + assert_eq!(alignment.spans[3].original, "1.234,56 €"); + assert_eq!( + alignment.spans[3].normalized, + "mil doscientos treinta y cuatro coma cincuenta y seis euros" + ); + assert_eq!(alignment.spans[3].kind, TokenKind::Money); + assert_eq!( + (alignment.spans[3].input_start, alignment.spans[3].input_end), + (13, 25) + ); + assert_source_ranges(input, &alignment); + assert_eq!(alignment.normalized, fst::es::normalize(input)); +} + +#[cfg(feature = "ffi")] +#[test] +fn exposes_alignment_through_the_c_ffi() { + use std::ffi::{CStr, CString}; + + use text_processing_rs::ffi::{nemo_tn_alignment_free, nemo_tn_fst_aligned}; + + unsafe { + let input = CString::new("The price is $1,234.56.").unwrap(); + let en = CString::new("en").unwrap(); + let result = nemo_tn_fst_aligned(input.as_ptr(), en.as_ptr()); + + assert!(!result.is_null()); + let alignment = &*result; + assert_eq!(alignment.span_count, 5); + assert_eq!( + CStr::from_ptr(alignment.normalized).to_str().unwrap(), + "The price is one thousand two hundred and thirty four dollars fifty six cents." + ); + + let spans = std::slice::from_raw_parts(alignment.spans, alignment.span_count); + let money = &spans[3]; + assert_eq!((money.input_start, money.input_end), (13, 22)); + assert_eq!( + CStr::from_ptr(money.original).to_str().unwrap(), + "$1,234.56" + ); + assert_eq!(CStr::from_ptr(money.kind).to_str().unwrap(), "money"); + + nemo_tn_alignment_free(result); + nemo_tn_alignment_free(std::ptr::null_mut()); + + let unsupported = CString::new("xx").unwrap(); + assert!(nemo_tn_fst_aligned(input.as_ptr(), unsupported.as_ptr()).is_null()); + } +} + +#[test] +fn rejects_an_unsupported_language() { + assert!(fst::normalize_aligned("$12.50", "xx").is_none()); +}