feat: add source-aligned text normalization APIs and CLI

AGENTS.md

@@ -7,6 +7,7 @@ cargo build                      # Build project
 cargo build --release            # Release build
 cargo build --features ffi       # Build with C FFI bindings
 cargo build --features wasm --target wasm32-unknown-unknown  # Build for WebAssembly
+cargo build --features fst-engine # Build the optional compiled NeMo FST TN engine
 cargo test                       # Run all tests
 cargo test --test en_tests       # Run a single integration test file
 cargo test en_tn_tests::test_name  # Run a single test by name
@@ -18,6 +19,7 @@ cargo clippy --all-targets       # Lint
 
 - **src/itn/**: ITN taggers (spoken → written, Inverse Text Normalization) per language (en, de, es, fr, hi, ja, zh)
 - **src/tn/**: TN taggers (written → spoken, Text Normalization)
+- **src/fst/**: Optional compiled NeMo FST TN engine and source-aligned TN spans, gated by `fst-engine`
 - **src/custom_rules.rs**: User-defined custom normalization rules (highest priority)
 - **src/ffi.rs**: C FFI bindings for Swift/Python integration (gated by `ffi` feature)
 - **src/wasm.rs**: JavaScript-callable wasm bindings (gated by `wasm` feature)
@@ -43,8 +45,8 @@ Taggers are tried in order of specificity (most specific first). If no tagger ma
 - **NEVER** create simplified or stub versions — implement full solutions or consult first
 - **NEVER** introduce mock data or fabricated test cases — use realistic spoken/written forms
 - Add unit/integration tests when adding new taggers or rules
-- Keep all changes local to the appropriate language module under `src/itn/<lang>/` or `src/tn/<lang>/`
-- Maintain feature-flag isolation: code behind `ffi` and `wasm` must not leak into the default build
+- Keep language-specific rule changes local to `src/itn/<lang>/` or `src/tn/<lang>/`; keep compiled-FST behavior in `src/fst/` and its bindings in the corresponding consumer surfaces
+- Maintain feature-flag isolation: `ffi`, `wasm`, and `fst-engine` code and dependencies must not leak into the default build
 
 ## Code Style (rustfmt + clippy)
 
@@ -67,6 +69,7 @@ Taggers are tried in order of specificity (most specific first). If no tagger ma
 
 - C FFI lives in `src/ffi.rs` behind the `ffi` feature; consumed by the Swift package in `swift/`
 - Wasm bindings live in `src/wasm.rs` behind the `wasm` feature; consumed by `wasm-tests/`
+- Compiled-FST alignment uses `fst::normalize_aligned` and half-open UTF-8 byte offsets; its Rust API is gated by `fst-engine`, and its C/WASM exports must preserve the same contract
 - Run `cargo build --features ffi` and the Swift tests under `swift-test/` after touching FFI signatures
 - Run the wasm test suite after touching `wasm.rs` or anything affecting the wasm-exposed surface
 

Cargo.toml

@@ -14,6 +14,7 @@ crate-type = ["lib", "staticlib", "cdylib"]
 [dependencies]
 lazy_static = "1"
 wasm-bindgen = { version = "0.2", optional = true }
+js-sys = { version = "0.3", optional = true }
 console_error_panic_hook = { version = "0.1", optional = true }
 # Optional: byte-exact NeMo parity via compiled weighted FST grammars.
 # Off by default — the pure-Rust rule path carries no rustfst/OpenFST weight.
@@ -26,7 +27,7 @@ proptest = "1"
 [features]
 default = []
 ffi = []  # Enable C FFI bindings
-wasm = ["dep:wasm-bindgen", "dep:console_error_panic_hook"]
+wasm = ["dep:wasm-bindgen", "dep:js-sys", "dep:console_error_panic_hook"]
 # Byte-exact NeMo parity engine (runs NeMo's compiled grammars via rustfst).
 # Adds rustfst + flate2 and ~7 MB of gzipped grammars (all languages combined).
 fst-engine = ["dep:rustfst", "dep:flate2"]

README.md

@@ -126,6 +126,29 @@ let result = tn_normalize_sentence("I paid $5 for 23 items");
 assert_eq!(result, "I paid five dollars for twenty three items");
 ```
 
+Compiled-FST TN can also retain the source range and semantic class for every
+normalized span. Offsets are half-open UTF-8 byte offsets:
+
+```rust
+use text_processing_rs::fst;
+
+let result = fst::normalize_aligned("The price is $1,234.56.", "en").unwrap();
+assert_eq!(
+    result.normalized,
+    "The price is one thousand two hundred and thirty four dollars fifty six cents."
+);
+
+let money = &result.spans[3];
+assert_eq!(money.input_start, 13);
+assert_eq!(money.input_end, 22);
+assert_eq!(money.original, "$1,234.56");
+assert_eq!(money.kind.as_str(), "money");
+```
+
+Build this API with `--features fst-engine`. It uses the same compiled NeMo
+classifier and verbalizer as `fst::<lang>::normalize`, rather than recovering
+alignment by diffing the final strings.
+
 ### Swift
 
 ```swift
@@ -149,6 +172,16 @@ let itnFr = NemoTextProcessing.normalizeSentence("j'ai vingt et un ans", languag
 
 let tn = NemoTextProcessing.tnNormalizeSentence("I paid $5 for 23 items")
 // "I paid five dollars for twenty three items"
+
+if let aligned = NemoTextProcessing.tnNormalizeAligned(
+    "The price is $1,234.56.",
+    language: "en"
+) {
+    let money = aligned.spans[3]
+    // money.original == "$1,234.56"
+    // money.normalized == "one thousand ... dollars fifty six cents"
+    // money.inputRange == 13..<22, money.kind == "money"
+}
 ```
 
 ### CLI
@@ -215,6 +248,7 @@ echo "2:30 PM" | nemo-tn               # → two thirty p m
 - Phone numbers, IP addresses, SSN
 - Case preservation for proper nouns and abbreviations
 - Sentence-level normalization with sliding window span matching
+- Source-to-normalized span alignment with semantic classes (compiled FST)
 - Custom rules for domain-specific terms
 - C FFI for integration with Swift, Python, and other languages
 
@@ -243,14 +277,35 @@ npm run wasm:publish
 ### CLI Tools
 
 ```bash
-# Build the Rust library (release, with FFI)
-cargo build --release --target aarch64-apple-darwin --features ffi
+# Build the Rust library for this Mac's architecture.
+RUST_TARGET="$(rustc -vV | sed -n 's/^host: //p')"
+cargo build --release --target "$RUST_TARGET" --features "ffi,fst-engine"
 
 # Build Swift CLI tools
 cd swift-test && swift build
 ```
 
-Binaries are at `swift-test/.build/debug/nemo-itn` and `swift-test/.build/debug/nemo-tn`.
+Binaries are at `swift-test/.build/debug/nemo-itn`,
+`swift-test/.build/debug/nemo-tn`, and
+`swift-test/.build/debug/nemo-tn-aligned`.
+
+#### nemo-tn
+```bash
+swift-test/.build/debug/nemo-tn -s 'The price is $1,234.56.'
+# output: The price is one thousand two hundred and thirty four point five six dollars
+```
+
+#### nemo-tn-aligned
+The aligned CLI emits compact JSON for argument input and JSON Lines for stdin:
+
+```bash
+swift-test/.build/debug/nemo-tn-aligned --lang en 'The price is $1,234.56.'
+# output: {"input":"The price is $1,234.56.","language":"en","normalized":"The price is one thousand two hundred and thirty four dollars fifty six cents.","spans":[{"input_end":3,"input_start":0,"kind":"word","normalized":"The","original":"The"},{"input_end":9,"input_start":4,"kind":"word","normalized":"price","original":"price"},{"input_end":12,"input_start":10,"kind":"word","normalized":"is","original":"is"},{"input_end":22,"input_start":13,"kind":"money","normalized":"one thousand two hundred and thirty four dollars fifty six cents","original":"$1,234.56"},{"input_end":23,"input_start":22,"kind":"punctuation","normalized":".","original":"."}]}
+```
+
+Each result contains `input`, `language`, the complete `normalized` text, and
+`spans` with `input_start`, `input_end`, `original`, `normalized`, and `kind`.
+Offsets are half-open UTF-8 byte offsets.
 
 ### Swift (XCFramework)
 

src/ffi.rs

@@ -10,6 +10,27 @@ use crate::{
     tn_normalize_sentence_with_max_span_lang, NormalizeOptions,
 };
 
+/// One source-to-normalized TN span returned through the C ABI.
+///
+/// All strings are owned by the containing [`NemoTnAlignment`] and remain valid
+/// until [`nemo_tn_alignment_free`] is called.
+#[repr(C)]
+pub struct NemoTnAlignedSpan {
+    pub input_start: usize,
+    pub input_end: usize,
+    pub original: *mut c_char,
+    pub normalized: *mut c_char,
+    pub kind: *mut c_char,
+}
+
+/// Sentence-level compiled-FST TN output and its source-span mapping.
+#[repr(C)]
+pub struct NemoTnAlignment {
+    pub normalized: *mut c_char,
+    pub spans: *mut NemoTnAlignedSpan,
+    pub span_count: usize,
+}
+
 /// Build [`NormalizeOptions`] from FFI primitives.
 ///
 /// `concat_compound_numbers`: any non-zero value enables concat behavior
@@ -494,6 +515,128 @@ pub unsafe extern "C" fn nemo_tn_fst(input: *const c_char, lang: *const c_char)
     }
 }
 
+/// Normalize with the compiled-FST engine and retain every source span.
+///
+/// Offsets are half-open UTF-8 byte offsets into `input`. The returned object
+/// and every pointer it owns must be released with
+/// [`nemo_tn_alignment_free`].
+///
+/// Returns null when the library was built without `fst-engine`, the language
+/// is unsupported, either input string is invalid UTF-8, or allocation of a C
+/// string fails.
+///
+/// # Safety
+/// - `input` and `lang` must be valid null-terminated UTF-8 strings.
+/// - The result must be freed exactly once with [`nemo_tn_alignment_free`].
+#[no_mangle]
+#[cfg(feature = "fst-engine")]
+pub unsafe extern "C" fn nemo_tn_fst_aligned(
+    input: *const c_char,
+    lang: *const c_char,
+) -> *mut NemoTnAlignment {
+    if input.is_null() || lang.is_null() {
+        return ptr::null_mut();
+    }
+    let input_str = match CStr::from_ptr(input).to_str() {
+        Ok(s) => s,
+        Err(_) => return ptr::null_mut(),
+    };
+    let lang_str = match CStr::from_ptr(lang).to_str() {
+        Ok(s) => s,
+        Err(_) => return ptr::null_mut(),
+    };
+
+    let Some(alignment) = crate::fst::normalize_aligned(input_str, lang_str) else {
+        return ptr::null_mut();
+    };
+    let Ok(normalized) = CString::new(alignment.normalized) else {
+        return ptr::null_mut();
+    };
+
+    let converted = alignment
+        .spans
+        .into_iter()
+        .map(|span| {
+            Some((
+                span.input_start,
+                span.input_end,
+                CString::new(span.original).ok()?,
+                CString::new(span.normalized).ok()?,
+                CString::new(span.kind.as_str()).ok()?,
+            ))
+        })
+        .collect::<Option<Vec<_>>>();
+    let Some(converted) = converted else {
+        return ptr::null_mut();
+    };
+
+    let spans = converted
+        .into_iter()
+        .map(
+            |(input_start, input_end, original, normalized, kind)| NemoTnAlignedSpan {
+                input_start,
+                input_end,
+                original: original.into_raw(),
+                normalized: normalized.into_raw(),
+                kind: kind.into_raw(),
+            },
+        )
+        .collect::<Vec<_>>()
+        .into_boxed_slice();
+    let span_count = spans.len();
+    let spans = Box::into_raw(spans) as *mut NemoTnAlignedSpan;
+
+    Box::into_raw(Box::new(NemoTnAlignment {
+        normalized: normalized.into_raw(),
+        spans,
+        span_count,
+    }))
+}
+
+#[no_mangle]
+#[cfg(not(feature = "fst-engine"))]
+pub unsafe extern "C" fn nemo_tn_fst_aligned(
+    _input: *const c_char,
+    _lang: *const c_char,
+) -> *mut NemoTnAlignment {
+    ptr::null_mut()
+}
+
+/// Free a result returned by [`nemo_tn_fst_aligned`].
+///
+/// Passing null is allowed.
+///
+/// # Safety
+/// - `alignment` must be null or a pointer returned by
+///   [`nemo_tn_fst_aligned`].
+/// - The pointer must not be freed more than once.
+#[no_mangle]
+pub unsafe extern "C" fn nemo_tn_alignment_free(alignment: *mut NemoTnAlignment) {
+    if alignment.is_null() {
+        return;
+    }
+
+    let alignment = Box::from_raw(alignment);
+    if !alignment.normalized.is_null() {
+        drop(CString::from_raw(alignment.normalized));
+    }
+    if !alignment.spans.is_null() {
+        let slice = ptr::slice_from_raw_parts_mut(alignment.spans, alignment.span_count);
+        let mut spans = Box::from_raw(slice);
+        for span in spans.iter_mut() {
+            if !span.original.is_null() {
+                drop(CString::from_raw(span.original));
+            }
+            if !span.normalized.is_null() {
+                drop(CString::from_raw(span.normalized));
+            }
+            if !span.kind.is_null() {
+                drop(CString::from_raw(span.kind));
+            }
+        }
+    }
+}
+
 #[cfg(feature = "fst-engine")]
 fn fst_normalize(input: &str, lang: &str) -> Option<String> {
     use crate::fst;