From 94fb3aec80ca1db35008589b0b721bc2d420e88d Mon Sep 17 00:00:00 2001 From: Mayuri S Date: Mon, 29 Jun 2026 11:39:13 +0530 Subject: [PATCH 1/2] Hi ITN: implement Roman semiotic class MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an inverse-text-normalization Roman numeral class for Hindi that converts spoken numbers following a fixed set of context key words (अध्याय / खंड / खण्ड / कक्षा) into Roman numerals. अध्याय तीन -> अध्याय III कक्षा बारह -> कक्षा XII Conversion is restricted to these predictable contexts; regnal/papal and product names are a documented limitation. Mirrors the TN Roman class structure (data/__init__.py, license headers, pytest + sparrowhawk tests). Signed-off-by: Mayuri S --- .../hi/data/roman/__init__.py | 13 +++ .../hi/data/roman/key_words.tsv | 4 + .../hi/data/roman/roman_numerals.tsv | 13 +++ .../hi/taggers/roman.py | 95 +++++++++++++++++++ .../hi/taggers/tokenize_and_classify.py | 4 + .../hi/verbalizers/roman.py | 38 ++++++++ .../hi/verbalizers/verbalize.py | 3 + .../test_cases_roman.txt | 11 +++ tests/nemo_text_processing/hi/test_roman.py | 31 ++++++ ..._sparrowhawk_inverse_text_normalization.sh | 5 + 10 files changed, 217 insertions(+) create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/roman/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/roman/key_words.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/roman/roman_numerals.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/taggers/roman.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/verbalizers/roman.py create mode 100644 tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_roman.txt create mode 100644 tests/nemo_text_processing/hi/test_roman.py diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/roman/__init__.py b/nemo_text_processing/inverse_text_normalization/hi/data/roman/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/roman/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/roman/key_words.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/roman/key_words.tsv new file mode 100644 index 000000000..208c2375c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/roman/key_words.tsv @@ -0,0 +1,4 @@ +अध्याय +खंड +खण्ड +कक्षा diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/roman/roman_numerals.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/roman/roman_numerals.tsv new file mode 100644 index 000000000..f443fd3a9 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/roman/roman_numerals.tsv @@ -0,0 +1,13 @@ +I 1 +V 5 +X 10 +L 50 +C 100 +D 500 +M 1000 +IV 4 +IX 9 +XL 40 +XC 90 +CD 400 +CM 900 diff --git a/nemo_text_processing/inverse_text_normalization/hi/taggers/roman.py b/nemo_text_processing/inverse_text_normalization/hi/taggers/roman.py new file mode 100644 index 000000000..ed1888eb5 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/taggers/roman.py @@ -0,0 +1,95 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hi.graph_utils import ( + DEVANAGARI_DIGIT, + NEMO_SIGMA, + GraphFst, + delete_space, + insert_space, +) +from nemo_text_processing.inverse_text_normalization.hi.utils import get_abs_path, load_labels + + +class RomanFst(GraphFst): + """ + Finite state transducer for classifying spoken numbers as Roman numerals + when they follow a small, fixed set of context key words (chapter, volume, + class numbering). The conversion is deliberately restricted to these + predictable contexts; regnal, papal and product names (e.g. भास्कर-II) are a + documented limitation because the same number is ambiguous between Arabic and + Roman form. + e.g. अध्याय तीन -> tokens { roman { key: "अध्याय" integer: "III" } } + e.g. कक्षा दस -> tokens { roman { key: "कक्षा" integer: "X" } } + + Args: + cardinal: CardinalFst, used to read spoken numbers. + """ + + MAX_NUMBER = 3999 + + def __init__(self, cardinal: GraphFst): + super().__init__(name="roman", kind="classify") + + key_words = [label[0] for label in load_labels(get_abs_path("data/roman/key_words.tsv"))] + key_words_fst = pynini.union(*[pynini.accep(word) for word in key_words]).optimize() + + roman_to_value = { + roman: int(value) for roman, value in load_labels(get_abs_path("data/roman/roman_numerals.tsv")) + } + value_to_roman = {value: roman for roman, value in roman_to_value.items()} + + not_quote = pynini.closure(pynini.difference(NEMO_SIGMA, pynini.accep('"')), 1) + strip_cardinal_tags = pynutil.delete('cardinal { integer: "') + not_quote + pynutil.delete('" }') + cardinal_to_devanagari = pynini.compose(cardinal.fst, strip_cardinal_tags).optimize() + + single_digit_to_devanagari = ( + pynini.string_file(get_abs_path("data/numbers/digit.tsv")).invert() + | pynini.string_file(get_abs_path("data/numbers/zero.tsv")).invert() + ) + glyph_to_ascii = pynini.union( + *[pynini.cross(glyph, str(value)) for value, glyph in enumerate(DEVANAGARI_DIGIT)] + ) + devanagari_to_ascii = pynini.cdrewrite(glyph_to_ascii, "", "", NEMO_SIGMA) + spoken_to_ascii = pynini.compose( + cardinal_to_devanagari | single_digit_to_devanagari, devanagari_to_ascii + ).optimize() + + ascii_to_roman = pynini.string_map( + [(str(value), self._int_to_roman(value, value_to_roman)) for value in range(1, self.MAX_NUMBER + 1)] + ).optimize() + spoken_to_roman = pynini.compose(spoken_to_ascii, ascii_to_roman).optimize() + + graph = ( + pynutil.insert("key: \"") + + key_words_fst + + pynutil.insert("\"") + + delete_space + + insert_space + + pynutil.insert("integer: \"") + + spoken_to_roman + + pynutil.insert("\"") + ) + self.fst = self.add_tokens(graph).optimize() + + def _int_to_roman(self, number, value_to_roman): + roman = "" + for value in sorted(value_to_roman, reverse=True): + while number >= value: + roman += value_to_roman[value] + number -= value + return roman diff --git a/nemo_text_processing/inverse_text_normalization/hi/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/hi/taggers/tokenize_and_classify.py index 50abab0e5..a9d2dc492 100644 --- a/nemo_text_processing/inverse_text_normalization/hi/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/inverse_text_normalization/hi/taggers/tokenize_and_classify.py @@ -33,6 +33,7 @@ from nemo_text_processing.inverse_text_normalization.hi.taggers.money import MoneyFst from nemo_text_processing.inverse_text_normalization.hi.taggers.ordinal import OrdinalFst from nemo_text_processing.inverse_text_normalization.hi.taggers.punctuation import PunctuationFst +from nemo_text_processing.inverse_text_normalization.hi.taggers.roman import RomanFst from nemo_text_processing.inverse_text_normalization.hi.taggers.telephone import TelephoneFst from nemo_text_processing.inverse_text_normalization.hi.taggers.time import TimeFst from nemo_text_processing.inverse_text_normalization.hi.taggers.whitelist import WhiteListFst @@ -89,6 +90,8 @@ def __init__( money_graph = money.fst telephone = TelephoneFst(cardinal) telephone_graph = telephone.fst + roman = RomanFst(cardinal) + roman_graph = roman.fst punct_graph = PunctuationFst().fst whitelist_graph = WhiteListFst().fst word_graph = WordFst().fst @@ -103,6 +106,7 @@ def __init__( | pynutil.add_weight(measure_graph, 1.1) | pynutil.add_weight(money_graph, 1.1) | pynutil.add_weight(telephone_graph, 1.1) + | pynutil.add_weight(roman_graph, 1.1) | pynutil.add_weight(word_graph, 100) | pynutil.add_weight(whitelist_graph, 1.01) ) diff --git a/nemo_text_processing/inverse_text_normalization/hi/verbalizers/roman.py b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/roman.py new file mode 100644 index 000000000..8cc69f48c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/roman.py @@ -0,0 +1,38 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hi.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_space, + insert_space, +) + + +class RomanFst(GraphFst): + """ + Finite state transducer for verbalizing Roman numerals + e.g. tokens { roman { key: "अध्याय" integer: "III" } } -> अध्याय III + """ + + def __init__(self): + super().__init__(name="roman", kind="verbalize") + key = pynutil.delete("key: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + integer = pynutil.delete("integer: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + graph = key + delete_space + insert_space + integer + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hi/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/verbalize.py index f1a6c55a3..f69dd545f 100644 --- a/nemo_text_processing/inverse_text_normalization/hi/verbalizers/verbalize.py +++ b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/verbalize.py @@ -21,6 +21,7 @@ from nemo_text_processing.inverse_text_normalization.hi.verbalizers.measure import MeasureFst from nemo_text_processing.inverse_text_normalization.hi.verbalizers.money import MoneyFst from nemo_text_processing.inverse_text_normalization.hi.verbalizers.ordinal import OrdinalFst +from nemo_text_processing.inverse_text_normalization.hi.verbalizers.roman import RomanFst from nemo_text_processing.inverse_text_normalization.hi.verbalizers.telephone import TelephoneFst from nemo_text_processing.inverse_text_normalization.hi.verbalizers.time import TimeFst from nemo_text_processing.inverse_text_normalization.hi.verbalizers.whitelist import WhiteListFst @@ -48,6 +49,7 @@ def __init__(self): measure_graph = MeasureFst(cardinal, decimal).fst money_graph = MoneyFst(cardinal, decimal).fst telephone_graph = TelephoneFst(cardinal).fst + roman_graph = RomanFst().fst word_graph = WordFst().fst whitelist_graph = WhiteListFst().fst @@ -63,5 +65,6 @@ def __init__(self): | measure_graph | money_graph | telephone_graph + | roman_graph ) self.fst = graph diff --git a/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_roman.txt b/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_roman.txt new file mode 100644 index 000000000..b1736b43c --- /dev/null +++ b/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_roman.txt @@ -0,0 +1,11 @@ +अध्याय एक~अध्याय I +अध्याय तीन~अध्याय III +अध्याय चार~अध्याय IV +अध्याय नौ~अध्याय IX +खंड पाँच~खंड V +खण्ड सात~खण्ड VII +कक्षा दस~कक्षा X +कक्षा बारह~कक्षा XII +अध्याय बीस~अध्याय XX +अध्याय चालीस~अध्याय XL +अध्याय निन्यानवे~अध्याय XCIX diff --git a/tests/nemo_text_processing/hi/test_roman.py b/tests/nemo_text_processing/hi/test_roman.py new file mode 100644 index 000000000..a4cca4750 --- /dev/null +++ b/tests/nemo_text_processing/hi/test_roman.py @@ -0,0 +1,31 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestRoman: + inverse_normalizer = InverseNormalizer(lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('hi/data_inverse_text_normalization/test_cases_roman.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() diff --git a/tests/nemo_text_processing/hi/test_sparrowhawk_inverse_text_normalization.sh b/tests/nemo_text_processing/hi/test_sparrowhawk_inverse_text_normalization.sh index 0e31a1a00..1c52da813 100644 --- a/tests/nemo_text_processing/hi/test_sparrowhawk_inverse_text_normalization.sh +++ b/tests/nemo_text_processing/hi/test_sparrowhawk_inverse_text_normalization.sh @@ -83,6 +83,11 @@ testITNWhiteList() { runtest $input } +testITNRoman() { + input=$PROJECT_DIR/hi/data_inverse_text_normalization/test_cases_roman.txt + runtest $input +} + # Load shUnit2 . $PROJECT_DIR/../shunit2/shunit2 From d2d4f55fa93dea0616f3b6f1b0c2ae535ec541b5 Mon Sep 17 00:00:00 2001 From: Mayuri S Date: Tue, 30 Jun 2026 16:30:23 +0530 Subject: [PATCH 2/2] Hi ITN Roman: address review feedback Signed-off-by: Mayuri S --- Jenkinsfile | 2 +- .../hi/taggers/roman.py | 45 +++++++------------ .../hi/verbalizers/roman.py | 6 +-- 3 files changed, 19 insertions(+), 34 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4ac66d02f..628d20781 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -28,7 +28,7 @@ pipeline { HY_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-0' MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1' JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1' - HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-26-5' + HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-30-26-0' KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6' DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' } diff --git a/nemo_text_processing/inverse_text_normalization/hi/taggers/roman.py b/nemo_text_processing/inverse_text_normalization/hi/taggers/roman.py index ed1888eb5..b446bf8fe 100644 --- a/nemo_text_processing/inverse_text_normalization/hi/taggers/roman.py +++ b/nemo_text_processing/inverse_text_normalization/hi/taggers/roman.py @@ -16,11 +16,10 @@ from pynini.lib import pynutil from nemo_text_processing.inverse_text_normalization.hi.graph_utils import ( - DEVANAGARI_DIGIT, - NEMO_SIGMA, GraphFst, delete_space, insert_space, + integer_to_devanagari, ) from nemo_text_processing.inverse_text_normalization.hi.utils import get_abs_path, load_labels @@ -33,8 +32,8 @@ class numbering). The conversion is deliberately restricted to these predictable contexts; regnal, papal and product names (e.g. भास्कर-II) are a documented limitation because the same number is ambiguous between Arabic and Roman form. - e.g. अध्याय तीन -> tokens { roman { key: "अध्याय" integer: "III" } } - e.g. कक्षा दस -> tokens { roman { key: "कक्षा" integer: "X" } } + e.g. अध्याय तीन -> tokens { roman { key_cardinal: "अध्याय" integer: "III" } } + e.g. कक्षा दस -> tokens { roman { key_cardinal: "कक्षा" integer: "X" } } Args: cardinal: CardinalFst, used to read spoken numbers. @@ -48,41 +47,27 @@ def __init__(self, cardinal: GraphFst): key_words = [label[0] for label in load_labels(get_abs_path("data/roman/key_words.tsv"))] key_words_fst = pynini.union(*[pynini.accep(word) for word in key_words]).optimize() - roman_to_value = { - roman: int(value) for roman, value in load_labels(get_abs_path("data/roman/roman_numerals.tsv")) + value_to_roman = { + int(value): roman for roman, value in load_labels(get_abs_path("data/roman/roman_numerals.tsv")) } - value_to_roman = {value: roman for roman, value in roman_to_value.items()} - not_quote = pynini.closure(pynini.difference(NEMO_SIGMA, pynini.accep('"')), 1) - strip_cardinal_tags = pynutil.delete('cardinal { integer: "') + not_quote + pynutil.delete('" }') - cardinal_to_devanagari = pynini.compose(cardinal.fst, strip_cardinal_tags).optimize() - - single_digit_to_devanagari = ( - pynini.string_file(get_abs_path("data/numbers/digit.tsv")).invert() - | pynini.string_file(get_abs_path("data/numbers/zero.tsv")).invert() - ) - glyph_to_ascii = pynini.union( - *[pynini.cross(glyph, str(value)) for value, glyph in enumerate(DEVANAGARI_DIGIT)] - ) - devanagari_to_ascii = pynini.cdrewrite(glyph_to_ascii, "", "", NEMO_SIGMA) - spoken_to_ascii = pynini.compose( - cardinal_to_devanagari | single_digit_to_devanagari, devanagari_to_ascii - ).optimize() - - ascii_to_roman = pynini.string_map( - [(str(value), self._int_to_roman(value, value_to_roman)) for value in range(1, self.MAX_NUMBER + 1)] + devanagari_to_roman = pynini.string_map( + [ + (integer_to_devanagari(value), self._int_to_roman(value, value_to_roman)) + for value in range(1, self.MAX_NUMBER + 1) + ] ).optimize() - spoken_to_roman = pynini.compose(spoken_to_ascii, ascii_to_roman).optimize() + spoken_to_roman = pynini.compose(cardinal.graph_no_exception, devanagari_to_roman).optimize() graph = ( - pynutil.insert("key: \"") + pynutil.insert('key_cardinal: "') + key_words_fst - + pynutil.insert("\"") + + pynutil.insert('"') + delete_space + insert_space - + pynutil.insert("integer: \"") + + pynutil.insert('integer: "') + spoken_to_roman - + pynutil.insert("\"") + + pynutil.insert('"') ) self.fst = self.add_tokens(graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hi/verbalizers/roman.py b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/roman.py index 8cc69f48c..5b53ee903 100644 --- a/nemo_text_processing/inverse_text_normalization/hi/verbalizers/roman.py +++ b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/roman.py @@ -26,13 +26,13 @@ class RomanFst(GraphFst): """ Finite state transducer for verbalizing Roman numerals - e.g. tokens { roman { key: "अध्याय" integer: "III" } } -> अध्याय III + e.g. tokens { roman { key_cardinal: "अध्याय" integer: "III" } } -> अध्याय III """ def __init__(self): super().__init__(name="roman", kind="verbalize") - key = pynutil.delete("key: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + key_cardinal = pynutil.delete("key_cardinal: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") integer = pynutil.delete("integer: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") - graph = key + delete_space + insert_space + integer + graph = key_cardinal + delete_space + insert_space + integer delete_tokens = self.delete_tokens(graph) self.fst = delete_tokens.optimize()