diff --git a/nemo_text_processing/inverse_text_normalization/inverse_normalize.py b/nemo_text_processing/inverse_text_normalization/inverse_normalize.py index 9a6fcc64c..4ce15346c 100644 --- a/nemo_text_processing/inverse_text_normalization/inverse_normalize.py +++ b/nemo_text_processing/inverse_text_normalization/inverse_normalize.py @@ -121,6 +121,11 @@ def __init__( from nemo_text_processing.inverse_text_normalization.hi.verbalizers.verbalize_final import ( VerbalizeFinalFst, ) + elif lang == 'te': # Telugu + from nemo_text_processing.inverse_text_normalization.te.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.inverse_text_normalization.te.verbalizers.verbalize_final import ( + VerbalizeFinalFst, + ) elif lang == 'hi_en': # Hindi-English code-switch from nemo_text_processing.inverse_text_normalization.hi_en.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.inverse_text_normalization.hi_en.verbalizers.verbalize_final import ( @@ -207,6 +212,7 @@ def parse_args(): 'zh', 'he', 'hi', + 'te', 'hy', 'mr', 'ja', diff --git a/nemo_text_processing/inverse_text_normalization/run_evaluate.py b/nemo_text_processing/inverse_text_normalization/run_evaluate.py index cf9b29fce..3a989f96c 100644 --- a/nemo_text_processing/inverse_text_normalization/run_evaluate.py +++ b/nemo_text_processing/inverse_text_normalization/run_evaluate.py @@ -44,6 +44,7 @@ def parse_args(): "fr", "hi", "hi_en", + "te", "hy", "ko", "mr", diff --git a/nemo_text_processing/inverse_text_normalization/te/__init__.py b/nemo_text_processing/inverse_text_normalization/te/__init__.py new file mode 100644 index 000000000..d3f64b4cb --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.te.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.inverse_text_normalization.te.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.inverse_text_normalization.te.verbalizers.verbalize_final import VerbalizeFinalFst diff --git a/nemo_text_processing/inverse_text_normalization/te/data/__init__.py b/nemo_text_processing/inverse_text_normalization/te/data/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/__init__.py b/nemo_text_processing/inverse_text_normalization/te/data/numbers/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/case_suffix.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/case_suffix.tsv new file mode 100644 index 000000000..98724c4f5 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/case_suffix.tsv @@ -0,0 +1,5 @@ +కి +కు +లో +తో +కోసం \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/crore_words.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/crore_words.tsv new file mode 100644 index 000000000..b6ae0b8d5 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/crore_words.tsv @@ -0,0 +1,3 @@ +కోటి +కోట్లు +కోట్ల \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/digit.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/digit.tsv new file mode 100644 index 000000000..6606ff8b3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +౧ ఒకటి +౨ రెండు +౩ మూడు +౪ నాలుగు +౫ ఐదు +౬ ఆరు +౭ ఏడు +౮ ఎనిమిది +౯ తొమ్మిది \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/digit_people.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/digit_people.tsv new file mode 100644 index 000000000..8f3cfaadf --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/digit_people.tsv @@ -0,0 +1,9 @@ +ఇద్దరు ౨ +ముగ్గురు ౩ +నలుగురు ౪ +ఐదుగురు ౫ +ఆరుగురు ౬ +ఏడుగురు ౭ +ఎనిమిదుగురు ౮ +తొమ్మిదుగురు ౯ +పది మంది ౧౦ \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/hundred.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/hundred.tsv new file mode 100644 index 000000000..366fd6221 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/hundred.tsv @@ -0,0 +1 @@ +వంద \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/hundred_words.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/hundred_words.tsv new file mode 100644 index 000000000..c231e10d7 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/hundred_words.tsv @@ -0,0 +1,4 @@ +వంద +వందల +వందలు +నూరు \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/labels_exception.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/labels_exception.tsv new file mode 100644 index 000000000..74311cf0e --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/labels_exception.tsv @@ -0,0 +1,2 @@ +# Telugu cardinal exceptions. +# Add entries here only when a valid cardinal expression should be excluded from cardinal classification. \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/lakh_words.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/lakh_words.tsv new file mode 100644 index 000000000..6b791a4ef --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/lakh_words.tsv @@ -0,0 +1,4 @@ +లక్ష +లక్షా +లక్షలు +లక్షల \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/special_numbers.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/special_numbers.tsv new file mode 100644 index 000000000..49da8c121 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/special_numbers.tsv @@ -0,0 +1,2 @@ +పంతొమ్మిది ౧౯ +ఒక ౧ diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/teens_and_ties.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/teens_and_ties.tsv new file mode 100644 index 000000000..5c0ca90ed --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/teens_and_ties.tsv @@ -0,0 +1,18 @@ +౧౦ పది +౧౧ పదకొండు +౧౨ పన్నెండు +౧౩ పదమూడు +౧౪ పద్నాలుగు +౧౫ పదిహేను +౧౬ పదహారు +౧౭ పదిహేడు +౧౮ పద్దెనిమిది +౧౯ పందొమ్మిది +౨౦ ఇరవై +౩౦ ముప్పై +౪౦ నలభై +౫౦ యాభై +౬౦ అరవై +౭౦ డెబ్బై +౮౦ ఎనభై +౯౦ తొంభై \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/thousand_words.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/thousand_words.tsv new file mode 100644 index 000000000..5b2afcc40 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/thousand_words.tsv @@ -0,0 +1,3 @@ +వెయ్యి +వేలు +వేల \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/thousands.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/thousands.tsv new file mode 100644 index 000000000..c44a35927 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/thousands.tsv @@ -0,0 +1,3 @@ +వెయ్యి +లక్ష +కోటి \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/ties_prefix.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/ties_prefix.tsv new file mode 100644 index 000000000..7ace1c04e --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/ties_prefix.tsv @@ -0,0 +1,8 @@ +ఇరవై ౨ +ముప్పై ౩ +నలభై ౪ +యాభై ౫ +అరవై ౬ +డెబ్బై ౭ +ఎనభై ౮ +తొంభై ౯ \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/zero.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/zero.tsv new file mode 100644 index 000000000..5f95b48d6 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/zero.tsv @@ -0,0 +1 @@ +౦ సున్నా \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/te/graph_utils.py b/nemo_text_processing/inverse_text_normalization/te/graph_utils.py new file mode 100644 index 000000000..025539f62 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/graph_utils.py @@ -0,0 +1,204 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import string +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.examples import plurals +from pynini.export import export +from pynini.lib import byte, pynutil, utf8 + +from nemo_text_processing.inverse_text_normalization.te.utils import get_abs_path, load_labels + +NEMO_CHAR = utf8.VALID_UTF8_CHAR + +graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + +NEMO_TE_DIGIT = pynini.union("౦", "౧", "౨", "౩", "౪", "౫", "౬", "౭", "౮", "౯").optimize() +TELUGU_DIGIT = ["౦", "౧", "౨", "౩", "౪", "౫", "౬", "౭", "౮", "౯"] + +NEMO_HEX = pynini.union(*string.hexdigits).optimize() +NEMO_NON_BREAKING_SPACE = u"\u00a0" +NEMO_ZWNJ = u"\u200c" +NEMO_SPACE = " " +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", u"\u00a0").optimize() +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() + +NEMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize() +NEMO_GRAPH = pynini.union(NEMO_CHAR, NEMO_PUNCT).optimize() + +NEMO_SIGMA = pynini.closure(NEMO_CHAR) + +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +delete_zero_or_one_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE, 0, 1)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") +delete_preserve_order = pynini.closure( + pynutil.delete(" preserve_order: true") + | (pynutil.delete(" field_order: \"") + NEMO_NOT_QUOTE + pynutil.delete("\"")) +) + + +MIN_NEG_WEIGHT = -0.0001 +MIN_POS_WEIGHT = 0.0001 +INPUT_CASED = "cased" +INPUT_LOWER_CASED = "lower_cased" +MINUS = pynini.union("ఋణాత్మక", "మైనస్").optimize() + + +def integer_to_telugu(n: int) -> str: + return ''.join(TELUGU_DIGIT[int(d)] for d in str(n)) + + +def generator_main(file_name: str, graphs: Dict[str, 'pynini.FstLike']): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logging.info(f'Created {file_name}') + + +def convert_space(fst) -> 'pynini.FstLike': + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA) + + +def string_map_cased(input_file: str, input_case: str = INPUT_LOWER_CASED): + labels = load_labels(input_file) + + if input_case == INPUT_CASED: + additional_labels = [] + for written, spoken, *weight in labels: + written_capitalized = written[0].upper() + written[1:] + additional_labels.extend( + [ + [written_capitalized, spoken.capitalize()], + [ + written_capitalized, + spoken.upper().replace(" AND ", " and "), + ], + ] + ) + + spoken_no_space = spoken.replace(" ", "") + + if len(spoken) == (2 * len(spoken_no_space) - 1): + logging.debug(f"This is weight {weight}") + if len(weight) == 0: + additional_labels.extend( + [[written, spoken_no_space], [written_capitalized, spoken_no_space.upper()]] + ) + else: + additional_labels.extend( + [ + [written, spoken_no_space, weight[0]], + [written_capitalized, spoken_no_space.upper(), weight[0]], + ] + ) + labels += additional_labels + + whitelist = pynini.string_map(labels).invert().optimize() + return whitelist + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + '/grammars/' + kind + '/' + name + '.far') + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> 'pynini.FstLike': + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> 'pynini.FstLike': + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> 'pynini.FstLike': + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/inverse_text_normalization/te/taggers/__init__.py b/nemo_text_processing/inverse_text_normalization/te/taggers/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/te/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/te/taggers/cardinal.py new file mode 100644 index 000000000..7c67fb129 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/taggers/cardinal.py @@ -0,0 +1,342 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright 2026 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.te.graph_utils import ( + INPUT_LOWER_CASED, + MINUS, + NEMO_SPACE, + NEMO_TE_DIGIT, + GraphFst, + delete_space, + integer_to_telugu, +) +from nemo_text_processing.inverse_text_normalization.te.utils import get_abs_path, load_labels + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying Telugu cardinals. + """ + + def __init__(self, input_case: str = INPUT_LOWER_CASED): + super().__init__(name="cardinal", kind="classify") + self.input_case = input_case + + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")).invert() + + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")).invert() + + graph_digit_people = pynini.string_file(get_abs_path("data/numbers/digit_people.tsv")) + + graph_digit |= graph_digit_people + graph_digit |= pynini.cross("ఒక", "౧") + + graph_teens_and_ties = pynini.string_file(get_abs_path("data/numbers/teens_and_ties.tsv")).invert() + + graph_special = pynini.string_file(get_abs_path("data/numbers/special_numbers.tsv")) + + graph_teens_and_ties |= graph_special + + self.graph_zero = graph_zero + self.graph_digit = graph_digit + graph_leading_zero_sequence = graph_zero + pynini.closure( + delete_space + (graph_zero | graph_digit), + 1, + ) + graph_ties_prefix = pynini.string_file(get_abs_path("data/numbers/ties_prefix.tsv")) + + graph_okkati = pynini.cross("ఒకటి", "౧") + graph_ties_plus_okkati = graph_ties_prefix + delete_space + graph_okkati + + self.graph_single_digit_with_zero = pynutil.insert("౦") + graph_digit + + self.graph_two_digit = ( + graph_teens_and_ties + | graph_ties_plus_okkati + | (graph_ties_prefix + delete_space + graph_digit) + | self.graph_single_digit_with_zero + ) + hundred_words = [row[0] for row in load_labels(get_abs_path("data/numbers/hundred_words.tsv"))] + thousand_words = [row[0] for row in load_labels(get_abs_path("data/numbers/thousand_words.tsv"))] + lakh_words = [row[0] for row in load_labels(get_abs_path("data/numbers/lakh_words.tsv"))] + crore_words = [row[0] for row in load_labels(get_abs_path("data/numbers/crore_words.tsv"))] + graph_hundred_word = pynini.union(*hundred_words) + delete_hundred = pynutil.delete(graph_hundred_word) + + graph_hundreds_component = pynini.union( + graph_digit + delete_space + delete_hundred, + pynutil.insert("౦"), + ) + graph_hundreds_component += delete_space + graph_hundreds_component += self.graph_two_digit | pynutil.insert("౦౦") + + graph_hundred_standalone = pynini.union(*[pynini.cross(word, "౧౦౦") for word in hundred_words]) + + two_digit_hundred_values = pynini.string_map( + [(integer_to_telugu(i), integer_to_telugu(i * 100)) for i in range(10, 100)] + ).optimize() + + graph_hundred_as_thousand = (self.graph_two_digit @ two_digit_hundred_values) + delete_space + delete_hundred + + graph_nuta_component = ( + pynutil.delete(pynini.union("నూట", "వంద")) + pynutil.insert("౧") + delete_space + self.graph_two_digit + ) + self.graph_hundreds = ( + graph_hundred_as_thousand | graph_hundreds_component | graph_hundred_standalone | graph_nuta_component + ) + + self.graph_hundred_component_at_least_one_none_zero_digit = graph_hundreds_component @ ( + pynini.closure(NEMO_TE_DIGIT) + (NEMO_TE_DIGIT - "౦") + pynini.closure(NEMO_TE_DIGIT) + ) + + graph_below_thousand = self.graph_hundreds | self.graph_two_digit | graph_digit + + graph_remainder_three_digit = ( + self.graph_hundreds | (pynutil.insert("౦") + self.graph_two_digit) | pynutil.insert("౦౦౦") + ) + + graph_group_two_digit_leading = self.graph_two_digit | graph_digit + graph_group_two_digit_padded = ( + self.graph_two_digit | (pynutil.insert("౦") + graph_digit) | pynutil.insert("౦౦") + ) + + case_suffixes = [row[0] for row in load_labels(get_abs_path("data/numbers/case_suffix.tsv"))] + + graph_case_suffix = pynini.union(*case_suffixes) + + delete_thousand = pynini.union( + *[pynutil.delete(word) for word in thousand_words], + *[pynutil.delete(word + graph_case_suffix) for word in thousand_words], + ) + delete_lakh = pynini.union( + *[pynutil.delete(word) for word in lakh_words], + *[pynutil.delete(word + graph_case_suffix) for word in lakh_words], + ) + delete_crore = pynini.union( + *[pynutil.delete(word) for word in crore_words], + *[pynutil.delete(word + graph_case_suffix) for word in crore_words], + ) + + graph_thousands = ( + graph_group_two_digit_leading + delete_space + delete_thousand + delete_space + graph_remainder_three_digit + ) + + graph_thousands_padded = ( + graph_group_two_digit_padded + delete_space + delete_thousand + delete_space + graph_remainder_three_digit + ) + + graph_bare_thousand = ( + pynutil.delete(thousand_words[0]) + pynutil.insert("౧") + delete_space + graph_remainder_three_digit + ) + + graph_bare_thousand_terminal = pynutil.delete(thousand_words[0]) + pynutil.insert("౧౦౦౦") + + graph_lakhs = ( + graph_group_two_digit_leading + + delete_space + + delete_lakh + + delete_space + + graph_group_two_digit_padded + + delete_space + + delete_thousand + + delete_space + + graph_remainder_three_digit + ) + + graph_bare_lakh = ( + graph_group_two_digit_leading + + delete_space + + delete_lakh + + delete_space + + pynutil.insert("౦౦") + + delete_space + + graph_remainder_three_digit + ) + + graph_lakh_terminal = ( + pynini.union( + pynini.cross("లక్ష", "౧"), + pynini.cross("లక్షలు", "౧"), + pynini.cross("లక్షల", "౧"), + ) + | (graph_group_two_digit_leading + delete_space + delete_lakh) + ) + pynutil.insert("౦౦౦౦౦") + + graph_crore_prefix_simple = ( + graph_group_two_digit_leading + | self.graph_hundreds + | graph_thousands + | graph_bare_thousand + | graph_lakhs + | graph_bare_lakh + | graph_lakh_terminal + ) + + graph_crore_terminal = pynutil.add_weight( + ( + pynini.cross("కోటి", "౧") + | pynini.cross("కోట్లు", "౧") + | pynini.cross("కోట్ల", "౧") + | (graph_crore_prefix_simple + delete_space + delete_crore) + ) + + pynutil.insert("౦౦౦౦౦౦౦"), + -0.1, + ) + + graph_bare_crore = ( + graph_crore_prefix_simple + + delete_space + + delete_crore + + delete_space + + pynutil.insert("౦౦") + + delete_space + + pynutil.insert("౦౦") + + delete_space + + graph_remainder_three_digit + ) + + graph_crores = ( + graph_crore_prefix_simple + + delete_space + + delete_crore + + delete_space + + graph_group_two_digit_padded + + delete_space + + delete_lakh + + delete_space + + graph_group_two_digit_padded + + delete_space + + delete_thousand + + delete_space + + graph_remainder_three_digit + ) + + graph_lower_seven = ( + graph_lakhs + | graph_bare_lakh + | graph_lakh_terminal + | (pynutil.insert("౦౦") + graph_thousands_padded) + | (pynutil.insert("౦౦౦") + graph_bare_thousand) + | (pynutil.insert("౦౦౦") + graph_bare_thousand_terminal) + | (pynutil.insert("౦౦౦౦") + graph_remainder_three_digit) + ) + graph_one_lakh = ( + (pynutil.delete("లక్ష") | pynutil.delete("లక్షా")) + + delete_space + + ( + (pynutil.insert("౧౦౦౦౦") + graph_digit) + | (pynutil.insert("౧౦౦౦") + self.graph_two_digit) + | (pynutil.insert("౧౦౦") + self.graph_hundreds) + | (pynutil.insert("౧౦") + graph_bare_thousand_terminal) + | (pynutil.insert("౧") + graph_thousands) + | (pynutil.insert("౧") + graph_thousands_padded) + ) + ) + + graph_one_crore = pynutil.add_weight( + ( + pynutil.delete(crore_words[0]) + + delete_space + + ( + (pynutil.insert("౧౦౦౦౦౦౦") + graph_digit) + | (pynutil.insert("౧౦౦౦౦౦") + self.graph_two_digit) + | (pynutil.insert("౧౦౦౦౦") + self.graph_hundreds) + | (pynutil.insert("౧౦౦౦") + graph_bare_thousand_terminal) + | (pynutil.insert("౧౦౦") + graph_thousands) + | (pynutil.insert("౧౦౦") + graph_thousands_padded) + | (pynutil.insert("౧") + graph_lakhs) + | (pynutil.insert("౧") + graph_bare_lakh) + | (pynutil.insert("౧") + graph_lakh_terminal) + ) + ), + -10.0, + ) + + graph_large_crore_prefix = ( + graph_group_two_digit_leading + | self.graph_hundreds + | graph_thousands + | graph_bare_thousand + | graph_bare_thousand_terminal + | graph_lakhs + | graph_bare_lakh + | graph_lakh_terminal + | graph_crores + | graph_bare_crore + | graph_crore_terminal + ) + + graph_large_crores = ( + graph_large_crore_prefix + + delete_space + + delete_crore + + (pynutil.insert("౦౦౦౦౦౦౦") | (delete_space + graph_lower_seven)) + ) + + graph_no_prefix = pynutil.add_weight( + pynini.union( + *[pynini.cross(word, "౧౦౦") for word in hundred_words], + *[pynini.cross(word, "౧౦౦౦") for word in thousand_words], + *[pynini.cross(word, "౧౦౦౦౦౦") for word in lakh_words], + *[pynini.cross(word, "౧౦౦౦౦౦౦౦") for word in crore_words], + ), + 2, + ) + + graph_base = pynini.union( + graph_large_crores, + graph_crores, + graph_bare_crore, + graph_one_crore, + graph_lakhs, + graph_bare_lakh, + graph_one_lakh, + graph_lakh_terminal, + graph_thousands, + graph_bare_thousand, + graph_bare_thousand_terminal, + graph_zero, + graph_no_prefix, + graph_below_thousand, + ) + + graph_normal = graph_base | (graph_base + graph_case_suffix) + + graph_normal = graph_normal @ pynini.union( + pynutil.delete(pynini.closure("౦")) + + pynini.difference(NEMO_TE_DIGIT, "౦") + + pynini.closure(NEMO_TE_DIGIT), + "౦", + ) + + graph = graph_normal | graph_leading_zero_sequence + + labels_exception = [pynini.string_file(get_abs_path("data/numbers/labels_exception.tsv"))] + graph_exception = pynini.union(*labels_exception).optimize() + + self.graph_no_exception = graph + self.graph = (pynini.project(graph, "input") - graph_exception.arcsort()) @ graph + optional_minus_graph = pynini.closure( + pynutil.insert("negative: ") + pynini.cross(MINUS, "\"-\"") + NEMO_SPACE, + 0, + 1, + ) + + final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph diff --git a/nemo_text_processing/inverse_text_normalization/te/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/te/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..b55338325 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/taggers/tokenize_and_classify.py @@ -0,0 +1,75 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright 2026 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.te.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.inverse_text_normalization.te.taggers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.te.taggers.word import WordFst + + +class ClassifyFst(GraphFst): + """ + Final classification grammar for Telugu ITN cardinal processing. + """ + + def __init__( + self, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + input_case: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify") + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, "te_itn.far") + + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logging.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logging.info("Creating Telugu ClassifyFst grammar.") + + cardinal = CardinalFst() + cardinal_graph = cardinal.fst + word_graph = WordFst().fst + + classify = pynutil.add_weight(cardinal_graph, 1.1) | pynutil.add_weight(word_graph, 100) + + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + + graph = token + pynini.closure(pynutil.add_weight(delete_extra_space + token, 1000.0)) + graph = delete_space + graph + delete_space + + self.fst = graph.optimize() + + if far_file: + generator_main( + far_file, + {"tokenize_and_classify": self.fst}, + ) + logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/inverse_text_normalization/te/taggers/word.py b/nemo_text_processing/inverse_text_normalization/te/taggers/word.py new file mode 100644 index 000000000..95045582a --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/taggers/word.py @@ -0,0 +1,30 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright 2026 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.te.graph_utils import NEMO_NOT_SPACE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying plain Telugu tokens. + """ + + def __init__(self): + super().__init__(name="word", kind="classify") + word = pynutil.insert('name: "') + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert('"') + self.fst = word.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/te/utils.py b/nemo_text_processing/inverse_text_normalization/te/utils.py new file mode 100644 index 000000000..5830b50d0 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/utils.py @@ -0,0 +1,63 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import logging +import os +import pynini + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + abs_path = os.path.dirname(os.path.abspath(__file__)) + os.sep + rel_path + + if not os.path.exists(abs_path): + logging.warning(f'{abs_path} does not exist') + return abs_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + label_tsv = open(abs_path, encoding="utf-8") + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels + + +from pynini.lib import pynutil + + +def apply_fst(text, fst): + """Given a string input, returns the output string + produced by traversing the path with lowest weight. + If no valid path accepts input string, returns an + error. + """ + try: + print(pynini.shortestpath(text @ fst).string()) + except pynini.FstOpError: + print(f"Error: No valid output with given input: '{text}'") diff --git a/nemo_text_processing/inverse_text_normalization/te/verbalizers/__init__.py b/nemo_text_processing/inverse_text_normalization/te/verbalizers/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/te/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/te/verbalizers/cardinal.py new file mode 100644 index 000000000..cf6c3a903 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/verbalizers/cardinal.py @@ -0,0 +1,55 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.te.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinal + e.g. + cardinal { integer: "౨౩" } -> ౨౩ + cardinal { integer: "౨౩" negative: "-" } -> -౨౩ + """ + + def __init__(self): + super().__init__(name="cardinal", kind="verbalize") + + optional_sign = pynini.closure( + pynutil.delete("negative:") + + delete_space + + pynutil.delete("\"") + + NEMO_NOT_QUOTE + + pynutil.delete("\"") + + delete_space, + 0, + 1, + ) + + graph = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + self.numbers = graph + graph = optional_sign + graph + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/te/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/te/verbalizers/verbalize.py new file mode 100644 index 000000000..af4080432 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/verbalizers/verbalize.py @@ -0,0 +1,32 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright 2026 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or impliesd. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo_text_processing.inverse_text_normalization.te.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.te.verbalizers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.te.verbalizers.word import WordFst + + +class VerbalizeFst(GraphFst): + """ + Composes the Telugu cardinal verbalizer grammar. + """ + + def __init__(self): + super().__init__(name="verbalize", kind="verbalize") + + cardinal_graph = CardinalFst().fst + word_graph = WordFst().fst + self.fst = (cardinal_graph | word_graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/te/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/te/verbalizers/verbalize_final.py new file mode 100644 index 000000000..fe839c467 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/verbalizers/verbalize_final.py @@ -0,0 +1,49 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright 2026 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.te.graph_utils import GraphFst, delete_extra_space, delete_space +from nemo_text_processing.inverse_text_normalization.te.verbalizers.verbalize import VerbalizeFst + + +class VerbalizeFinalFst(GraphFst): + """ + Verbalizes full Telugu ITN token output. + + Example: + tokens { cardinal { integer: "౨౩" } } -> ౨౩ + tokens { name: "పరుగులు" } -> పరుగులు + """ + + def __init__(self): + super().__init__(name="verbalize_final", kind="verbalize") + + verbalize = VerbalizeFst().fst + + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + verbalize + + delete_space + + pynutil.delete("}") + ) + + graph = delete_space + graph + pynini.closure(delete_extra_space + graph) + delete_space + + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/te/verbalizers/word.py b/nemo_text_processing/inverse_text_normalization/te/verbalizers/word.py new file mode 100644 index 000000000..28603646c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/verbalizers/word.py @@ -0,0 +1,38 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright 2026 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.te.graph_utils import ( + NEMO_CHAR, + NEMO_SIGMA, + GraphFst, + delete_space, +) + + +class WordFst(GraphFst): + """ + Finite state transducer for verbalizing plain Telugu tokens. + """ + + def __init__(self): + super().__init__(name="word", kind="verbalize") + chars = pynini.closure(NEMO_CHAR - " ", 1) + char = pynutil.delete("name:") + delete_space + pynutil.delete('"') + chars + pynutil.delete('"') + graph = char @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) + + self.fst = graph.optimize() diff --git a/tests/nemo_text_processing/te/__init__.py b/tests/nemo_text_processing/te/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/tests/nemo_text_processing/te/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/nemo_text_processing/te/data_inverse_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/te/data_inverse_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..538edadc0 --- /dev/null +++ b/tests/nemo_text_processing/te/data_inverse_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,48 @@ +నాలుగు నాలుగులు~౪ నాలుగులు +ఆరు మంది ఆటగాళ్లు బయటకు~౬ మంది ఆటగాళ్లు బయటకు +నాలుగు ఓవర్లలో పదిహేడు పరుగులు~౪ ఓవర్లలో ౧౭ పరుగులు +ఐదు చాక్లెట్లు తొమ్మిది టాఫీలు~౫ చాక్లెట్లు ౯ టాఫీలు +పది వేల తొంభై తొమ్మిది~౧౦౦౯౯ +లక్షా ఒకటి~౧౦౦౦౦౧ +ఐదు నాలుగులు~౫ నాలుగులు +నాలుగు మంది మృతి పద్దెనిమిది మందికి గాయాలు~౪ మంది మృతి ౧౮ మందికి గాయాలు +ఐదు వేల నూట రెండు కోట్ల ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు~౫౧౦౨౨౩౪౫౫౬౭ +అధ్యాయం పది శ్లోకం రెండు~అధ్యాయం ౧౦ శ్లోకం ౨ +నూట ఒకటి~౧౦౧ +వంద~౧౦౦ +నూట ఐదు~౧౦౫ +రెండు వందలు~౨౦౦ +రెండు వందల మూడు~౨౦౩ +ఐదు వందల రెండు~౫౦౨ +నూట పదకొండు~౧౧౧ +నూట పంతొమ్మిది~౧౧౯ +నూట ముప్పై~౧౩౦ +నూట యాభై~౧౫౦ +మూడు వందల పన్నెండు~౩౧౨ +ఐదు వందల ఇరవై ఐదు~౫౨౫ +నాలుగు వందల ఇరవై ఐదు~౪౨౫ +తొమ్మిది వందల పదిహేను~౯౧౫ +వెయ్యి ఒకటి~౧౦౦౧ +ఐదు వేల ఐదు వందల యాభై ఒకటి అరటిపండ్లు~౫౫౫౧ అరటిపండ్లు +ఎనభై ఐదు~౮౫ +తొంభై ఒకటి~౯౧ +వెయ్యి మూడు వందల ఇరవై మూడు~౧౩౨౩ +వెయ్యి మూడు వందల నలభై ఐదు~౧౩౪౫ +పదమూడు వేల నాలుగు వందల యాభై ఆరు~౧౩౪౫౬ +పన్నెండు వేల మూడు వందల నలభై ఆరు~౧౨౩౪౬ +లక్ష ఇరవై మూడు వేల నాలుగు వందల యాభై ఆరు~౧౨౩౪౫౬ +ఎనిమిది లక్షల డెబ్బై రెండు వేల తొమ్మిది వందల ఎనభై ఏడు~౮౭౨౯౮౭ +తొమ్మిది లక్షల ఎనభై ఏడు వేల ఆరు వందల తొమ్మిది~౯౮౭౬౦౯ +తొంభై ఎనిమిది లక్షల డెబ్బై ఆరు వేల ఏడు వందల ఎనభై తొమ్మిది~౯౮౭౬౭౮౯ +ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు~౨౩౪౫౫౬౭ +కోటి ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు~౧౨౩౪౫౫౬౭ +కోటి ఇరవై ఒక లక్షల ఇరవై ఒక వేల రెండు వందల పన్నెండు~౧౨౧౨౧౨౧౨ +నూట పన్నెండు కోట్ల ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు~౧౧౨౨౩౪౫౫౬౭ +నూట రెండు కోట్ల ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు~౧౦౨౨౩౪౫౫౬౭ +వెయ్యి నూట రెండు కోట్ల ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు~౧౧౦౨౨౩౪౫౫౬౭ +ఐదు వేల నూట రెండు కోట్ల ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు~౫౧౦౨౨౩౪౫౫౬౭ +సున్నా ఐదు~౦౫ +సున్నా ఒకటి~౦౧ +సున్నా ఏడు మూడు~౦౭౩ +సున్నా సున్నా సున్నా ఒకటి~౦౦౦౧ +సున్నా సున్నా సున్నా~౦౦౦ \ No newline at end of file diff --git a/tests/nemo_text_processing/te/test_cardinal.py b/tests/nemo_text_processing/te/test_cardinal.py new file mode 100644 index 000000000..85a10a177 --- /dev/null +++ b/tests/nemo_text_processing/te/test_cardinal.py @@ -0,0 +1,35 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.te.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.inverse_text_normalization.te.verbalizers.verbalize_final import VerbalizeFinalFst + +from ..utils import parse_test_case_file + + +class TestCardinal: + tagger = ClassifyFst(overwrite_cache=True).fst + verbalizer = VerbalizeFinalFst().fst + + @parameterized.expand(parse_test_case_file('te/data_inverse_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + tagged = pynini.shortestpath(test_input @ self.tagger).string() + pred = pynini.shortestpath(tagged @ self.verbalizer).string() + assert pred == expected