diff --git a/nemo_text_processing/text_normalization/normalize.py b/nemo_text_processing/text_normalization/normalize.py index d8ebf2f4d..9101ae604 100644 --- a/nemo_text_processing/text_normalization/normalize.py +++ b/nemo_text_processing/text_normalization/normalize.py @@ -166,6 +166,7 @@ def __init__( if post_process: self.post_processor = PostProcessingFst(cache_dir=cache_dir, overwrite_cache=overwrite_cache) + elif lang == 'it': from nemo_text_processing.text_normalization.it.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.text_normalization.it.verbalizers.verbalize_final import VerbalizeFinalFst @@ -191,6 +192,13 @@ def __init__( elif lang == 'ko': from nemo_text_processing.text_normalization.ko.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.text_normalization.ko.verbalizers.verbalize_final import VerbalizeFinalFst + elif lang == 'te': + from nemo_text_processing.text_normalization.te.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.text_normalization.te.verbalizers.post_processing import PostProcessingFst + from nemo_text_processing.text_normalization.te.verbalizers.verbalize_final import VerbalizeFinalFst + + if post_process: + self.post_processor = PostProcessingFst(cache_dir=cache_dir, overwrite_cache=overwrite_cache) else: raise NotImplementedError(f"Language {lang} has not been supported yet.") @@ -391,7 +399,11 @@ def normalize( return text output = SPACE_DUP.sub(' ', output[1:]) - if self.lang in ["en", "hi", "vi"] and hasattr(self, 'post_processor') and self.post_processor is not None: + if ( + self.lang in ["en", "hi", "te", "vi"] + and hasattr(self, 'post_processor') + and self.post_processor is not None + ): output = self.post_process(output) if punct_post_process: @@ -737,7 +749,7 @@ def parse_args(): parser.add_argument( "--language", help="language", - choices=["en", "de", "es", "fr", "hu", "sv", "zh", "ar", "it", "hy", "ja", "hi", "ko", "vi", "pt"], + choices=["en", "de", "es", "fr", "hu", "sv", "zh", "ar", "it", "hy", "ja", "hi", "te", "ko", "vi", "pt"], default="en", type=str, ) diff --git a/nemo_text_processing/text_normalization/run_evaluate.py b/nemo_text_processing/text_normalization/run_evaluate.py index 3a1964bbd..0a5420f55 100644 --- a/nemo_text_processing/text_normalization/run_evaluate.py +++ b/nemo_text_processing/text_normalization/run_evaluate.py @@ -35,7 +35,7 @@ def parse_args(): parser.add_argument( "--lang", help="language", - choices=['ar', 'de', 'en', 'es', 'fr', 'hu', 'it', 'ru', 'sv', 'zh', 'hy', 'hi', 'ko', 'vi', 'pt'], + choices=['ar', 'de', 'en', 'es', 'fr', 'hu', 'it', 'ru', 'sv', 'zh', 'hy', 'hi', 'te', 'ko', 'vi', 'pt'], default="en", type=str, ) @@ -67,6 +67,10 @@ def parse_args(): to_lower = args.input_case == "lower_cased" training_data = load_files([file_path], to_lower=to_lower) + # print("Loading training data: " + file_path) + # to_lower = True if args.input_case == "lower_cased" else False + # training_data = load_files([file_path], to_lower=to_lower) + if args.filter: training_data = filter_loaded_data(training_data) diff --git a/nemo_text_processing/text_normalization/te/__init__.py b/nemo_text_processing/text_normalization/te/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/te/data/numbers/__init__.py b/nemo_text_processing/text_normalization/te/data/numbers/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/te/data/numbers/digit.tsv b/nemo_text_processing/text_normalization/te/data/numbers/digit.tsv new file mode 100644 index 000000000..ab878ec99 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/data/numbers/digit.tsv @@ -0,0 +1,18 @@ +1 ఒకటి +2 రెండు +3 మూడు +4 నాలుగు +5 ఐదు +6 ఆరు +7 ఏడు +8 ఎనిమిది +9 తొమ్మిది +౧ ఒకటి +౨ రెండు +౩ మూడు +౪ నాలుగు +౫ ఐదు +౬ ఆరు +౭ ఏడు +౮ ఎనిమిది +౯ తొమ్మిది \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/te/data/numbers/teens_and_ties.tsv b/nemo_text_processing/text_normalization/te/data/numbers/teens_and_ties.tsv new file mode 100644 index 000000000..4dfef7734 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/data/numbers/teens_and_ties.tsv @@ -0,0 +1,90 @@ +౧౦ పది +౧౧ పదకొండు +౧౨ పన్నెండు +౧౩ పదమూడు +౧౪ పద్నాలుగు +౧౫ పదిహేను +౧౬ పదహారు +౧౭ పదిహేడు +౧౮ పధ్ధెనిమిది +౧౯ పంతొమ్మిది +౨౦ ఇరవై +౨౧ ఇరవై ఒకటి +౨౨ ఇరవై రెండు +౨౩ ఇరవై మూడు +౨౪ ఇరవై నాలుగు +౨౫ ఇరవై ఐదు +౨౬ ఇరవై ఆరు +౨౭ ఇరవై ఏడు +౨౮ ఇరవై ఎనిమిది +౨౯ ఇరవై తొమ్మిది +౩౦ ముప్పై +౩౧ ముప్పై ఒకటి +౩౨ ముప్పై రెండు +౩౩ ముప్పై మూడు +౩౪ ముప్పై నాలుగు +౩౫ ముప్పై ఐదు +౩౬ ముప్పై ఆరు +౩౭ ముప్పై ఏడు +౩౮ ముప్పై ఎనిమిది +౩౯ ముప్పై తొమ్మిది +౪౦ నలభై +౪౧ నలభై ఒకటి +౪౨ నలభై రెండు +౪౩ నలభై మూడు +౪౪ నలభై నాలుగు +౪౫ నలభై ఐదు +౪౬ నలభై ఆరు +౪౭ నలభై ఏడు +౪౮ నలభై ఎనిమిది +౪౯ నలభై తొమ్మిది +౫౦ యాభై +౫౧ యాభై ఒకటి +౫౨ యాభై రెండు +౫౩ యాభై మూడు +౫౪ యాభై నాలుగు +౫౫ యాభై ఐదు +౫౬ యాభై ఆరు +౫౭ యాభై ఏడు +౫౮ యాభై ఎనిమిది +౫౯ యాభై తొమ్మిది +౬౦ అరవై +౬౧ అరవై ఒకటి +౬౨ అరవై రెండు +౬౩ అరవై మూడు +౬౪ అరవై నాలుగు +౬౫ అరవై ఐదు +౬౬ అరవై ఆరు +౬౭ అరవై ఏడు +౬౮ అరవై ఎనిమిది +౬౯ అరవై తొమ్మిది +౭౦ డెబ్బై +౭౧ డెబ్బై ఒకటి +౭౨ డెబ్బై రెండు +౭౩ డెబ్బై మూడు +౭౪ డెబ్బై నాలుగు +౭౫ డెబ్బై ఐదు +౭౬ డెబ్బై ఆరు +౭౭ డెబ్బై ఏడు +౭౮ డెబ్బై ఎనిమిది +౭౯ డెబ్బై తొమ్మిది +౮౦ ఎనభై +౮౧ ఎనభై ఒకటి +౮౨ ఎనభై రెండు +౮౩ ఎనభై మూడు +౮౪ ఎనభై నాలుగు +౮౫ ఎనభై ఐదు +౮౬ ఎనభై ఆరు +౮౭ ఎనభై ఏడు +౮౮ ఎనభై ఎనిమిది +౮౯ ఎనభై తొమ్మిది +౯౦ తొంభై +౯౧ తొంభై ఒకటి +౯౨ తొంభై రెండు +౯౩ తొంభై మూడు +౯౪ తొంభై నాలుగు +౯౫ తొంభై ఐదు +౯౬ తొంభై ఆరు +౯౭ తొంభై ఏడు +౯౮ తొంభై ఎనిమిది +౯౯ తొంభై తొమ్మిది diff --git a/nemo_text_processing/text_normalization/te/data/numbers/teens_and_ties_en.tsv b/nemo_text_processing/text_normalization/te/data/numbers/teens_and_ties_en.tsv new file mode 100644 index 000000000..ffb621cf6 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/data/numbers/teens_and_ties_en.tsv @@ -0,0 +1,90 @@ +10 పది +11 పదకొండు +12 పన్నెండు +13 పదమూడు +14 పద్నాలుగు +15 పదిహేను +16 పదహారు +17 పదిహేడు +18 పధ్ధెనిమిది +19 పంతొమ్మిది +20 ఇరవై +21 ఇరవై ఒకటి +22 ఇరవై రెండు +23 ఇరవై మూడు +24 ఇరవై నాలుగు +25 ఇరవై ఐదు +26 ఇరవై ఆరు +27 ఇరవై ఏడు +28 ఇరవై ఎనిమిది +29 ఇరవై తొమ్మిది +30 ముప్పై +31 ముప్పై ఒకటి +32 ముప్పై రెండు +33 ముప్పై మూడు +34 ముప్పై నాలుగు +35 ముప్పై ఐదు +36 ముప్పై ఆరు +37 ముప్పై ఏడు +38 ముప్పై ఎనిమిది +39 ముప్పై తొమ్మిది +40 నలభై +41 నలభై ఒకటి +42 నలభై రెండు +43 నలభై మూడు +44 నలభై నాలుగు +45 నలభై ఐదు +46 నలభై ఆరు +47 నలభై ఏడు +48 నలభై ఎనిమిది +49 నలభై తొమ్మిది +50 యాభై +51 యాభై ఒకటి +52 యాభై రెండు +53 యాభై మూడు +54 యాభై నాలుగు +55 యాభై ఐదు +56 యాభై ఆరు +57 యాభై ఏడు +58 యాభై ఎనిమిది +59 యాభై తొమ్మిది +60 అరవై +61 అరవై ఒకటి +62 అరవై రెండు +63 అరవై మూడు +64 అరవై నాలుగు +65 అరవై ఐదు +66 అరవై ఆరు +67 అరవై ఏడు +68 అరవై ఎనిమిది +69 అరవై తొమ్మిది +70 డెబ్బై +71 డెబ్బై ఒకటి +72 డెబ్బై రెండు +73 డెబ్బై మూడు +74 డెబ్బై నాలుగు +75 డెబ్బై ఐదు +76 డెబ్బై ఆరు +77 డెబ్బై ఏడు +78 డెబ్బై ఎనిమిది +79 డెబ్బై తొమ్మిది +80 ఎనభై +81 ఎనభై ఒకటి +82 ఎనభై రెండు +83 ఎనభై మూడు +84 ఎనభై నాలుగు +85 ఎనభై ఐదు +86 ఎనభై ఆరు +87 ఎనభై ఏడు +88 ఎనభై ఎనిమిది +89 ఎనభై తొమ్మిది +90 తొంభై +91 తొంభై ఒకటి +92 తొంభై రెండు +93 తొంభై మూడు +94 తొంభై నాలుగు +95 తొంభై ఐదు +96 తొంభై ఆరు +97 తొంభై ఏడు +98 తొంభై ఎనిమిది +99 తొంభై తొమ్మిది \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/te/data/numbers/teens_and_ties_thousand.tsv b/nemo_text_processing/text_normalization/te/data/numbers/teens_and_ties_thousand.tsv new file mode 100644 index 000000000..a6bcff46b --- /dev/null +++ b/nemo_text_processing/text_normalization/te/data/numbers/teens_and_ties_thousand.tsv @@ -0,0 +1,16 @@ +21 ఇరవై ఒక +31 ముప్పై ఒక +41 నలభై ఒక +51 యాభై ఒక +61 అరవై ఒక +71 డెబ్బై ఒక +81 ఎనభై ఒక +91 తొంభై ఒక +౨౧ ఇరవై ఒక +౩౧ ముప్పై ఒక +౪౧ నలభై ఒక +౫౧ యాభై ఒక +౬౧ అరవై ఒక +౭౧ డెబ్బై ఒక +౮౧ ఎనభై ఒక +౯౧ తొంభై ఒక \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/te/data/numbers/zero.tsv b/nemo_text_processing/text_normalization/te/data/numbers/zero.tsv new file mode 100644 index 000000000..309aab75e --- /dev/null +++ b/nemo_text_processing/text_normalization/te/data/numbers/zero.tsv @@ -0,0 +1,2 @@ +0 సున్నా +౦ సున్నా \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/te/graph_utils.py b/nemo_text_processing/text_normalization/te/graph_utils.py new file mode 100644 index 000000000..e9a0e2eb7 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/graph_utils.py @@ -0,0 +1,225 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import string +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.export import export +from pynini.lib import byte, pynutil, utf8 + +NEMO_CHAR = utf8.VALID_UTF8_CHAR +NEMO_DIGIT = byte.DIGIT + +# Telugu native digits (౦, ౧, ౨, ౩, ౪, ౫, ౬, ౭, ౮, ౯) +NEMO_TE_DIGIT = pynini.union("౦", "౧", "౨", "౩", "౪", "౫", "౬", "౭", "౮", "౯").optimize() +NEMO_TE_NON_ZERO = pynini.union("౧", "౨", "౩", "౪", "౫", "౬", "౭", "౮", "౯").optimize() +NEMO_TE_ZERO = "౦" + +# Combined Telugu and Arabic digits for graphs that need to accept both +NEMO_ALL_DIGIT = pynini.union(NEMO_TE_DIGIT, NEMO_DIGIT).optimize() +NEMO_ALL_ZERO = pynini.union("౦", "0").optimize() +NEMO_ALL_NON_ZERO = pynini.union(NEMO_TE_NON_ZERO, "1", "2", "3", "4", "5", "6", "7", "8", "9").optimize() + +# Telugu fractional terms +TE_DEDH = "ఒకటిన్నర" +TE_DHAI = "రెండున్నర" +TE_HALF_WORD = "అర" +TE_QUARTER_WORD = "పావు" +TE_THREE_QUARTERS_WORD = "ముప్పావు" + +# Telugu native decimal representations +TE_POINT_FIVE = ".౫" # .5 +TE_ONE_POINT_FIVE = "౧.౫" # 1.5 +TE_TWO_POINT_FIVE = "౨.౫" # 2.5 +TE_DECIMAL_25 = ".౨౫" # .25 +TE_DECIMAL_75 = ".౭౫" # .75 + +# Arabic/English decimal representations +EN_POINT_FIVE = ".5" +EN_ONE_POINT_FIVE = "1.5" +EN_TWO_POINT_FIVE = "2.5" +EN_DECIMAL_25 = ".25" +EN_DECIMAL_75 = ".75" + +# Combined Telugu and English decimal patterns +POINT_FIVE = pynini.union(TE_POINT_FIVE, EN_POINT_FIVE).optimize() +ONE_POINT_FIVE = pynini.union(TE_ONE_POINT_FIVE, EN_ONE_POINT_FIVE).optimize() +TWO_POINT_FIVE = pynini.union(TE_TWO_POINT_FIVE, EN_TWO_POINT_FIVE).optimize() +DECIMAL_25 = pynini.union(TE_DECIMAL_25, EN_DECIMAL_25).optimize() +DECIMAL_75 = pynini.union(TE_DECIMAL_75, EN_DECIMAL_75).optimize() + +# Symbol constants +TE_BY = "బై" # Transliterated 'by' for fractions/ratios (e.g., 2/3 -> రెండు బై మూడు) +LOWERCASE_X = "x" +UPPERCASE_X = "X" +ASTERISK = "*" +HYPHEN = "-" +SLASH = "/" +COMMA = "," +PERIOD = "." +TE_PERIOD = "." # Telugu text uses standard Western periods + +NEMO_LOWER = pynini.union(*string.ascii_lowercase).optimize() +NEMO_UPPER = pynini.union(*string.ascii_uppercase).optimize() +NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize() +NEMO_HEX = pynini.union(*string.hexdigits).optimize() +NEMO_NON_BREAKING_SPACE = u"\u00a0" +NEMO_SPACE = " " +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", u"\u00a0").optimize() +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() +TO_LOWER = pynini.union(*[pynini.cross(x, y) for x, y in zip(string.ascii_uppercase, string.ascii_lowercase)]) +TO_UPPER = pynini.invert(TO_LOWER) +NEMO_SIGMA = pynini.closure(NEMO_CHAR) + +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +delete_zero_or_one_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE, 0, 1)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") +delete_preserve_order = pynini.closure( + pynutil.delete(" preserve_order: true") + | (pynutil.delete(" field_order: \"") + NEMO_NOT_QUOTE + pynutil.delete("\"")) +) + +MIN_NEG_WEIGHT = -0.0001 +MIN_POS_WEIGHT = 0.0001 +INPUT_CASED = "cased" +INPUT_LOWER_CASED = "lower_cased" +MINUS = pynini.union(" మైనస్ ", " ఋణ ").optimize() # 'మైనస్' (Minus) is highly preferred in modern ITN + + +def capitalized_input_graph( + graph: 'pynini.FstLike', original_graph_weight: float = None, capitalized_graph_weight: float = None +) -> 'pynini.FstLike': + """ + Allow graph input to be capitalized, e.g. for ITN) + Args: + graph: FstGraph + original_graph_weight: weight to add to the original `graph` + capitalized_graph_weight: weight to add to the capitalized graph + """ + capitalized_graph = pynini.compose(TO_LOWER + NEMO_SIGMA, graph).optimize() + + if original_graph_weight is not None: + graph = pynutil.add_weight(graph, weight=original_graph_weight) + + if capitalized_graph_weight is not None: + capitalized_graph = pynutil.add_weight(capitalized_graph, weight=capitalized_graph_weight) + + graph |= capitalized_graph + return graph + + +def generator_main(file_name: str, graphs: Dict[str, 'pynini.FstLike']): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logging.info(f'Created {file_name}') + + +def convert_space(fst) -> 'pynini.FstLike': + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA) + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + '/grammars/' + kind + '/' + name + '.far') + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> 'pynini.FstLike': + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> 'pynini.FstLike': + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> 'pynini.FstLike': + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/text_normalization/te/taggers/__init__.py b/nemo_text_processing/text_normalization/te/taggers/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/te/taggers/cardinal.py b/nemo_text_processing/text_normalization/te/taggers/cardinal.py new file mode 100644 index 000000000..caee8889f --- /dev/null +++ b/nemo_text_processing/text_normalization/te/taggers/cardinal.py @@ -0,0 +1,1404 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.te.graph_utils import ( + NEMO_ALL_DIGIT, + NEMO_ALL_ZERO, + GraphFst, + insert_space, +) +from nemo_text_processing.text_normalization.te.utils import get_abs_path + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals, e.g. + -౨౩ -> cardinal { negative: "true" integer: "ఇరవై మూడు" } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True, lm: bool = False): + super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + + digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + teens_ties_te = pynini.string_file(get_abs_path("data/numbers/teens_and_ties.tsv")) + teens_ties_en = pynini.string_file(get_abs_path("data/numbers/teens_and_ties_en.tsv")) + teens_ties_thousand = pynutil.add_weight( + pynini.string_file(get_abs_path("data/numbers/teens_and_ties_thousand.tsv")), + -0.2, + ) + teens_ties = pynini.union(teens_ties_te, teens_ties_en) + teens_and_ties = pynutil.add_weight(teens_ties, -0.1) + + self.digit = digit + self.zero = zero + self.teens_and_ties = teens_and_ties + + # Single digit graph for digit-by-digit reading + single_digit_graph = digit | zero + self.single_digits_graph = single_digit_graph + pynini.closure(insert_space + single_digit_graph) + + def create_graph_suffix(digit_graph, suffix, zeros_counts): + zero = pynutil.add_weight(pynutil.delete(NEMO_ALL_ZERO | pynini.accep("౦")), -0.1) + if zeros_counts == 0: + return digit_graph + suffix + + return digit_graph + (zero**zeros_counts) + suffix + + def create_larger_number_graph(digit_graph, suffix, zeros_counts, sub_graph): + insert_space = pynutil.insert(" ") + zero = pynutil.add_weight(pynutil.delete(NEMO_ALL_ZERO | pynini.accep("౦")), -0.1) + if zeros_counts == 0: + return digit_graph + suffix + insert_space + sub_graph + + return digit_graph + suffix + (zero**zeros_counts) + insert_space + sub_graph + + # Hundred graph + + suffix_hundreds = pynutil.insert(" వంద") + + digit_except_one = ( + pynini.union("2", "3", "4", "5", "6", "7", "8", "9", "౨", "౩", "౪", "౫", "౬", "౭", "౮", "౯") @ digit + ).optimize() + + graph_hundreds = pynini.cross("100", "వంద") | pynini.cross("౧౦౦", "వంద") + graph_hundreds |= (pynini.cross("10", "నూట ") | pynini.cross("౧౦", "నూట ")) + digit + graph_hundreds |= (pynini.cross("1", "నూట ") | pynini.cross("౧", "నూట ")) + teens_ties + + graph_hundreds |= create_graph_suffix(digit_except_one, pynutil.insert(" వందలు"), 2) + graph_hundreds |= create_larger_number_graph(digit_except_one, pynutil.insert(" వందల"), 1, digit) + graph_hundreds |= create_larger_number_graph(digit_except_one, pynutil.insert(" వందల"), 0, teens_ties) + + graph_hundreds = graph_hundreds.optimize() + self.graph_hundreds = graph_hundreds + + # Transducer for eleven hundred -> 1100 or twenty one hundred eleven -> 2111 + graph_hundreds_as_thousand = create_graph_suffix(teens_and_ties, suffix_hundreds, 2) + graph_hundreds_as_thousand |= create_larger_number_graph(teens_and_ties, suffix_hundreds, 1, digit) + graph_hundreds_as_thousand |= create_larger_number_graph(teens_and_ties, suffix_hundreds, 0, teens_ties) + self.graph_hundreds_as_thousand = graph_hundreds_as_thousand + + # Thousands and Ten thousands graph + one_thousand_prefix = pynutil.delete(pynini.union("1", "౧")) + digit_except_one = ( + pynini.union("2", "3", "4", "5", "6", "7", "8", "9", "౨", "౩", "౪", "౫", "౬", "౭", "౮", "౯") @ digit + ).optimize() + + suffix_thousand_exact = pynutil.insert("వెయ్యి") + suffix_thousand_plural = pynutil.insert(" వేలు") + suffix_thousand_before = pynutil.insert(" వేల") + + # 1000 + graph_thousands = pynini.cross("1000", "వెయ్యి") + graph_thousands |= pynini.cross("౧౦౦౦", "వెయ్యి") + + graph_thousands |= create_larger_number_graph( + one_thousand_prefix, + suffix_thousand_exact, + 2, + digit, + ) # 1001–1009 + graph_thousands |= create_larger_number_graph( + one_thousand_prefix, + suffix_thousand_exact, + 1, + teens_ties, + ) # 1010–1099 + graph_thousands |= create_larger_number_graph( + one_thousand_prefix, + suffix_thousand_exact, + 0, + graph_hundreds, + ) # 1100–1999 + graph_thousands |= create_graph_suffix( + digit_except_one, + suffix_thousand_plural, + 3, + ) # 2000–9000 + graph_thousands |= create_larger_number_graph( + digit_except_one, + suffix_thousand_before, + 2, + digit, + ) # 2001–2009, 3001–9009 + graph_thousands |= create_larger_number_graph( + digit_except_one, + suffix_thousand_before, + 1, + teens_ties, + ) # 2010–2099, 3010–9099 + graph_thousands |= create_larger_number_graph( + digit_except_one, + suffix_thousand_before, + 0, + graph_hundreds, + ) # 2100–9999 + + graph_thousands = graph_thousands.optimize() + self.graph_thousands = graph_thousands + + # Ten thousands graph + suffix_ten_thousand_exact = pynutil.insert(" వెయ్యి") + suffix_ten_thousand_plural = pynutil.insert(" వేలు") + suffix_ten_thousand_before = pynutil.insert(" వేల") + + graph_ten_thousands = create_graph_suffix( + teens_ties_thousand, + suffix_ten_thousand_exact, + 3, + ) # 21000, 31000, 41000... + graph_ten_thousands |= create_larger_number_graph( + teens_ties_thousand, + suffix_ten_thousand_before, + 2, + digit, + ) # 21001, 31001... + graph_ten_thousands |= create_larger_number_graph( + teens_ties_thousand, + suffix_ten_thousand_before, + 1, + teens_ties, + ) # 21010, 41010... + graph_ten_thousands |= create_larger_number_graph( + teens_ties_thousand, + suffix_ten_thousand_before, + 0, + graph_hundreds, + ) # 21100, 91234... + graph_ten_thousands |= create_graph_suffix( + teens_and_ties, + suffix_ten_thousand_plural, + 3, + ) # 10000, 11000, 12000, 19000, 20000, 22000... + graph_ten_thousands |= create_larger_number_graph( + teens_and_ties, + suffix_ten_thousand_before, + 2, + digit, + ) # 10001, 22001... + graph_ten_thousands |= create_larger_number_graph( + teens_and_ties, + suffix_ten_thousand_before, + 1, + teens_ties, + ) # 10010, 23010... + graph_ten_thousands |= create_larger_number_graph( + teens_and_ties, + suffix_ten_thousand_before, + 0, + graph_hundreds, + ) # 10100, 99999... + + graph_ten_thousands = graph_ten_thousands.optimize() + self.graph_ten_thousands = graph_ten_thousands + + # Lakhs graph and ten lakhs graph + one_lakh_prefix = pynutil.delete(pynini.union("1", "౧")) + + suffix_lakh_exact = pynutil.insert("లక్ష") + suffix_lakha_digit = pynutil.insert("లక్షా") + suffix_lakh_plural = pynutil.insert(" లక్షలు") + suffix_lakh_before = pynutil.insert(" లక్షల") + + # 100000 + graph_lakhs = pynini.cross("100000", "లక్ష") + graph_lakhs |= pynini.cross("౧౦౦౦౦౦", "లక్ష") + + graph_lakhs |= create_larger_number_graph( + one_lakh_prefix, + suffix_lakha_digit, + 4, + digit, + ) # 100001–100009 -> లక్షా ఒకటి + graph_lakhs |= create_larger_number_graph( + one_lakh_prefix, + suffix_lakh_exact, + 3, + teens_ties, + ) # 100010–100099 -> లక్ష పది + graph_lakhs |= create_larger_number_graph( + one_lakh_prefix, + suffix_lakh_exact, + 2, + graph_hundreds, + ) # 100100–100999 -> లక్ష వంద + graph_lakhs |= create_larger_number_graph( + one_lakh_prefix, + suffix_lakh_exact, + 1, + graph_thousands, + ) # 101000–109999 -> లక్ష వెయ్యి + graph_lakhs |= create_larger_number_graph( + one_lakh_prefix, + suffix_lakh_exact, + 0, + graph_ten_thousands, + ) # 110000–199999 -> లక్ష పది వేలు + graph_lakhs |= create_graph_suffix( + digit_except_one, + suffix_lakh_plural, + 5, + ) # 200000–900000 -> రెండు లక్షలు + graph_lakhs |= create_larger_number_graph( + digit_except_one, + suffix_lakh_before, + 4, + digit, + ) # 200001–200009 -> రెండు లక్షల ఒకటి + graph_lakhs |= create_larger_number_graph( + digit_except_one, + suffix_lakh_before, + 3, + teens_ties, + ) # 200010–200099 + graph_lakhs |= create_larger_number_graph( + digit_except_one, + suffix_lakh_before, + 2, + graph_hundreds, + ) # 200100–200999 + graph_lakhs |= create_larger_number_graph( + digit_except_one, + suffix_lakh_before, + 1, + graph_thousands, + ) # 201000–209999 + graph_lakhs |= create_larger_number_graph( + digit_except_one, + suffix_lakh_before, + 0, + graph_ten_thousands, + ) # 210000–999999 + graph_lakhs = graph_lakhs.optimize() + self.graph_lakhs = graph_lakhs + + # Ten lakhs graph + suffix_ten_lakh_exact = pynutil.insert(" లక్ష") + suffix_ten_lakh_plural = pynutil.insert(" లక్షలు") + suffix_ten_lakh_before = pynutil.insert(" లక్షల") + + graph_ten_lakhs = create_graph_suffix( + teens_ties_thousand, + suffix_ten_lakh_exact, + 5, + ) # 2100000, 3100000, ... -> ఇరవై ఒక లక్ష + graph_ten_lakhs |= create_larger_number_graph( + teens_ties_thousand, + suffix_ten_lakh_before, + 4, + digit, + ) # 2100001, 3100001, ... -> ఇరవై ఒక లక్షల ఒకటి + graph_ten_lakhs |= create_larger_number_graph( + teens_ties_thousand, + suffix_ten_lakh_before, + 3, + teens_ties, + ) + graph_ten_lakhs |= create_larger_number_graph( + teens_ties_thousand, + suffix_ten_lakh_before, + 2, + graph_hundreds, + ) + graph_ten_lakhs |= create_larger_number_graph( + teens_ties_thousand, + suffix_ten_lakh_before, + 1, + graph_thousands, + ) + graph_ten_lakhs |= create_larger_number_graph( + teens_ties_thousand, + suffix_ten_lakh_before, + 0, + graph_ten_thousands, + ) + graph_ten_lakhs |= create_graph_suffix( + teens_and_ties, + suffix_ten_lakh_plural, + 5, + ) # 1000000, 1100000, 2200000, 9900000 -> పది లక్షలు + graph_ten_lakhs |= create_larger_number_graph( + teens_and_ties, + suffix_ten_lakh_before, + 4, + digit, + ) # 1000001, 2200001 + graph_ten_lakhs |= create_larger_number_graph( + teens_and_ties, + suffix_ten_lakh_before, + 3, + teens_ties, + ) + graph_ten_lakhs |= create_larger_number_graph( + teens_and_ties, + suffix_ten_lakh_before, + 2, + graph_hundreds, + ) + graph_ten_lakhs |= create_larger_number_graph( + teens_and_ties, + suffix_ten_lakh_before, + 1, + graph_thousands, + ) + graph_ten_lakhs |= create_larger_number_graph( + teens_and_ties, + suffix_ten_lakh_before, + 0, + graph_ten_thousands, + ) + + graph_ten_lakhs = graph_ten_lakhs.optimize() + self.graph_ten_lakhs = graph_ten_lakhs + + # Crores graph & ten crores graph + one_crore_prefix = pynutil.delete(pynini.union("1", "౧")) + + suffix_crore_exact = pynutil.insert("కోటి") + suffix_crore_plural = pynutil.insert(" కోట్లు") + suffix_crore_before = pynutil.insert(" కోట్ల") + + # 10000000 + graph_crores = pynini.cross("10000000", "కోటి") + graph_crores |= pynini.cross("౧౦౦౦౦౦౦౦", "కోటి") + + graph_crores |= create_larger_number_graph( + one_crore_prefix, + suffix_crore_exact, + 6, + digit, + ) # 10000001–10000009 + graph_crores |= create_larger_number_graph( + one_crore_prefix, + suffix_crore_exact, + 5, + teens_ties, + ) # 10000010–10000099 + graph_crores |= create_larger_number_graph( + one_crore_prefix, + suffix_crore_exact, + 4, + graph_hundreds, + ) # 10000100–10000999 + graph_crores |= create_larger_number_graph( + one_crore_prefix, + suffix_crore_exact, + 3, + graph_thousands, + ) # 10001000–10009999 + graph_crores |= create_larger_number_graph( + one_crore_prefix, + suffix_crore_exact, + 2, + graph_ten_thousands, + ) # 10010000–10099999 + graph_crores |= create_larger_number_graph( + one_crore_prefix, + suffix_crore_exact, + 1, + graph_lakhs, + ) # 10100000–10999999 + graph_crores |= create_larger_number_graph( + one_crore_prefix, + suffix_crore_exact, + 0, + graph_ten_lakhs, + ) # 11000000–19999999 + graph_crores |= create_graph_suffix( + digit_except_one, + suffix_crore_plural, + 7, + ) # 20000000–90000000 + graph_crores |= create_larger_number_graph( + digit_except_one, + suffix_crore_before, + 6, + digit, + ) # 20000001–90000009 + graph_crores |= create_larger_number_graph( + digit_except_one, + suffix_crore_before, + 5, + teens_ties, + ) # 20000010–90000099 + graph_crores |= create_larger_number_graph( + digit_except_one, + suffix_crore_before, + 4, + graph_hundreds, + ) # 20000100–90000999 + graph_crores |= create_larger_number_graph( + digit_except_one, + suffix_crore_before, + 3, + graph_thousands, + ) # 20001000–90009999 + graph_crores |= create_larger_number_graph( + digit_except_one, + suffix_crore_before, + 2, + graph_ten_thousands, + ) # 20010000–90099999 + graph_crores |= create_larger_number_graph( + digit_except_one, + suffix_crore_before, + 1, + graph_lakhs, + ) # 20100000–90999999 + graph_crores |= create_larger_number_graph( + digit_except_one, + suffix_crore_before, + 0, + graph_ten_lakhs, + ) # 21000000–99999999 + + graph_crores = graph_crores.optimize() + + # Ten crores graph + suffix_ten_crore_exact = pynutil.insert(" కోటి") + suffix_ten_crore_plural = pynutil.insert(" కోట్లు") + suffix_ten_crore_before = pynutil.insert(" కోట్ల") + + graph_ten_crores = create_graph_suffix( + teens_ties_thousand, + suffix_ten_crore_exact, + 7, + ) # 210000000, 310000000, ... -> ఇరవై ఒక కోటి + graph_ten_crores |= create_larger_number_graph( + teens_ties_thousand, + suffix_ten_crore_exact, + 6, + digit, + ) # 210000001, 310000001, ... + graph_ten_crores |= create_larger_number_graph( + teens_ties_thousand, + suffix_ten_crore_exact, + 5, + teens_ties, + ) + graph_ten_crores |= create_larger_number_graph( + teens_ties_thousand, + suffix_ten_crore_exact, + 4, + graph_hundreds, + ) + graph_ten_crores |= create_larger_number_graph( + teens_ties_thousand, + suffix_ten_crore_exact, + 3, + graph_thousands, + ) + graph_ten_crores |= create_larger_number_graph( + teens_ties_thousand, + suffix_ten_crore_exact, + 2, + graph_ten_thousands, + ) + graph_ten_crores |= create_larger_number_graph( + teens_ties_thousand, + suffix_ten_crore_exact, + 1, + graph_lakhs, + ) + graph_ten_crores |= create_larger_number_graph( + teens_ties_thousand, + suffix_ten_crore_exact, + 0, + graph_ten_lakhs, + ) + graph_ten_crores |= create_graph_suffix( + teens_and_ties, + suffix_ten_crore_plural, + 7, + ) # 100000000, 110000000, 220000000, 990000000 + graph_ten_crores |= create_larger_number_graph( + teens_and_ties, + suffix_ten_crore_before, + 6, + digit, + ) + graph_ten_crores |= create_larger_number_graph( + teens_and_ties, + suffix_ten_crore_before, + 5, + teens_ties, + ) + graph_ten_crores |= create_larger_number_graph( + teens_and_ties, + suffix_ten_crore_before, + 4, + graph_hundreds, + ) + graph_ten_crores |= create_larger_number_graph( + teens_and_ties, + suffix_ten_crore_before, + 3, + graph_thousands, + ) + graph_ten_crores |= create_larger_number_graph( + teens_and_ties, + suffix_ten_crore_before, + 2, + graph_ten_thousands, + ) + graph_ten_crores |= create_larger_number_graph( + teens_and_ties, + suffix_ten_crore_before, + 1, + graph_lakhs, + ) + graph_ten_crores |= create_larger_number_graph( + teens_and_ties, + suffix_ten_crore_before, + 0, + graph_ten_lakhs, + ) + + graph_ten_crores = graph_ten_crores.optimize() + + # Arabs graph as hundreds of crores + suffix_hundred_crores_plural = pynutil.insert(" కోట్లు") + suffix_hundred_crores_before = pynutil.insert(" కోట్ల") + suffix_hundred_crores_one = pynutil.insert(" కోటి") + + # 100,110,123,200,234 ... crore prefixes + + hundred_crore_prefix = ( + pynini.cross("100", "వంద") + | pynini.cross("౧౦౦", "వంద") + | ((pynini.cross("10", "నూట ") | pynini.cross("౧౦", "నూట ")) + digit) + | ((pynini.cross("1", "నూట ") | pynini.cross("౧", "నూట ")) + teens_ties) + | create_graph_suffix( + digit_except_one, + pynutil.insert(" వందల"), + 2, + ) + | create_larger_number_graph( + digit_except_one, + pynutil.insert(" వందల"), + 1, + digit_except_one, + ) + | create_larger_number_graph( + digit_except_one, + pynutil.insert(" వందల"), + 0, + teens_ties, + ) + ).optimize() + + # 201,301,401...901 crore prefixes రెండు వందల ఒక + hundred_one_crore_prefix = pynutil.add_weight( + digit_except_one + + pynutil.delete(NEMO_ALL_ZERO | pynini.accep("౦")) + + (pynini.cross("1", " వందల ఒక") | pynini.cross("౧", " వందల ఒక")), + -0.3, + ) + # 2010000000 -> రెండు వందల ఒక కోటి + graph_arabs = create_graph_suffix( + hundred_one_crore_prefix, + suffix_hundred_crores_one, + 7, + ) + graph_arabs |= create_larger_number_graph( + hundred_one_crore_prefix, + suffix_hundred_crores_one, + 6, + digit, + ) + graph_arabs |= create_larger_number_graph( + hundred_one_crore_prefix, + suffix_hundred_crores_one, + 5, + teens_ties, + ) + graph_arabs |= create_larger_number_graph( + hundred_one_crore_prefix, + suffix_hundred_crores_one, + 4, + graph_hundreds, + ) + graph_arabs |= create_larger_number_graph( + hundred_one_crore_prefix, + suffix_hundred_crores_one, + 3, + graph_thousands, + ) + graph_arabs |= create_larger_number_graph( + hundred_one_crore_prefix, + suffix_hundred_crores_one, + 2, + graph_ten_thousands, + ) + graph_arabs |= create_larger_number_graph( + hundred_one_crore_prefix, + suffix_hundred_crores_one, + 1, + graph_lakhs, + ) + graph_arabs |= create_larger_number_graph( + hundred_one_crore_prefix, + suffix_hundred_crores_one, + 0, + graph_ten_lakhs, + ) + + # Normal: + # 100 కోట్లు + # 110 కోట్లు + # 123 కోట్లు + # 200 కోట్లు + graph_arabs |= create_graph_suffix( + hundred_crore_prefix, + suffix_hundred_crores_plural, + 7, + ) + graph_arabs |= create_larger_number_graph( + hundred_crore_prefix, + suffix_hundred_crores_before, + 6, + digit, + ) + graph_arabs |= create_larger_number_graph( + hundred_crore_prefix, + suffix_hundred_crores_before, + 5, + teens_ties, + ) + graph_arabs |= create_larger_number_graph( + hundred_crore_prefix, + suffix_hundred_crores_before, + 4, + graph_hundreds, + ) + graph_arabs |= create_larger_number_graph( + hundred_crore_prefix, + suffix_hundred_crores_before, + 3, + graph_thousands, + ) + graph_arabs |= create_larger_number_graph( + hundred_crore_prefix, + suffix_hundred_crores_before, + 2, + graph_ten_thousands, + ) + graph_arabs |= create_larger_number_graph( + hundred_crore_prefix, + suffix_hundred_crores_before, + 1, + graph_lakhs, + ) + graph_arabs |= create_larger_number_graph( + hundred_crore_prefix, + suffix_hundred_crores_before, + 0, + graph_ten_lakhs, + ) + graph_arabs |= create_larger_number_graph( + hundred_crore_prefix, + suffix_hundred_crores_before, + 0, + graph_crores, + ) + + graph_arabs = graph_arabs.optimize() + graph_ten_arabs = graph_arabs + # Ten Arabs graph as thousands of crores + # 10000000000 -> వెయ్యి కోట్లు + # 20000000000 -> రెండు వేల కోట్లు + # 999999999999 -> తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది కోట్లు ... + + suffix_thousand_crores_plural = pynutil.insert(" కోట్లు") + suffix_thousand_crores_before = pynutil.insert(" కోట్ల") + + thousand_crore_prefix = ( + pynini.cross("1000", "వెయ్యి") + | pynini.cross("౧౦౦౦", "వెయ్యి") + | create_graph_suffix(digit_except_one, pynutil.insert(" వేల"), 3) + | create_larger_number_graph(digit_except_one, pynutil.insert(" వేల"), 2, digit) + | create_larger_number_graph(digit_except_one, pynutil.insert(" వేల"), 1, teens_ties) + | create_larger_number_graph(digit_except_one, pynutil.insert(" వేల"), 0, hundred_crore_prefix) + | create_larger_number_graph(one_thousand_prefix, pynutil.insert("వెయ్యి"), 2, digit) + | create_larger_number_graph(one_thousand_prefix, pynutil.insert("వెయ్యి"), 1, teens_ties) + | create_larger_number_graph(one_thousand_prefix, pynutil.insert("వెయ్యి"), 0, hundred_crore_prefix) + ).optimize() + + ten_thousand_crore_prefix = ( + create_graph_suffix(teens_ties_thousand, pynutil.insert(" వెయ్యి"), 3) + | create_larger_number_graph(teens_ties_thousand, pynutil.insert(" వేల"), 2, digit) + | create_larger_number_graph(teens_ties_thousand, pynutil.insert(" వేల"), 1, teens_ties) + | create_larger_number_graph(teens_ties_thousand, pynutil.insert(" వేల"), 0, hundred_crore_prefix) + | create_graph_suffix(teens_and_ties, pynutil.insert(" వేల"), 3) + | create_larger_number_graph(teens_and_ties, pynutil.insert(" వేల"), 2, digit) + | create_larger_number_graph(teens_and_ties, pynutil.insert(" వేల"), 1, teens_ties) + | create_larger_number_graph(teens_and_ties, pynutil.insert(" వేల"), 0, hundred_crore_prefix) + ).optimize() + + crore_count_prefix = thousand_crore_prefix | ten_thousand_crore_prefix + + graph_ten_arabs = create_graph_suffix( + crore_count_prefix, + suffix_thousand_crores_plural, + 7, + ) + graph_ten_arabs |= create_larger_number_graph( + crore_count_prefix, + suffix_thousand_crores_before, + 6, + digit, + ) + graph_ten_arabs |= create_larger_number_graph( + crore_count_prefix, + suffix_thousand_crores_before, + 5, + teens_ties, + ) + graph_ten_arabs |= create_larger_number_graph( + crore_count_prefix, + suffix_thousand_crores_before, + 4, + graph_hundreds, + ) + graph_ten_arabs |= create_larger_number_graph( + crore_count_prefix, + suffix_thousand_crores_before, + 3, + graph_thousands, + ) + graph_ten_arabs |= create_larger_number_graph( + crore_count_prefix, + suffix_thousand_crores_before, + 2, + graph_ten_thousands, + ) + graph_ten_arabs |= create_larger_number_graph( + crore_count_prefix, + suffix_thousand_crores_before, + 1, + graph_lakhs, + ) + graph_ten_arabs |= create_larger_number_graph( + crore_count_prefix, + suffix_thousand_crores_before, + 0, + graph_ten_lakhs, + ) + + graph_ten_arabs = pynutil.add_weight( + graph_ten_arabs, + -1.0, + ).optimize() + + # Kharabs graph as ten-thousand crores + # 100000000000 -> పది వేల కోట్లు + # 210000000000 -> ఇరవై ఒక వెయ్యి కోట్లు + # 999999999999 -> తొంభై తొమ్మిది వేల తొమ్మిది వందల తొంభై తొమ్మిది కోట్లు ... + + suffix_kharab_crores_plural = pynutil.insert(" కోట్లు") + suffix_kharab_crores_before = pynutil.insert(" కోట్ల") + + kharab_crore_prefix = ten_thousand_crore_prefix + graph_kharabs = create_graph_suffix( + kharab_crore_prefix, + suffix_kharab_crores_plural, + 7, + ) + graph_kharabs |= create_larger_number_graph( + kharab_crore_prefix, + suffix_kharab_crores_before, + 6, + digit, + ) + graph_kharabs |= create_larger_number_graph( + kharab_crore_prefix, + suffix_kharab_crores_before, + 5, + teens_ties, + ) + graph_kharabs |= create_larger_number_graph( + kharab_crore_prefix, + suffix_kharab_crores_before, + 4, + graph_hundreds, + ) + graph_kharabs |= create_larger_number_graph( + kharab_crore_prefix, + suffix_kharab_crores_before, + 3, + graph_thousands, + ) + graph_kharabs |= create_larger_number_graph( + kharab_crore_prefix, + suffix_kharab_crores_before, + 2, + graph_ten_thousands, + ) + graph_kharabs |= create_larger_number_graph( + kharab_crore_prefix, + suffix_kharab_crores_before, + 1, + graph_lakhs, + ) + graph_kharabs |= create_larger_number_graph( + kharab_crore_prefix, + suffix_kharab_crores_before, + 0, + graph_ten_lakhs, + ) + + graph_kharabs = pynutil.add_weight( + graph_kharabs, + -1.2, + ).optimize() + graph_ten_kharabs = graph_kharabs + + # Ten Kharabs graph as lakh-crores + # 1000000000000 -> లక్ష కోట్లు + # 2000000000000 -> రెండు లక్షల కోట్లు + # 21000000000000 -> ఇరవై ఒక లక్ష కోట్లు + + suffix_lakh_crores_plural = pynutil.insert(" కోట్లు") + suffix_lakh_crores_before = pynutil.insert(" కోట్ల") + + lakh_crore_prefix = ( + # 100000 crores -> లక్ష + pynini.cross("100000", "లక్ష") + | pynini.cross("౧౦౦౦౦౦", "లక్ష") + # లక్ష + remainder + | create_larger_number_graph( + one_lakh_prefix, + suffix_lakh_exact, + 4, + digit, + ) + | create_larger_number_graph( + one_lakh_prefix, + suffix_lakh_exact, + 3, + teens_ties, + ) + | create_larger_number_graph( + one_lakh_prefix, + suffix_lakh_exact, + 2, + graph_hundreds, + ) + | create_larger_number_graph( + one_lakh_prefix, + suffix_lakh_exact, + 1, + graph_thousands, + ) + | create_larger_number_graph( + one_lakh_prefix, + suffix_lakh_exact, + 0, + ten_thousand_crore_prefix, + ) + # రెండు లక్షల ... + | create_graph_suffix( + digit_except_one, + pynutil.insert(" లక్షల"), + 5, + ) + | create_larger_number_graph( + digit_except_one, + pynutil.insert(" లక్షల"), + 4, + digit, + ) + | create_larger_number_graph( + digit_except_one, + pynutil.insert(" లక్షల"), + 3, + teens_ties, + ) + | create_larger_number_graph( + digit_except_one, + pynutil.insert(" లక్షల"), + 2, + graph_hundreds, + ) + | create_larger_number_graph( + digit_except_one, + pynutil.insert(" లక్షల"), + 1, + graph_thousands, + ) + | create_larger_number_graph( + digit_except_one, + pynutil.insert(" లక్షల"), + 0, + ten_thousand_crore_prefix, + ) + ).optimize() + + ten_lakh_crore_prefix = ( + create_graph_suffix( + teens_ties_thousand, + pynutil.insert(" లక్ష"), + 5, + ) # 21 lakh crores -> ఇరవై ఒక లక్ష + | create_larger_number_graph( + teens_ties_thousand, + pynutil.insert(" లక్షల"), + 4, + digit, + ) + | create_larger_number_graph( + teens_ties_thousand, + pynutil.insert(" లక్షల"), + 3, + teens_ties, + ) + | create_larger_number_graph( + teens_ties_thousand, + pynutil.insert(" లక్షల"), + 2, + graph_hundreds, + ) + | create_larger_number_graph( + teens_ties_thousand, + pynutil.insert(" లక్షల"), + 1, + graph_thousands, + ) + | create_larger_number_graph( + teens_ties_thousand, + pynutil.insert(" లక్షల"), + 0, + ten_thousand_crore_prefix, + ) + # 10 lakh crores, 22 lakh crores ... + | create_graph_suffix( + teens_and_ties, + pynutil.insert(" లక్షల"), + 5, + ) + | create_larger_number_graph( + teens_and_ties, + pynutil.insert(" లక్షల"), + 4, + digit, + ) + | create_larger_number_graph( + teens_and_ties, + pynutil.insert(" లక్షల"), + 3, + teens_ties, + ) + | create_larger_number_graph( + teens_and_ties, + pynutil.insert(" లక్షల"), + 2, + graph_hundreds, + ) + | create_larger_number_graph( + teens_and_ties, + pynutil.insert(" లక్షల"), + 1, + graph_thousands, + ) + | create_larger_number_graph( + teens_and_ties, + pynutil.insert(" లక్షల"), + 0, + ten_thousand_crore_prefix, + ) + ).optimize() + + lakh_crore_count_prefix = (lakh_crore_prefix | ten_lakh_crore_prefix).optimize() + graph_ten_kharabs = create_graph_suffix( + lakh_crore_count_prefix, + suffix_lakh_crores_plural, + 7, + ) + graph_ten_kharabs |= create_larger_number_graph( + lakh_crore_count_prefix, + suffix_lakh_crores_before, + 6, + digit, + ) + graph_ten_kharabs |= create_larger_number_graph( + lakh_crore_count_prefix, + suffix_lakh_crores_before, + 5, + teens_ties, + ) + graph_ten_kharabs |= create_larger_number_graph( + lakh_crore_count_prefix, + suffix_lakh_crores_before, + 4, + graph_hundreds, + ) + graph_ten_kharabs |= create_larger_number_graph( + lakh_crore_count_prefix, + suffix_lakh_crores_before, + 3, + graph_thousands, + ) + graph_ten_kharabs |= create_larger_number_graph( + lakh_crore_count_prefix, + suffix_lakh_crores_before, + 2, + graph_ten_thousands, + ) + graph_ten_kharabs |= create_larger_number_graph( + lakh_crore_count_prefix, + suffix_lakh_crores_before, + 1, + graph_lakhs, + ) + graph_ten_kharabs |= create_larger_number_graph( + lakh_crore_count_prefix, + suffix_lakh_crores_before, + 0, + graph_ten_lakhs, + ) + + graph_ten_kharabs = pynutil.add_weight( + graph_ten_kharabs, + -1.4, + ).optimize() + + # Nils graph as lakh-crores + # 10000000000000 -> పది లక్షల కోట్లు + # 21000000000000 -> ఇరవై ఒక లక్ష కోట్లు + + suffix_nil_crores_plural = pynutil.insert(" కోట్లు") + suffix_nil_crores_before = pynutil.insert(" కోట్ల") + + nil_crore_count_prefix = (lakh_crore_prefix | ten_lakh_crore_prefix).optimize() + + graph_nils = create_graph_suffix( + nil_crore_count_prefix, + suffix_nil_crores_plural, + 7, + ) + graph_nils |= create_larger_number_graph(nil_crore_count_prefix, suffix_nil_crores_before, 6, digit) + graph_nils |= create_larger_number_graph(nil_crore_count_prefix, suffix_nil_crores_before, 5, teens_ties) + graph_nils |= create_larger_number_graph(nil_crore_count_prefix, suffix_nil_crores_before, 4, graph_hundreds) + graph_nils |= create_larger_number_graph(nil_crore_count_prefix, suffix_nil_crores_before, 3, graph_thousands) + graph_nils |= create_larger_number_graph( + nil_crore_count_prefix, suffix_nil_crores_before, 2, graph_ten_thousands + ) + graph_nils |= create_larger_number_graph(nil_crore_count_prefix, suffix_nil_crores_before, 1, graph_lakhs) + graph_nils |= create_larger_number_graph(nil_crore_count_prefix, suffix_nil_crores_before, 0, graph_ten_lakhs) + + graph_nils = pynutil.add_weight(graph_nils, -2.4).optimize() + + # Ten Nils graph as crore-crores + # 100000000000000 -> కోటి కోట్లు + # 110000000000000 -> కోటి పది లక్షల కోట్లు + # 100000001000000 -> కోటి కోట్లు పది లక్షలు + + suffix_ten_nil_crores_plural = pynutil.insert(" కోట్లు") + suffix_ten_nil_crores_before = pynutil.insert(" కోట్ల") + + ten_nil_lakh_remainder_before_kotlu = ( + create_graph_suffix(teens_and_ties, pynutil.insert(" లక్షల"), 5) + | create_larger_number_graph(teens_and_ties, pynutil.insert(" లక్షల"), 4, digit) + | create_larger_number_graph(teens_and_ties, pynutil.insert(" లక్షల"), 3, teens_ties) + | create_larger_number_graph(teens_and_ties, pynutil.insert(" లక్షల"), 2, graph_hundreds) + | create_larger_number_graph(teens_and_ties, pynutil.insert(" లక్షల"), 1, graph_thousands) + | create_larger_number_graph(teens_and_ties, pynutil.insert(" లక్షల"), 0, graph_ten_thousands) + ).optimize() + + ten_nil_crore_count_prefix = ( + graph_crores + | graph_ten_crores + | create_larger_number_graph( + one_crore_prefix, + suffix_crore_exact, + 0, + ten_nil_lakh_remainder_before_kotlu, + ) + | create_larger_number_graph( + digit_except_one, + suffix_crore_before, + 0, + ten_nil_lakh_remainder_before_kotlu, + ) + | create_larger_number_graph( + teens_and_ties, + suffix_ten_crore_before, + 0, + ten_nil_lakh_remainder_before_kotlu, + ) + ).optimize() + + graph_ten_nils = create_graph_suffix( + ten_nil_crore_count_prefix, + suffix_ten_nil_crores_plural, + 7, + ) + graph_ten_nils |= create_larger_number_graph( + ten_nil_crore_count_prefix, suffix_ten_nil_crores_before, 6, digit + ) + graph_ten_nils |= create_larger_number_graph( + ten_nil_crore_count_prefix, suffix_ten_nil_crores_before, 5, teens_ties + ) + graph_ten_nils |= create_larger_number_graph( + ten_nil_crore_count_prefix, suffix_ten_nil_crores_before, 4, graph_hundreds + ) + graph_ten_nils |= create_larger_number_graph( + ten_nil_crore_count_prefix, suffix_ten_nil_crores_before, 3, graph_thousands + ) + graph_ten_nils |= create_larger_number_graph( + ten_nil_crore_count_prefix, suffix_ten_nil_crores_before, 2, graph_ten_thousands + ) + graph_ten_nils |= create_larger_number_graph( + ten_nil_crore_count_prefix, suffix_ten_nil_crores_before, 1, graph_lakhs + ) + graph_ten_nils |= create_larger_number_graph( + ten_nil_crore_count_prefix, suffix_ten_nil_crores_before, 0, graph_ten_lakhs + ) + + graph_ten_nils = pynutil.add_weight(graph_ten_nils, -3.0).optimize() + + # Padmas graph as koti-crores + # 1000000000000000 -> పది కోటి కోట్లు + # 2100000000000000 -> ఇరవై ఒక కోటి కోట్లు + # 9999999999999999 -> తొంభై తొమ్మిది కోటి ... కోట్లు ... + + suffix_padma_crores_plural = pynutil.insert(" కోట్లు") + suffix_padma_crores_before = pynutil.insert(" కోట్ల") + + padma_lakh_remainder_before_kotlu = ( + create_graph_suffix(teens_and_ties, pynutil.insert(" లక్షల"), 5) + | create_larger_number_graph(teens_and_ties, pynutil.insert(" లక్షల"), 4, digit) + | create_larger_number_graph(teens_and_ties, pynutil.insert(" లక్షల"), 3, teens_ties) + | create_larger_number_graph(teens_and_ties, pynutil.insert(" లక్షల"), 2, graph_hundreds) + | create_larger_number_graph(teens_and_ties, pynutil.insert(" లక్షల"), 1, graph_thousands) + | create_larger_number_graph(teens_and_ties, pynutil.insert(" లక్షల"), 0, graph_ten_thousands) + ).optimize() + + padma_crore_count_prefix = ( + create_graph_suffix(teens_ties_thousand, pynutil.insert(" కోటి"), 7) + | create_graph_suffix(teens_and_ties, pynutil.insert(" కోటి"), 7) + | create_larger_number_graph(teens_ties_thousand, pynutil.insert(" కోటి"), 6, digit) + | create_larger_number_graph(teens_and_ties, pynutil.insert(" కోటి"), 6, digit) + | create_larger_number_graph(teens_ties_thousand, pynutil.insert(" కోటి"), 5, teens_ties) + | create_larger_number_graph(teens_and_ties, pynutil.insert(" కోటి"), 5, teens_ties) + | create_larger_number_graph(teens_ties_thousand, pynutil.insert(" కోటి"), 4, graph_hundreds) + | create_larger_number_graph(teens_and_ties, pynutil.insert(" కోటి"), 4, graph_hundreds) + | create_larger_number_graph(teens_ties_thousand, pynutil.insert(" కోటి"), 3, graph_thousands) + | create_larger_number_graph(teens_and_ties, pynutil.insert(" కోటి"), 3, graph_thousands) + | create_larger_number_graph(teens_ties_thousand, pynutil.insert(" కోటి"), 2, graph_ten_thousands) + | create_larger_number_graph(teens_and_ties, pynutil.insert(" కోటి"), 2, graph_ten_thousands) + | create_larger_number_graph(teens_ties_thousand, pynutil.insert(" కోటి"), 1, graph_lakhs) + | create_larger_number_graph(teens_and_ties, pynutil.insert(" కోటి"), 1, graph_lakhs) + | create_larger_number_graph( + teens_ties_thousand, pynutil.insert(" కోటి"), 0, padma_lakh_remainder_before_kotlu + ) + | create_larger_number_graph(teens_and_ties, pynutil.insert(" కోటి"), 0, padma_lakh_remainder_before_kotlu) + ).optimize() + + graph_padmas = create_graph_suffix( + padma_crore_count_prefix, + suffix_padma_crores_plural, + 7, + ) + + graph_padmas |= create_larger_number_graph(padma_crore_count_prefix, suffix_padma_crores_before, 6, digit) + graph_padmas |= create_larger_number_graph(padma_crore_count_prefix, suffix_padma_crores_before, 5, teens_ties) + graph_padmas |= create_larger_number_graph( + padma_crore_count_prefix, suffix_padma_crores_before, 4, graph_hundreds + ) + graph_padmas |= create_larger_number_graph( + padma_crore_count_prefix, suffix_padma_crores_before, 3, graph_thousands + ) + graph_padmas |= create_larger_number_graph( + padma_crore_count_prefix, suffix_padma_crores_before, 2, graph_ten_thousands + ) + graph_padmas |= create_larger_number_graph( + padma_crore_count_prefix, suffix_padma_crores_before, 1, graph_lakhs + ) + graph_padmas |= create_larger_number_graph( + padma_crore_count_prefix, suffix_padma_crores_before, 0, graph_ten_lakhs + ) + + graph_padmas = pynutil.add_weight( + graph_padmas, + -4.0, + ).optimize() + graph_ten_padmas = graph_padmas + + # Ten Padmas graph as hundred-koti-crores + # 10000000000000000 -> వంద కోటి కోట్లు + # 20100000000000000 -> రెండు వందల ఒక కోటి కోట్లు + + suffix_ten_padma_crores_plural = pynutil.insert(" కోట్లు") + suffix_ten_padma_crores_before = pynutil.insert(" కోట్ల") + + ten_padma_one_crore_count_prefix = create_graph_suffix( + hundred_one_crore_prefix, + pynutil.insert(" కోటి"), + 7, + ).optimize() + + ten_padma_crore_count_prefix = ( + ten_padma_one_crore_count_prefix + | create_graph_suffix(hundred_crore_prefix, pynutil.insert(" కోటి"), 7) + | create_larger_number_graph(hundred_crore_prefix, pynutil.insert(" కోటి"), 6, digit) + | create_larger_number_graph(hundred_crore_prefix, pynutil.insert(" కోటి"), 5, teens_ties) + | create_larger_number_graph(hundred_crore_prefix, pynutil.insert(" కోటి"), 4, graph_hundreds) + | create_larger_number_graph(hundred_crore_prefix, pynutil.insert(" కోటి"), 3, graph_thousands) + | create_larger_number_graph(hundred_crore_prefix, pynutil.insert(" కోటి"), 2, graph_ten_thousands) + | create_larger_number_graph(hundred_crore_prefix, pynutil.insert(" కోటి"), 1, graph_lakhs) + | create_larger_number_graph( + hundred_crore_prefix, pynutil.insert(" కోటి"), 0, padma_lakh_remainder_before_kotlu + ) + ).optimize() + + graph_ten_padmas = create_graph_suffix( + ten_padma_crore_count_prefix, + suffix_ten_padma_crores_plural, + 7, + ) + graph_ten_padmas |= create_larger_number_graph( + ten_padma_crore_count_prefix, suffix_ten_padma_crores_before, 6, digit + ) + graph_ten_padmas |= create_larger_number_graph( + ten_padma_crore_count_prefix, suffix_ten_padma_crores_before, 5, teens_ties + ) + graph_ten_padmas |= create_larger_number_graph( + ten_padma_crore_count_prefix, suffix_ten_padma_crores_before, 4, graph_hundreds + ) + graph_ten_padmas |= create_larger_number_graph( + ten_padma_crore_count_prefix, suffix_ten_padma_crores_before, 3, graph_thousands + ) + graph_ten_padmas |= create_larger_number_graph( + ten_padma_crore_count_prefix, suffix_ten_padma_crores_before, 2, graph_ten_thousands + ) + graph_ten_padmas |= create_larger_number_graph( + ten_padma_crore_count_prefix, suffix_ten_padma_crores_before, 1, graph_lakhs + ) + graph_ten_padmas |= create_larger_number_graph( + ten_padma_crore_count_prefix, suffix_ten_padma_crores_before, 0, graph_ten_lakhs + ) + + graph_ten_padmas = pynutil.add_weight( + graph_ten_padmas, + -4.5, + ).optimize() + + # Shankhs graph as thousand-koti-crores + # 100000000000000000 -> వెయ్యి కోటి కోట్లు + # 200000000000000000 -> రెండు వేల కోటి కోట్లు + # 1000000000000000000 -> పది వేల కోటి కోట్లు + + suffix_shankh_koti = pynutil.insert(" కోటి") + suffix_shankh_crores_plural = pynutil.insert(" కోట్లు") + suffix_shankh_crores_before = pynutil.insert(" కోట్ల") + + shankh_koti_crore_count_prefix = (thousand_crore_prefix | ten_thousand_crore_prefix).optimize() + + shankh_koti_count_prefix = ( + create_graph_suffix(shankh_koti_crore_count_prefix, suffix_shankh_koti, 7) + | create_larger_number_graph(shankh_koti_crore_count_prefix, suffix_shankh_koti, 6, digit) + | create_larger_number_graph(shankh_koti_crore_count_prefix, suffix_shankh_koti, 5, teens_ties) + | create_larger_number_graph(shankh_koti_crore_count_prefix, suffix_shankh_koti, 4, graph_hundreds) + | create_larger_number_graph(shankh_koti_crore_count_prefix, suffix_shankh_koti, 3, graph_thousands) + | create_larger_number_graph(shankh_koti_crore_count_prefix, suffix_shankh_koti, 2, graph_ten_thousands) + | create_larger_number_graph(shankh_koti_crore_count_prefix, suffix_shankh_koti, 1, graph_lakhs) + | create_larger_number_graph( + shankh_koti_crore_count_prefix, suffix_shankh_koti, 0, padma_lakh_remainder_before_kotlu + ) + ).optimize() + + graph_shankhs = create_graph_suffix( + shankh_koti_count_prefix, + suffix_shankh_crores_plural, + 7, + ) + + graph_shankhs |= create_larger_number_graph(shankh_koti_count_prefix, suffix_shankh_crores_before, 6, digit) + graph_shankhs |= create_larger_number_graph( + shankh_koti_count_prefix, suffix_shankh_crores_before, 5, teens_ties + ) + graph_shankhs |= create_larger_number_graph( + shankh_koti_count_prefix, suffix_shankh_crores_before, 4, graph_hundreds + ) + graph_shankhs |= create_larger_number_graph( + shankh_koti_count_prefix, suffix_shankh_crores_before, 3, graph_thousands + ) + graph_shankhs |= create_larger_number_graph( + shankh_koti_count_prefix, suffix_shankh_crores_before, 2, graph_ten_thousands + ) + graph_shankhs |= create_larger_number_graph( + shankh_koti_count_prefix, suffix_shankh_crores_before, 1, graph_lakhs + ) + graph_shankhs |= create_larger_number_graph( + shankh_koti_count_prefix, suffix_shankh_crores_before, 0, graph_ten_lakhs + ) + + graph_shankhs = pynutil.add_weight( + graph_shankhs, + -5.0, + ).optimize() + graph_ten_shankhs = graph_shankhs + + # Only match exactly 2 digits to avoid interfering with telephone numbers, decimals, etc. + single_digit = digit | zero + graph_leading_zero = zero + insert_space + single_digit + graph_leading_zero = pynutil.add_weight(graph_leading_zero, 0.5) + + graph_without_leading_zeros = ( + digit + | zero + | teens_and_ties + | graph_hundreds + | graph_thousands + | graph_ten_thousands + | graph_lakhs + | graph_ten_lakhs + | graph_crores + | graph_ten_crores + | graph_arabs + | graph_ten_arabs + | graph_kharabs + | graph_ten_kharabs + | graph_nils + | graph_ten_nils + | graph_padmas + | graph_ten_padmas + | graph_shankhs + | graph_ten_shankhs + ) + self.graph_without_leading_zeros = graph_without_leading_zeros.optimize() + + # Handle numbers with leading zeros by reading digit-by-digit + cardinal_with_leading_zeros = pynini.compose( + NEMO_ALL_ZERO + pynini.closure(NEMO_ALL_DIGIT), self.single_digits_graph + ) + cardinal_with_leading_zeros = pynutil.add_weight(cardinal_with_leading_zeros, 0.5) + + # Full graph including leading zeros - for standalone cardinal matching + final_graph = graph_without_leading_zeros | cardinal_with_leading_zeros + + optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + + self.final_graph = final_graph.optimize() + final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.final_graph + pynutil.insert("\"") + final_graph = self.add_tokens(final_graph) + self.fst = final_graph diff --git a/nemo_text_processing/text_normalization/te/taggers/punctuation.py b/nemo_text_processing/text_normalization/te/taggers/punctuation.py new file mode 100644 index 000000000..47c5c5314 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/taggers/punctuation.py @@ -0,0 +1,62 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from unicodedata import category + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.te.graph_utils import NEMO_NOT_SPACE, NEMO_SIGMA, GraphFst + + +class PunctuationFst(GraphFst): + """ + Finite state transducer for classifying punctuation + e.g. a, -> tokens { name: "a" } tokens { name: "," } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transductions are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="punctuation", kind="classify", deterministic=deterministic) + s = "!#%&\'()*+,-./:;<=>?@^_`{|}~\"" + + punct_symbols_to_exclude = ["[", "]"] + punct_unicode = [ + chr(i) + for i in range(sys.maxunicode) + if category(chr(i)).startswith("P") and chr(i) not in punct_symbols_to_exclude + ] + + self.punct_marks = [p for p in punct_unicode + list(s)] + + punct = pynini.union(*self.punct_marks) + punct = pynini.closure(punct, 1) + + emphasis = ( + pynini.accep("<") + + pynini.union( + (pynini.closure(NEMO_NOT_SPACE - pynini.union("<", ">"), 1) + pynini.closure(pynini.accep("/"), 0, 1)), + (pynini.accep("/") + pynini.closure(NEMO_NOT_SPACE - pynini.union("<", ">"), 1)), + ) + + pynini.accep(">") + ) + punct = plurals._priority_union(emphasis, punct, NEMO_SIGMA) + + self.graph = punct + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/nemo_text_processing/text_normalization/te/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/te/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..25df2aa6c --- /dev/null +++ b/nemo_text_processing/text_normalization/te/taggers/tokenize_and_classify.py @@ -0,0 +1,170 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.te.graph_utils import ( + NEMO_SPACE, + NEMO_WHITE_SPACE, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.te.taggers.cardinal import CardinalFst + +# from nemo_text_processing.text_normalization.te.taggers.date import DateFst +# from nemo_text_processing.text_normalization.te.taggers.decimal import DecimalFst +# from nemo_text_processing.text_normalization.te.taggers.electronic import ElectronicFst +# from nemo_text_processing.text_normalization.te.taggers.fraction import FractionFst +# from nemo_text_processing.text_normalization.te.taggers.measure import MeasureFst +# from nemo_text_processing.text_normalization.te.taggers.money import MoneyFst +# from nemo_text_processing.text_normalization.te.taggers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.te.taggers.punctuation import PunctuationFst + +# from nemo_text_processing.text_normalization.te.taggers.telephone import TelephoneFst +# from nemo_text_processing.text_normalization.te.taggers.time import TimeFst +# from nemo_text_processing.text_normalization.te.taggers.whitelist import WhiteListFst +from nemo_text_processing.text_normalization.te.taggers.word import WordFst + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence including punctuation. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + input_case: str, + deterministic: bool = True, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + whitelist_file = os.path.basename(whitelist) if whitelist else "" + far_file = os.path.join( + cache_dir, + f"te_tn_{deterministic}_deterministic_{input_case}_{whitelist_file}_tokenize.far", + ) + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logging.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logging.info(f"Creating ClassifyFst grammars.") + + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + + # decimal = DecimalFst(cardinal=cardinal, deterministic=deterministic) + # decimal_graph = decimal.fst + + # fraction = FractionFst(cardinal=cardinal, deterministic=deterministic) + # fraction_graph = fraction.fst + + # date = DateFst(cardinal=cardinal) + # date_graph = date.fst + + # timefst = TimeFst(cardinal=cardinal) + # time_graph = timefst.fst + + # ordinal = OrdinalFst(cardinal=cardinal, deterministic=deterministic) + # ordinal_graph = ordinal.fst + + # measure = MeasureFst(cardinal=cardinal, decimal=decimal, ordinal=ordinal, input_case=input_case) + # measure_graph = measure.fst + + # money = MoneyFst(cardinal=cardinal) + # money_graph = money.fst + + # whitelist_graph = WhiteListFst( + # input_case=input_case, deterministic=deterministic, input_file=whitelist + # ).fst + + punctuation = PunctuationFst(deterministic=deterministic) + punct_graph = punctuation.fst + + # telephone = TelephoneFst() + # telephone_graph = telephone.fst + + # electronic = ElectronicFst(deterministic=deterministic) + # electronic_graph = electronic.fst + + classify = ( + # pynutil.add_weight(whitelist_graph, 1.01)| + pynutil.add_weight(cardinal_graph, 1.1) + # | pynutil.add_weight(decimal_graph, 1.1) + # | pynutil.add_weight(fraction_graph, 1.1) + # | pynutil.add_weight(date_graph, 1.1) + # | pynutil.add_weight(time_graph, 1.1) + # | pynutil.add_weight(measure_graph, 1.1) + # | pynutil.add_weight(money_graph, 1.1) + # | pynutil.add_weight(telephone_graph, 1.1) + # | pynutil.add_weight(ordinal_graph, 1.1) + # | pynutil.add_weight(electronic_graph, 1.1) + ) + + word_graph = WordFst(punctuation=punctuation, deterministic=deterministic).fst + + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=2.1) + pynutil.insert(" }") + punct = pynini.closure( + pynini.union( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space), + (pynutil.insert(NEMO_SPACE) + punct), + ), + 1, + ) + + classify = pynini.union(classify, pynutil.add_weight(word_graph, 100)) + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(NEMO_SPACE)) + + token + + pynini.closure(pynutil.insert(NEMO_SPACE) + punct) + ) + + graph = token_plus_punct + pynini.closure( + pynini.union( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space), + (pynutil.insert(NEMO_SPACE) + punct + pynutil.insert(NEMO_SPACE)), + ) + + token_plus_punct + ) + + graph = delete_space + graph + delete_space + graph = pynini.union(graph, punct) + + self.fst = graph.optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) + logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/te/taggers/word.py b/nemo_text_processing/text_normalization/te/taggers/word.py new file mode 100644 index 000000000..2d73e412b --- /dev/null +++ b/nemo_text_processing/text_normalization/te/taggers/word.py @@ -0,0 +1,61 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.te.graph_utils import ( + MIN_NEG_WEIGHT, + NEMO_NOT_SPACE, + GraphFst, + convert_space, +) +from nemo_text_processing.text_normalization.te.taggers.punctuation import PunctuationFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying Telugu words. + e.g. సోనా -> tokens { name: "సోనా" } + + Args: + punctuation: PunctuationFst + deterministic: if True will provide a single transduction option, + for False multiple transductions are generated (used for audio-based normalization) + """ + + def __init__(self, punctuation: PunctuationFst, deterministic: bool = True): + super().__init__(name="word", kind="classify", deterministic=deterministic) + + # Define Telugu characters and symbols using pynini.union + TELUGU_CHAR = pynini.union( + *[chr(i) for i in range(0x0C00, 0x0C03 + 1)], # Telugu signs (Anusvara, Visarga) + *[chr(i) for i in range(0x0C05, 0x0C39 + 1)], # Telugu vowels and consonants + *[chr(i) for i in range(0x0C3E, 0x0C4D + 1)], # Telugu vowel signs (diacritics) and Virama + ).optimize() + + # Include punctuation in the graph + punct = punctuation.graph + default_graph = pynini.closure(pynini.difference(NEMO_NOT_SPACE, punct.project("input")), 1) + symbols_to_exclude = (pynini.union("$", "€", "₩", "£", "¥", "#", "%") | punct).optimize() + + # Use TELUGU_CHAR in the graph + graph = pynini.closure(pynini.difference(TELUGU_CHAR, symbols_to_exclude), 1) + graph = pynutil.add_weight(graph, MIN_NEG_WEIGHT) | default_graph + + # Ensure no spaces around punctuation + graph = pynini.closure(graph + pynini.closure(punct + graph, 0, 1)) + + self.graph = convert_space(graph) + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/nemo_text_processing/text_normalization/te/utils.py b/nemo_text_processing/text_normalization/te/utils.py new file mode 100644 index 000000000..fa0471fb6 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/utils.py @@ -0,0 +1,74 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import os +import pynini + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + label_tsv = open(abs_path, encoding="utf-8") + labels = list(csv.reader(label_tsv, delimiter="\t")) + label_tsv.close() + return labels + + +def augment_labels_with_punct_at_end(labels): + """ + augments labels: if key ends on a punctuation that value does not have, add a new label + where the value maintains the punctuation + + Args: + labels : input labels + Returns: + additional labels + """ + res = [] + for label in labels: + if len(label) > 1: + if label[0][-1] == "." and label[1][-1] != ".": + res.append([label[0], label[1] + "."] + label[2:]) + return res + + +def apply_fst(text, fst): + """Given a string input, returns the output string + produced by traversing the path with lowest weight. + If no valid path accepts input string, returns an + error. + """ + try: + print(pynini.shortestpath(text @ fst).string()) + except pynini.FstOpError: + print(f"Error: No valid output with given input: '{text}'") diff --git a/nemo_text_processing/text_normalization/te/verbalizers/__init__.py b/nemo_text_processing/text_normalization/te/verbalizers/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/te/verbalizers/cardinal.py b/nemo_text_processing/text_normalization/te/verbalizers/cardinal.py new file mode 100644 index 000000000..bf00734c3 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/verbalizers/cardinal.py @@ -0,0 +1,49 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.te.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinal, e.g. + cardinal { negative: "true" integer: "23" } -> minus twenty three + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + + self.optional_sign = pynini.cross("negative: \"true\"", "minus ") + if not deterministic: + self.optional_sign |= pynini.cross("negative: \"true\"", "negative ") + self.optional_sign |= pynini.cross("negative: \"true\"", "dash ") + + self.optional_sign = pynini.closure(self.optional_sign + delete_space, 0, 1) + + integer = pynini.closure(NEMO_NOT_QUOTE) + + self.integer = delete_space + pynutil.delete("\"") + integer + pynutil.delete("\"") + integer = pynutil.delete("integer:") + self.integer + + self.numbers = self.optional_sign + integer + delete_tokens = self.delete_tokens(self.numbers) + + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/te/verbalizers/post_processing.py b/nemo_text_processing/text_normalization/te/verbalizers/post_processing.py new file mode 100644 index 000000000..6dd341b1d --- /dev/null +++ b/nemo_text_processing/text_normalization/te/verbalizers/post_processing.py @@ -0,0 +1,96 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.te.graph_utils import ( + MIN_NEG_WEIGHT, + NEMO_CHAR, + NEMO_SIGMA, + generator_main, +) +from nemo_text_processing.text_normalization.te.taggers.punctuation import PunctuationFst +from nemo_text_processing.utils.logging import logger + + +class PostProcessingFst: + """ + Finite state transducer that post-processing an entire sentence after verbalization is complete, e.g. + removes extra spaces around punctuation marks " ( one hundred and twenty three ) " -> "(one hundred and twenty three)" + + Args: + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, cache_dir: str = None, overwrite_cache: bool = False): + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, "te_tn_post_processing.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["post_process_graph"] + logger.info(f'Post processing graph was restored from {far_file}.') + else: + self.fst = self.get_punct_postprocess_graph() + + if far_file: + generator_main(far_file, {"post_process_graph": self.fst}) + + def get_punct_postprocess_graph(self): + """ + Returns graph to post process punctuation marks. + + By default, spaces are removed before punctuation marks like comma, period, etc. + """ + punct_marks_all = PunctuationFst().punct_marks + + # Punctuation marks that should NOT have space before them + # (most punctuation except quotes, dashes, and opening brackets) + quotes = ["'", "\"", "«"] + dashes = ["-", "—"] + brackets = ["<", "{", "(", r"\["] + allow_space_before_punct = quotes + dashes + brackets + + no_space_before_punct = [m for m in punct_marks_all if m not in allow_space_before_punct] + # Add Telugu-specific punctuation + no_space_before_punct.extend(["।", ",", ".", ";", ":", "!", "?"]) + # Remove duplicates + no_space_before_punct = list(set(no_space_before_punct)) + no_space_before_punct = pynini.union(*no_space_before_punct) + + delete_space = pynutil.delete(" ") + + # Delete space before no_space_before_punct marks + non_punct = pynini.difference(NEMO_CHAR, no_space_before_punct).optimize() + graph = ( + pynini.closure(non_punct) + + pynini.closure( + no_space_before_punct | pynutil.add_weight(delete_space + no_space_before_punct, MIN_NEG_WEIGHT) + ) + + pynini.closure(non_punct) + ) + graph = pynini.closure(graph).optimize() + + # Remove space after opening brackets + no_space_after_punct = pynini.union(*brackets) + no_space_after_punct = pynini.cdrewrite(delete_space, no_space_after_punct, NEMO_SIGMA, NEMO_SIGMA).optimize() + graph = pynini.compose(graph, no_space_after_punct).optimize() + + return graph diff --git a/nemo_text_processing/text_normalization/te/verbalizers/verbalize.py b/nemo_text_processing/text_normalization/te/verbalizers/verbalize.py new file mode 100644 index 000000000..9218c53f6 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/verbalizers/verbalize.py @@ -0,0 +1,90 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.te.graph_utils import GraphFst +from nemo_text_processing.text_normalization.te.verbalizers.cardinal import CardinalFst + +# from nemo_text_processing.text_normalization.te.verbalizers.date import DateFst +# from nemo_text_processing.text_normalization.te.verbalizers.decimal import DecimalFst +# from nemo_text_processing.text_normalization.te.verbalizers.electronic import ElectronicFst +# from nemo_text_processing.text_normalization.te.verbalizers.fraction import FractionFst +# from nemo_text_processing.text_normalization.te.verbalizers.measure import MeasureFst +# from nemo_text_processing.text_normalization.te.verbalizers.money import MoneyFst +# from nemo_text_processing.text_normalization.te.verbalizers.ordinal import OrdinalFst +# from nemo_text_processing.text_normalization.te.verbalizers.telephone import TelephoneFst +# from nemo_text_processing.text_normalization.te.verbalizers.time import TimeFst +# from nemo_text_processing.text_normalization.te.verbalizers.whitelist import WhiteListFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + + # decimal = DecimalFst(deterministic=deterministic) + # decimal_graph = decimal.fst + + # fraction = FractionFst(cardinal=cardinal, deterministic=deterministic) + # fraction_graph = fraction.fst + + # date = DateFst() + # date_graph = date.fst + + # time = TimeFst(cardinal=cardinal) + # time_graph = time.fst + + # ordinal = OrdinalFst(deterministic=deterministic) + # ordinal_graph = ordinal.fst + + # measure = MeasureFst(cardinal=cardinal, decimal=decimal) + # measure_graph = measure.fst + + # money = MoneyFst() + # money_graph = money.fst + + # telephone = TelephoneFst() + # telephone_graph = telephone.fst + + # electronic = ElectronicFst(deterministic=deterministic) + # electronic_graph = electronic.fst + + # whitelist_graph = WhiteListFst(deterministic=deterministic).fst + + graph = ( + cardinal_graph + # | decimal_graph + # | fraction_graph + # | date_graph + # | time_graph + # | measure_graph + # | money_graph + # | ordinal_graph + # | whitelist_graph + # | telephone_graph + # | electronic_graph + ) + + self.fst = graph diff --git a/nemo_text_processing/text_normalization/te/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/te/verbalizers/verbalize_final.py new file mode 100644 index 000000000..ee41577c6 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/verbalizers/verbalize_final.py @@ -0,0 +1,76 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.te.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.te.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.text_normalization.te.verbalizers.word import WordFst + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence, e.g. + tokens { name: "its" } tokens { time { hours: "twelve" minutes: "thirty" } } tokens { name: "now" } tokens { name: "." } -> its twelve thirty now . + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"en_tn_{deterministic}_deterministic_verbalizer.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + logging.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + else: + verbalize = VerbalizeFst(deterministic=deterministic).fst + word = WordFst(deterministic=deterministic).fst + types = verbalize | word + + if deterministic: + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + else: + graph = delete_space + types + delete_space + + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + + self.fst = graph.optimize() + if far_file: + generator_main(far_file, {"verbalize": self.fst}) + logging.info(f"VerbalizeFinalFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/te/verbalizers/word.py b/nemo_text_processing/text_normalization/te/verbalizers/word.py new file mode 100644 index 000000000..cb4f41784 --- /dev/null +++ b/nemo_text_processing/text_normalization/te/verbalizers/word.py @@ -0,0 +1,43 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.te.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + + +class WordFst(GraphFst): + """ + Finite state transducer for verbalizing Telugu words. + e.g. tokens { name: "ఏడు" } -> ఏడు + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="verbalize", deterministic=deterministic) + chars = pynini.closure(NEMO_CHAR - " ", 1) + punct = pynini.union("!", "?", ".", ",", "-", ":", ";", "।") # Add other punctuation marks as needed + char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"") + + # Ensure no spaces around punctuation + graph = char + pynini.closure(delete_space + punct, 0, 1) + + # Explicitly remove spaces before punctuation + remove_space_before_punct = pynini.cdrewrite(pynini.cross(" ", ""), "", punct, NEMO_SIGMA) + graph = graph @ remove_space_before_punct + + self.fst = graph.optimize() diff --git a/tests/nemo_text_processing/te/__init__.py b/tests/nemo_text_processing/te/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/tests/nemo_text_processing/te/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/nemo_text_processing/te/data_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/te/data_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..068c67f66 --- /dev/null +++ b/tests/nemo_text_processing/te/data_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,51 @@ +4 నాలుగులు~నాలుగు నాలుగులు +౬ మంది ఆటగాళ్లు బయటకు~ఆరు మంది ఆటగాళ్లు బయటకు +4 ఓవర్లలో 17 పరుగులు~నాలుగు ఓవర్లలో పదిహేడు పరుగులు +౫ చాక్లెట్లు ౯ టాఫీలు~ఐదు చాక్లెట్లు తొమ్మిది టాఫీలు +10099~పది వేల తొంభై తొమ్మిది +100001~లక్షా ఒకటి +౫ నాలుగులు~ఐదు నాలుగులు +4 మంది మృతి 18 మందికి గాయాలు~నాలుగు మంది మృతి పధ్ధెనిమిది మందికి గాయాలు +51022345567~ఐదు వేల నూట రెండు కోట్ల ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు +అధ్యాయం ౧౦ శ్లోకం ౨~అధ్యాయం పది శ్లోకం రెండు +101~నూట ఒకటి +100~వంద +౧౦౦~వంద +౧౦౫~నూట ఐదు +200~రెండు వందలు +౨౦౦~రెండు వందలు +203~రెండు వందల మూడు +౫౦౨~ఐదు వందల రెండు +111~నూట పదకొండు +౧౧౯~నూట పంతొమ్మిది +130~నూట ముప్పై +౧౫౦~నూట యాభై +312~మూడు వందల పన్నెండు +౫౨౫~ఐదు వందల ఇరవై ఐదు +425~నాలుగు వందల ఇరవై ఐదు +౯౧౫~తొమ్మిది వందల పదిహేను +1001~వెయ్యి ఒకటి +5551 అరటిపండ్లు~ఐదు వేల ఐదు వందల యాభై ఒకటి అరటిపండ్లు +౫౫౫౧ అరటిపండ్లు~ఐదు వేల ఐదు వందల యాభై ఒకటి అరటిపండ్లు +85~ఎనభై ఐదు +౯౧~తొంభై ఒకటి +౧౩౨౩~వెయ్యి మూడు వందల ఇరవై మూడు +1345~వెయ్యి మూడు వందల నలభై ఐదు +౧౩౪౫౬~పదమూడు వేల నాలుగు వందల యాభై ఆరు +12346~పన్నెండు వేల మూడు వందల నలభై ఆరు +౧౨౩౪౫౬~లక్ష ఇరవై మూడు వేల నాలుగు వందల యాభై ఆరు +872987~ఎనిమిది లక్షల డెబ్బై రెండు వేల తొమ్మిది వందల ఎనభై ఏడు +౯౮౭౬౦౯~తొమ్మిది లక్షల ఎనభై ఏడు వేల ఆరు వందల తొమ్మిది +9876789~తొంభై ఎనిమిది లక్షల డెబ్బై ఆరు వేల ఏడు వందల ఎనభై తొమ్మిది +౨౩౪౫౫౬౭~ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు +12345567~కోటి ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు +౧౨౧౨౧౨౧౨~కోటి ఇరవై ఒక లక్షల ఇరవై ఒక వేల రెండు వందల పన్నెండు +1122345567~నూట పన్నెండు కోట్ల ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు +౧౦౨౨౩౪౫౫౬౭~నూట రెండు కోట్ల ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు +11022345567~వెయ్యి నూట రెండు కోట్ల ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు +౫౧౦౨౨౩౪౫౫౬౭~ఐదు వేల నూట రెండు కోట్ల ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు +౦౫~సున్నా ఐదు +01~సున్నా ఒకటి +౦౭౩~సున్నా ఏడు మూడు +0001~సున్నా సున్నా సున్నా ఒకటి +౦౦౦~సున్నా సున్నా సున్నా diff --git a/tests/nemo_text_processing/te/test_cardinal.py b/tests/nemo_text_processing/te/test_cardinal.py new file mode 100644 index 000000000..799cf29c4 --- /dev/null +++ b/tests/nemo_text_processing/te/test_cardinal.py @@ -0,0 +1,37 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestCardinal: + normalizer = Normalizer( + input_case='cased', + lang='te', + cache_dir=CACHE_DIR, + overwrite_cache=False, + post_process=False, + ) + + @parameterized.expand(parse_test_case_file('te/data_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip()