diff --git a/nemo_text_processing/text_normalization/kn/__init__.py b/nemo_text_processing/text_normalization/kn/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/kn/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/kn/data/__init__.py b/nemo_text_processing/text_normalization/kn/data/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/kn/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/kn/data/numbers/__init__.py b/nemo_text_processing/text_normalization/kn/data/numbers/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/kn/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/kn/data/numbers/digit.tsv b/nemo_text_processing/text_normalization/kn/data/numbers/digit.tsv new file mode 100644 index 000000000..d4e5a1171 --- /dev/null +++ b/nemo_text_processing/text_normalization/kn/data/numbers/digit.tsv @@ -0,0 +1,18 @@ +1 ಒಂದು +2 ಎರಡು +3 ಮೂರು +4 ನಾಲ್ಕು +5 ಐದು +6 ಆರು +7 ಏಳು +8 ಎಂಟು +9 ಒಂಬತ್ತು +೧ ಒಂದು +೨ ಎರಡು +೩ ಮೂರು +೪ ನಾಲ್ಕು +೫ ಐದು +೬ ಆರು +೭ ಏಳು +೮ ಎಂಟು +೯ ಒಂಬತ್ತು \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/kn/data/numbers/hundreds.tsv b/nemo_text_processing/text_normalization/kn/data/numbers/hundreds.tsv new file mode 100644 index 000000000..20f354ad0 --- /dev/null +++ b/nemo_text_processing/text_normalization/kn/data/numbers/hundreds.tsv @@ -0,0 +1,18 @@ +೧ ನೂರ +೨ ಇನ್ನೂರ +೩ ಮುನ್ನೂರ +೪ ನಾನೂರ +೫ ಐನೂರ +೬ ಆರುನೂರ +೭ ಏಳುನೂರ +೮ ಎಂಟುನೂರ +೯ ಒಂಬೈನೂರ +1 ನೂರ +2 ಇನ್ನೂರ +3 ಮುನ್ನೂರ +4 ನಾನೂರ +5 ಐನೂರ +6 ಆರುನೂರ +7 ಏಳುನೂರ +8 ಎಂಟುನೂರ +9 ಒಂಬೈನೂರ diff --git a/nemo_text_processing/text_normalization/kn/data/numbers/teens_and_ties.tsv b/nemo_text_processing/text_normalization/kn/data/numbers/teens_and_ties.tsv new file mode 100644 index 000000000..d7cc5a427 --- /dev/null +++ b/nemo_text_processing/text_normalization/kn/data/numbers/teens_and_ties.tsv @@ -0,0 +1,90 @@ +೧೦ ಹತ್ತು +೧೧ ಹನ್ನೊಂದು +೧೨ ಹನ್ನೆರಡು +೧೩ ಹದಿಮೂರು +೧೪ ಹದಿನಾಲ್ಕು +೧೫ ಹದಿನೈದು +೧೬ ಹದಿನಾರು +೧೭ ಹದಿನೇಳು +೧೮ ಹದಿನೆಂಟು +೧೯ ಹತ್ತೊಂಬತ್ತು +೨೦ ಇಪ್ಪತ್ತು +೨೧ ಇಪ್ಪತ್ತೊಂದು +೨೨ ಇಪ್ಪತ್ತೆರಡು +೨೩ ಇಪ್ಪತ್ತಮೂರು +೨೪ ಇಪ್ಪತ್ತನಾಲ್ಕು +೨೫ ಇಪ್ಪತ್ತೈದು +೨೬ ಇಪ್ಪತ್ತಾರು +೨೭ ಇಪ್ಪತ್ತೇಳು +೨೮ ಇಪ್ಪತ್ತೆಂಟು +೨೯ ಇಪ್ಪತ್ತೊಂಬತ್ತು +೩೦ ಮೂವತ್ತು +೩೧ ಮೂವತ್ತೊಂದು +೩೨ ಮೂವತ್ತೆರಡು +೩೩ ಮೂವತ್ತಮೂರು +೩೪ ಮೂವತ್ತನಾಲ್ಕು +೩೫ ಮೂವತ್ತೈದು +೩೬ ಮೂವತ್ತಾರು +೩೭ ಮೂವತ್ತೇಳು +೩೮ ಮೂವತ್ತೆಂಟು +೩೯ ಮೂವತ್ತೊಂಬತ್ತು +೪೦ ನಲವತ್ತು +೪೧ ನಲವತ್ತೊಂದು +೪೨ ನಲವತ್ತೆರಡು +೪೩ ನಲವತ್ತಮೂರು +೪೪ ನಲವತ್ತನಾಲ್ಕು +೪೫ ನಲವತ್ತೈದು +೪೬ ನಲವತ್ತಾರು +೪೭ ನಲವತ್ತೇಳು +೪೮ ನಲವತ್ತೆಂಟು +೪೯ ನಲವತ್ತೊಂಬತ್ತು +೫೦ ಐವತ್ತು +೫೧ ಐವತ್ತೊಂದು +೫೨ ಐವತ್ತೆರಡು +೫೩ ಐವತ್ತಮೂರು +೫೪ ಐವತ್ತನಾಲ್ಕು +೫೫ ಐವತ್ತೈದು +೫೬ ಐವತ್ತಾರು +೫೭ ಐವತ್ತೇಳು +೫೮ ಐವತ್ತೆಂಟು +೫೯ ಐವತ್ತೊಂಬತ್ತು +೬೦ ಅರವತ್ತು +೬೧ ಅರವತ್ತೊಂದು +೬೨ ಅರವತ್ತೆರಡು +೬೩ ಅರವತ್ತಮೂರು +೬೪ ಅರವತ್ತನಾಲ್ಕು +೬೫ ಅರವತ್ತೈದು +೬೬ ಅರವತ್ತಾರು +೬೭ ಅರವತ್ತೇಳು +೬೮ ಅರವತ್ತೆಂಟು +೬೯ ಅರವತ್ತೊಂಬತ್ತು +೭೦ ಎಪ್ಪತ್ತು +೭೧ ಎಪ್ಪತ್ತೊಂದು +೭೨ ಎಪ್ಪತ್ತೆರಡು +೭೩ ಎಪ್ಪತ್ತಮೂರು +೭೪ ಎಪ್ಪತ್ತನಾಲ್ಕು +೭೫ ಎಪ್ಪತ್ತೈದು +೭೬ ಎಪ್ಪತ್ತಾರು +೭೭ ಎಪ್ಪತ್ತೇಳು +೭೮ ಎಪ್ಪತ್ತೆಂಟು +೭೯ ಎಪ್ಪತ್ತೊಂಬತ್ತು +೮೦ ಎಂಬತ್ತು +೮೧ ಎಂಬತ್ತೊಂದು +೮೨ ಎಂಬತ್ತೆರಡು +೮೩ ಎಂಬತ್ತಮೂರು +೮೪ ಎಂಬತ್ತನಾಲ್ಕು +೮೫ ಎಂಬತ್ತೈದು +೮೬ ಎಂಬತ್ತಾರು +೮೭ ಎಂಬತ್ತೇಳು +೮೮ ಎಂಬತ್ತೆಂಟು +೮೯ ಎಂಬತ್ತೊಂಬತ್ತು +೯೦ ತೊಂಬತ್ತು +೯೧ ತೊಂಬತ್ತೊಂದು +೯೨ ತೊಂಬತ್ತೆರಡು +೯೩ ತೊಂಬತ್ತಮೂರು +೯೪ ತೊಂಬತ್ತನಾಲ್ಕು +೯೫ ತೊಂಬತ್ತೈದು +೯೬ ತೊಂಬತ್ತಾರು +೯೭ ತೊಂಬತ್ತೇಳು +೯೮ ತೊಂಬತ್ತೆಂಟು +೯೯ ತೊಂಬತ್ತೊಂಬತ್ತು \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/kn/data/numbers/teens_and_ties_en.tsv b/nemo_text_processing/text_normalization/kn/data/numbers/teens_and_ties_en.tsv new file mode 100644 index 000000000..1a4a6fbd8 --- /dev/null +++ b/nemo_text_processing/text_normalization/kn/data/numbers/teens_and_ties_en.tsv @@ -0,0 +1,90 @@ +10 ಹತ್ತು +11 ಹನ್ನೊಂದು +12 ಹನ್ನೆರಡು +13 ಹದಿಮೂರು +14 ಹದಿನಾಲ್ಕು +15 ಹದಿನೈದು +16 ಹದಿನಾರು +17 ಹದಿನೇಳು +18 ಹದಿನೆಂಟು +19 ಹತ್ತೊಂಬತ್ತು +20 ಇಪ್ಪತ್ತು +21 ಇಪ್ಪತ್ತೊಂದು +22 ಇಪ್ಪತ್ತೆರಡು +23 ಇಪ್ಪತ್ತಮೂರು +24 ಇಪ್ಪತ್ತನಾಲ್ಕು +25 ಇಪ್ಪತ್ತೈದು +26 ಇಪ್ಪತ್ತಾರು +27 ಇಪ್ಪತ್ತೇಳು +28 ಇಪ್ಪತ್ತೆಂಟು +29 ಇಪ್ಪತ್ತೊಂಬತ್ತು +30 ಮೂವತ್ತು +31 ಮೂವತ್ತೊಂದು +32 ಮೂವತ್ತೆರಡು +33 ಮೂವತ್ತಮೂರು +34 ಮೂವತ್ತನಾಲ್ಕು +35 ಮೂವತ್ತೈದು +36 ಮೂವತ್ತಾರು +37 ಮೂವತ್ತೇಳು +38 ಮೂವತ್ತೆಂಟು +39 ಮೂವತ್ತೊಂಬತ್ತು +40 ನಲವತ್ತು +41 ನಲವತ್ತೊಂದು +42 ನಲವತ್ತೆರಡು +43 ನಲವತ್ತಮೂರು +44 ನಲವತ್ತನಾಲ್ಕು +45 ನಲವತ್ತೈದು +46 ನಲವತ್ತಾರು +47 ನಲವತ್ತೇಳು +48 ನಲವತ್ತೆಂಟು +49 ನಲವತ್ತೊಂಬತ್ತು +50 ಐವತ್ತು +51 ಐವತ್ತೊಂದು +52 ಐವತ್ತೆರಡು +53 ಐವತ್ತಮೂರು +54 ಐವತ್ತನಾಲ್ಕು +55 ಐವತ್ತೈದು +56 ಐವತ್ತಾರು +57 ಐವತ್ತೇಳು +58 ಐವತ್ತೆಂಟು +59 ಐವತ್ತೊಂಬತ್ತು +60 ಅರವತ್ತು +61 ಅರವತ್ತೊಂದು +62 ಅರವತ್ತೆರಡು +63 ಅರವತ್ತಮೂರು +64 ಅರವತ್ತನಾಲ್ಕು +65 ಅರವತ್ತೈದು +66 ಅರವತ್ತಾರು +67 ಅರವತ್ತೇಳು +68 ಅರವತ್ತೆಂಟು +69 ಅರವತ್ತೊಂಬತ್ತು +70 ಎಪ್ಪತ್ತು +71 ಎಪ್ಪತ್ತೊಂದು +72 ಎಪ್ಪತ್ತೆರಡು +73 ಎಪ್ಪತ್ತಮೂರು +74 ಎಪ್ಪತ್ತನಾಲ್ಕು +75 ಎಪ್ಪತ್ತೈದು +76 ಎಪ್ಪತ್ತಾರು +77 ಎಪ್ಪತ್ತೇಳು +78 ಎಪ್ಪತ್ತೆಂಟು +79 ಎಪ್ಪತ್ತೊಂಬತ್ತು +80 ಎಂಬತ್ತು +81 ಎಂಬತ್ತೊಂದು +82 ಎಂಬತ್ತೆರಡು +83 ಎಂಬತ್ತಮೂರು +84 ಎಂಬತ್ತನಾಲ್ಕು +85 ಎಂಬತ್ತೈದು +86 ಎಂಬತ್ತಾರು +87 ಎಂಬತ್ತೇಳು +88 ಎಂಬತ್ತೆಂಟು +89 ಎಂಬತ್ತೊಂಬತ್ತು +90 ತೊಂಬತ್ತು +91 ತೊಂಬತ್ತೊಂದು +92 ತೊಂಬತ್ತೆರಡು +93 ತೊಂಬತ್ತಮೂರು +94 ತೊಂಬತ್ತನಾಲ್ಕು +95 ತೊಂಬತ್ತೈದು +96 ತೊಂಬತ್ತಾರು +97 ತೊಂಬತ್ತೇಳು +98 ತೊಂಬತ್ತೆಂಟು +99 ತೊಂಬತ್ತೊಂಬತ್ತು \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/kn/data/numbers/zero.tsv b/nemo_text_processing/text_normalization/kn/data/numbers/zero.tsv new file mode 100644 index 000000000..6caf614a2 --- /dev/null +++ b/nemo_text_processing/text_normalization/kn/data/numbers/zero.tsv @@ -0,0 +1,2 @@ +0 ಸೊನ್ನೆ +೦ ಸೊನ್ನೆ diff --git a/nemo_text_processing/text_normalization/kn/graph_utils.py b/nemo_text_processing/text_normalization/kn/graph_utils.py new file mode 100644 index 000000000..5c5cadddd --- /dev/null +++ b/nemo_text_processing/text_normalization/kn/graph_utils.py @@ -0,0 +1,224 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import string +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.export import export +from pynini.lib import byte, pynutil, utf8 + +NEMO_CHAR = utf8.VALID_UTF8_CHAR +NEMO_DIGIT = byte.DIGIT + +NEMO_KN_DIGIT = pynini.union("೦", "೧", "೨", "೩", "೪", "೫", "೬", "೭", "೮", "೯").optimize() +NEMO_KN_NON_ZERO = pynini.union("೧", "೨", "೩", "೪", "೫", "೬", "೭", "೮", "೯").optimize() +NEMO_KN_ZERO = "೦" +# Combined Kannada and Arabic digits for graphs that need to accept both +NEMO_ALL_DIGIT = pynini.union(NEMO_KN_DIGIT, NEMO_DIGIT).optimize() +NEMO_ALL_ZERO = pynini.union("೦", "0").optimize() +NEMO_ALL_NON_ZERO = pynini.union(NEMO_KN_NON_ZERO, "1", "2", "3", "4", "5", "6", "7", "8", "9").optimize() + +KN_ONDUVARE = "ಒಂದೂವರೆ" # 1.5 +KN_ERADUVARE = "ಎರಡೂವರೆ" # 2.5 +KN_KALLU = "ಒಂದು ಕಾಲು" # quarter more (1.25) +KN_UVARE = "ಊವರೆ" # half more (X.5) +KN_KALU_KADIME = "ಕಾಲು ಕಡಿಮೆ" # quarter less (0.75) + +# Kannada decimal representations +KN_POINT_FIVE = ".೫" # .5 +KN_ONE_POINT_FIVE = "೧.೫" # 1.5 +KN_TWO_POINT_FIVE = "೨.೫" # 2.5 +KN_DECIMAL_25 = ".೨೫" # .25 +KN_DECIMAL_75 = ".೭೫" # .75 + +# Arabic/English decimal representations +EN_POINT_FIVE = ".5" +EN_ONE_POINT_FIVE = "1.5" +EN_TWO_POINT_FIVE = "2.5" +EN_DECIMAL_25 = ".25" +EN_DECIMAL_75 = ".75" + +# Combined Kannada and English decimal patterns +POINT_FIVE = pynini.union(KN_POINT_FIVE, EN_POINT_FIVE).optimize() +ONE_POINT_FIVE = pynini.union(KN_ONE_POINT_FIVE, EN_ONE_POINT_FIVE).optimize() +TWO_POINT_FIVE = pynini.union(KN_TWO_POINT_FIVE, EN_TWO_POINT_FIVE).optimize() +DECIMAL_25 = pynini.union(KN_DECIMAL_25, EN_DECIMAL_25).optimize() +DECIMAL_75 = pynini.union(KN_DECIMAL_75, EN_DECIMAL_75).optimize() + +# Symbol constants +KN_BY = "ಬಾಯಿ" +LOWERCASE_X = "x" +UPPERCASE_X = "X" +ASTERISK = "*" +HYPHEN = "-" +SLASH = "/" +COMMA = "," +PERIOD = "." +HI_PERIOD = "।" + +NEMO_LOWER = pynini.union(*string.ascii_lowercase).optimize() +NEMO_UPPER = pynini.union(*string.ascii_uppercase).optimize() +NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize() +NEMO_HEX = pynini.union(*string.hexdigits).optimize() +NEMO_NON_BREAKING_SPACE = u"\u00a0" +NEMO_SPACE = " " +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", u"\u00a0").optimize() +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() +TO_LOWER = pynini.union(*[pynini.cross(x, y) for x, y in zip(string.ascii_uppercase, string.ascii_lowercase)]) +TO_UPPER = pynini.invert(TO_LOWER) +NEMO_SIGMA = pynini.closure(NEMO_CHAR) + + +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +delete_zero_or_one_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE, 0, 1)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") +delete_preserve_order = pynini.closure( + pynutil.delete(" preserve_order: true") + | (pynutil.delete(" field_order: \"") + NEMO_NOT_QUOTE + pynutil.delete("\"")) +) + + +MIN_NEG_WEIGHT = -0.0001 +MIN_POS_WEIGHT = 0.0001 +INPUT_CASED = "cased" +INPUT_LOWER_CASED = "lower_cased" +MINUS = pynini.union(" ಋಣಾತ್ಮಕ ", " ಋಣಾತ್ಮಕ ").optimize() + + +def capitalized_input_graph( + graph: 'pynini.FstLike', original_graph_weight: float = None, capitalized_graph_weight: float = None +) -> 'pynini.FstLike': + """ + Allow graph input to be capitalized, e.g. for ITN) + + Args: + graph: FstGraph + original_graph_weight: weight to add to the original `graph` + capitalized_graph_weight: weight to add to the capitalized graph + """ + capitalized_graph = pynini.compose(TO_LOWER + NEMO_SIGMA, graph).optimize() + + if original_graph_weight is not None: + graph = pynutil.add_weight(graph, weight=original_graph_weight) + + if capitalized_graph_weight is not None: + capitalized_graph = pynutil.add_weight(capitalized_graph, weight=capitalized_graph_weight) + + graph |= capitalized_graph + return graph + + +def generator_main(file_name: str, graphs: Dict[str, 'pynini.FstLike']): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logging.info(f'Created {file_name}') + + +def convert_space(fst) -> 'pynini.FstLike': + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA) + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + '/grammars/' + kind + '/' + name + '.far') + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> 'pynini.FstLike': + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> 'pynini.FstLike': + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> 'pynini.FstLike': + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/text_normalization/kn/taggers/__init__.py b/nemo_text_processing/text_normalization/kn/taggers/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/kn/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/kn/taggers/cardinal.py b/nemo_text_processing/text_normalization/kn/taggers/cardinal.py new file mode 100644 index 000000000..104a8e42e --- /dev/null +++ b/nemo_text_processing/text_normalization/kn/taggers/cardinal.py @@ -0,0 +1,772 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDIIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.kn.graph_utils import ( + NEMO_ALL_DIGIT, + NEMO_ALL_ZERO, + GraphFst, + insert_space, +) +from nemo_text_processing.text_normalization.kn.utils import get_abs_path + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals in Kannada, e.g. + -23 -> cardinal { negative: "true" integer: "ಇಪ್ಪತ್ತಮೂರು" } + 123 -> cardinal { integer: "ನೂರ ಇಪ್ಪತ್ತಮೂರು" } + 500 -> cardinal { integer: "ಐನೂರು" } + 1001 -> cardinal { integer: "ಒಂದು ಸಾವಿರದ ಒಂದು" } + + Uses contracted hundred forms (ಐನೂರು, ಇನ್ನೂರು) and genitive connectors + (ಸಾವಿರದ, ಲಕ್ಷದ, ಕೋಟಿಯ) when followed by non-zero remainders. + + Numbers above ಕೋಟಿ (10^7) use ಕೋಟಿ as base unit: + 10^9 -> ನೂರು ಕೋಟಿ (not ಅರಬ್) + 10^11 -> ಹತ್ತು ಸಾವಿರ ಕೋಟಿ (not ಖರಬ್) + + Coverage limit: Numbers up to 99 lakh crore (99,99,999 crore or ~10^14) are supported. + Numbers with crore coefficients > 99,99,999 (i.e., requiring 8+ digit coefficient + normalization) will fall through as raw text. This covers practical TTS use cases. + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True, lm: bool = False): + super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + + digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + teens_ties_kn = pynini.string_file(get_abs_path("data/numbers/teens_and_ties.tsv")) + teens_ties_en = pynini.string_file(get_abs_path("data/numbers/teens_and_ties_en.tsv")) + teens_ties = pynini.union(teens_ties_kn, teens_ties_en) + teens_and_ties = pynutil.add_weight(teens_ties, -0.1) + + self.digit = digit + self.zero = zero + self.teens_and_ties = teens_and_ties + + # Single digit graph for digit-by-digit reading + single_digit_graph = digit | zero + self.single_digits_graph = single_digit_graph + pynini.closure(insert_space + single_digit_graph) + + # Helper for deleting zeros + delete_zero = pynutil.add_weight(pynutil.delete(NEMO_ALL_ZERO), -0.1) + + # ============================================================ + # HUNDREDS (100-999) + # Using contracted forms: ನೂರ/ನೂರು, ಐನೂರ/ಐನೂರು, ಒಂಬೈನೂರ/ಒಂಬೈನೂರು + # ============================================================ + hundreds = pynini.string_file(get_abs_path("data/numbers/hundreds.tsv")) + self.hundreds = hundreds + suffix_u = pynutil.insert("ು") + + # Standalone: 500 -> ಐನೂರು (with ು) + graph_hundreds_standalone = hundreds + (delete_zero**2) + suffix_u + # With digit: 501 -> ಐನೂರ ಒಂದು + graph_hundreds_with_digit = hundreds + delete_zero + insert_space + digit + # With teens/ties: 523 -> ಐನೂರ ಇಪ್ಪತ್ತಮೂರು + graph_hundreds_with_teens = hundreds + insert_space + teens_ties + + graph_hundreds = graph_hundreds_standalone | graph_hundreds_with_digit | graph_hundreds_with_teens + graph_hundreds.optimize() + self.graph_hundreds = graph_hundreds + + # ============================================================ + # THOUSANDS (1000-99999) - ಸಾವಿರ/ಸಾವಿರದ + # Standalone: ಸಾವಿರ | With remainder: ಸಾವಿರದ + # ============================================================ + suffix_thousand_standalone = pynutil.insert(" ಸಾವಿರ") + suffix_thousand_genitive = pynutil.insert(" ಸಾವಿರದ") + + # Standalone thousands (X000, XX000) + graph_thousands_standalone = digit + (delete_zero**3) + suffix_thousand_standalone + graph_ten_thousands_standalone = teens_and_ties + (delete_zero**3) + suffix_thousand_standalone + + # Thousands with remainder - use genitive ಸಾವಿರದ + # X00Y, X0YZ, XYZW patterns + graph_thousands_with_digit = digit + (delete_zero**2) + suffix_thousand_genitive + insert_space + digit + graph_thousands_with_teens = digit + delete_zero + suffix_thousand_genitive + insert_space + teens_ties + graph_thousands_with_hundreds = digit + suffix_thousand_genitive + insert_space + graph_hundreds + + graph_ten_thousands_with_digit = ( + teens_and_ties + (delete_zero**2) + suffix_thousand_genitive + insert_space + digit + ) + graph_ten_thousands_with_teens = ( + teens_and_ties + delete_zero + suffix_thousand_genitive + insert_space + teens_ties + ) + graph_ten_thousands_with_hundreds = teens_and_ties + suffix_thousand_genitive + insert_space + graph_hundreds + + graph_thousands = ( + graph_thousands_standalone + | graph_thousands_with_digit + | graph_thousands_with_teens + | graph_thousands_with_hundreds + ) + graph_thousands.optimize() + self.graph_thousands = graph_thousands + + graph_ten_thousands = ( + graph_ten_thousands_standalone + | graph_ten_thousands_with_digit + | graph_ten_thousands_with_teens + | graph_ten_thousands_with_hundreds + ) + graph_ten_thousands.optimize() + self.graph_ten_thousands = graph_ten_thousands + + # ============================================================ + # LAKHS (100000-9999999) - ಲಕ್ಷ/ಲಕ್ಷದ + # ============================================================ + suffix_lakh_standalone = pynutil.insert(" ಲಕ್ಷ") + suffix_lakh_genitive = pynutil.insert(" ಲಕ್ಷದ") + + # Standalone lakhs + graph_lakhs_standalone = digit + (delete_zero**5) + suffix_lakh_standalone + graph_ten_lakhs_standalone = teens_and_ties + (delete_zero**5) + suffix_lakh_standalone + + # Lakhs with remainder + graph_lakhs_with_digit = digit + (delete_zero**4) + suffix_lakh_genitive + insert_space + digit + graph_lakhs_with_teens = digit + (delete_zero**3) + suffix_lakh_genitive + insert_space + teens_ties + graph_lakhs_with_hundreds = digit + (delete_zero**2) + suffix_lakh_genitive + insert_space + graph_hundreds + graph_lakhs_with_thousands = digit + delete_zero + suffix_lakh_genitive + insert_space + graph_thousands + graph_lakhs_with_ten_thousands = digit + suffix_lakh_genitive + insert_space + graph_ten_thousands + + graph_ten_lakhs_with_digit = teens_and_ties + (delete_zero**4) + suffix_lakh_genitive + insert_space + digit + graph_ten_lakhs_with_teens = ( + teens_and_ties + (delete_zero**3) + suffix_lakh_genitive + insert_space + teens_ties + ) + graph_ten_lakhs_with_hundreds = ( + teens_and_ties + (delete_zero**2) + suffix_lakh_genitive + insert_space + graph_hundreds + ) + graph_ten_lakhs_with_thousands = ( + teens_and_ties + delete_zero + suffix_lakh_genitive + insert_space + graph_thousands + ) + graph_ten_lakhs_with_ten_thousands = teens_and_ties + suffix_lakh_genitive + insert_space + graph_ten_thousands + + graph_lakhs = ( + graph_lakhs_standalone + | graph_lakhs_with_digit + | graph_lakhs_with_teens + | graph_lakhs_with_hundreds + | graph_lakhs_with_thousands + | graph_lakhs_with_ten_thousands + ) + graph_lakhs.optimize() + self.graph_lakhs = graph_lakhs + + graph_ten_lakhs = ( + graph_ten_lakhs_standalone + | graph_ten_lakhs_with_digit + | graph_ten_lakhs_with_teens + | graph_ten_lakhs_with_hundreds + | graph_ten_lakhs_with_thousands + | graph_ten_lakhs_with_ten_thousands + ) + graph_ten_lakhs.optimize() + self.graph_ten_lakhs = graph_ten_lakhs + + # ============================================================ + # CRORES (10000000-999999999) - ಕೋಟಿ/ಕೋಟಿಯ + # Note: ಕೋಟಿ ends in vowel, so genitive is ಕೋಟಿಯ (not ಕೋಟಿದ) + # ============================================================ + suffix_crore_standalone = pynutil.insert(" ಕೋಟಿ") + suffix_crore_genitive = pynutil.insert(" ಕೋಟಿಯ") + + # Standalone crores + graph_crores_standalone = digit + (delete_zero**7) + suffix_crore_standalone + graph_ten_crores_standalone = teens_and_ties + (delete_zero**7) + suffix_crore_standalone + + # Crores with remainder + graph_crores_with_digit = digit + (delete_zero**6) + suffix_crore_genitive + insert_space + digit + graph_crores_with_teens = digit + (delete_zero**5) + suffix_crore_genitive + insert_space + teens_ties + graph_crores_with_hundreds = digit + (delete_zero**4) + suffix_crore_genitive + insert_space + graph_hundreds + graph_crores_with_thousands = digit + (delete_zero**3) + suffix_crore_genitive + insert_space + graph_thousands + graph_crores_with_ten_thousands = ( + digit + (delete_zero**2) + suffix_crore_genitive + insert_space + graph_ten_thousands + ) + graph_crores_with_lakhs = digit + delete_zero + suffix_crore_genitive + insert_space + graph_lakhs + graph_crores_with_ten_lakhs = digit + suffix_crore_genitive + insert_space + graph_ten_lakhs + + graph_ten_crores_with_digit = teens_and_ties + (delete_zero**6) + suffix_crore_genitive + insert_space + digit + graph_ten_crores_with_teens = ( + teens_and_ties + (delete_zero**5) + suffix_crore_genitive + insert_space + teens_ties + ) + graph_ten_crores_with_hundreds = ( + teens_and_ties + (delete_zero**4) + suffix_crore_genitive + insert_space + graph_hundreds + ) + graph_ten_crores_with_thousands = ( + teens_and_ties + (delete_zero**3) + suffix_crore_genitive + insert_space + graph_thousands + ) + graph_ten_crores_with_ten_thousands = ( + teens_and_ties + (delete_zero**2) + suffix_crore_genitive + insert_space + graph_ten_thousands + ) + graph_ten_crores_with_lakhs = teens_and_ties + delete_zero + suffix_crore_genitive + insert_space + graph_lakhs + graph_ten_crores_with_ten_lakhs = teens_and_ties + suffix_crore_genitive + insert_space + graph_ten_lakhs + + graph_crores = ( + graph_crores_standalone + | graph_crores_with_digit + | graph_crores_with_teens + | graph_crores_with_hundreds + | graph_crores_with_thousands + | graph_crores_with_ten_thousands + | graph_crores_with_lakhs + | graph_crores_with_ten_lakhs + ) + graph_crores.optimize() + self.graph_crores = graph_crores + + graph_ten_crores = ( + graph_ten_crores_standalone + | graph_ten_crores_with_digit + | graph_ten_crores_with_teens + | graph_ten_crores_with_hundreds + | graph_ten_crores_with_thousands + | graph_ten_crores_with_ten_thousands + | graph_ten_crores_with_lakhs + | graph_ten_crores_with_ten_lakhs + ) + graph_ten_crores.optimize() + self.graph_ten_crores = graph_ten_crores + + # ============================================================ + # HUNDRED CRORES and above (10^9+) + # Instead of ಅರಬ್/ಖರಬ್, use natural Kannada: ನೂರು ಕೋಟಿ, ಸಾವಿರ ಕೋಟಿ + # ============================================================ + + # Combined graph for any crore remainder (1 to 99,99,99,999) + graph_crore_remainder = ( + digit + | teens_and_ties + | graph_hundreds + | graph_thousands + | graph_ten_thousands + | graph_lakhs + | graph_ten_lakhs + | graph_crores + | graph_ten_crores + ) + + # Hundred crores (10^9): 100-999 ಕೋಟಿ with any remainder + graph_hundred_crores_standalone = graph_hundreds + (delete_zero**7) + suffix_crore_standalone + graph_hundred_crores_with_digit = ( + graph_hundreds + (delete_zero**6) + suffix_crore_genitive + insert_space + digit + ) + graph_hundred_crores_with_teens = ( + graph_hundreds + (delete_zero**5) + suffix_crore_genitive + insert_space + teens_ties + ) + graph_hundred_crores_with_hundreds = ( + graph_hundreds + (delete_zero**4) + suffix_crore_genitive + insert_space + graph_hundreds + ) + graph_hundred_crores_with_thousands = ( + graph_hundreds + (delete_zero**3) + suffix_crore_genitive + insert_space + graph_thousands + ) + graph_hundred_crores_with_ten_thousands = ( + graph_hundreds + (delete_zero**2) + suffix_crore_genitive + insert_space + graph_ten_thousands + ) + graph_hundred_crores_with_lakhs = ( + graph_hundreds + delete_zero + suffix_crore_genitive + insert_space + graph_lakhs + ) + graph_hundred_crores_with_ten_lakhs = graph_hundreds + suffix_crore_genitive + insert_space + graph_ten_lakhs + + graph_hundred_crores = ( + graph_hundred_crores_standalone + | graph_hundred_crores_with_digit + | graph_hundred_crores_with_teens + | graph_hundred_crores_with_hundreds + | graph_hundred_crores_with_thousands + | graph_hundred_crores_with_ten_thousands + | graph_hundred_crores_with_lakhs + | graph_hundred_crores_with_ten_lakhs + ) + graph_hundred_crores.optimize() + + # Thousand crores (10^10): 1000-9999 ಕೋಟಿ + # Use coefficient patterns that output ಸಾವಿರದ (not ಸಾವಿರ ಕೋಟಿ) and append ಕೋಟಿ at end + # This prevents ಕೋಟಿ from appearing twice + + # For X000 crore (e.g., 1000 crore = 10000000000) + graph_thousand_crores_standalone = digit + (delete_zero**10) + pynutil.insert(" ಸಾವಿರ ಕೋಟಿ") + + # For X000 crore with sub-crore remainder (uses "ಸಾವಿರ ಕೋಟಿಯ") + suffix_thousand_crore_genitive = pynutil.insert(" ಸಾವಿರ ಕೋಟಿಯ") + graph_thousand_crores_with_digit = ( + digit + (delete_zero**9) + suffix_thousand_crore_genitive + insert_space + digit + ) + graph_thousand_crores_with_teens = ( + digit + (delete_zero**8) + suffix_thousand_crore_genitive + insert_space + teens_ties + ) + graph_thousand_crores_with_hundreds = ( + digit + (delete_zero**7) + suffix_thousand_crore_genitive + insert_space + graph_hundreds + ) + graph_thousand_crores_with_thousands = ( + digit + (delete_zero**6) + suffix_thousand_crore_genitive + insert_space + graph_thousands + ) + graph_thousand_crores_with_ten_thousands = ( + digit + (delete_zero**5) + suffix_thousand_crore_genitive + insert_space + graph_ten_thousands + ) + graph_thousand_crores_with_lakhs = ( + digit + (delete_zero**4) + suffix_thousand_crore_genitive + insert_space + graph_lakhs + ) + graph_thousand_crores_with_ten_lakhs = ( + digit + (delete_zero**3) + suffix_thousand_crore_genitive + insert_space + graph_ten_lakhs + ) + + # For X00Y, X0YZ, XYZW crore patterns (e.g., 1001 crore, 1234 crore) + # Use "ಸಾವಿರದ" (genitive), not "ಸಾವಿರ ಕೋಟಿ", then append ಕೋಟಿ at end + # X00Y crore standalone (e.g., 1001 crore = 10010000000) + graph_thousand_crores_x00y = ( + digit + + (delete_zero**2) + + suffix_thousand_genitive + + insert_space + + digit + + (delete_zero**7) + + suffix_crore_standalone + ) + # X0YZ crore standalone (e.g., 1023 crore) + graph_thousand_crores_x0yz = ( + digit + + delete_zero + + suffix_thousand_genitive + + insert_space + + teens_ties + + (delete_zero**7) + + suffix_crore_standalone + ) + # XYZW crore standalone (e.g., 1234 crore) + graph_thousand_crores_xyzw = ( + digit + + suffix_thousand_genitive + + insert_space + + graph_hundreds + + (delete_zero**7) + + suffix_crore_standalone + ) + + # X00Y, X0YZ, XYZW crore with sub-crore remainder + graph_thousand_crores_x00y_with_rem = ( + digit + + (delete_zero**2) + + suffix_thousand_genitive + + insert_space + + digit + + suffix_crore_genitive + + insert_space + + graph_ten_lakhs + ) + graph_thousand_crores_x0yz_with_rem = ( + digit + + delete_zero + + suffix_thousand_genitive + + insert_space + + teens_ties + + suffix_crore_genitive + + insert_space + + graph_ten_lakhs + ) + graph_thousand_crores_xyzw_with_rem = ( + digit + + suffix_thousand_genitive + + insert_space + + graph_hundreds + + suffix_crore_genitive + + insert_space + + graph_ten_lakhs + ) + + graph_thousand_crores = ( + graph_thousand_crores_standalone + | graph_thousand_crores_with_digit + | graph_thousand_crores_with_teens + | graph_thousand_crores_with_hundreds + | graph_thousand_crores_with_thousands + | graph_thousand_crores_with_ten_thousands + | graph_thousand_crores_with_lakhs + | graph_thousand_crores_with_ten_lakhs + | graph_thousand_crores_x00y + | graph_thousand_crores_x0yz + | graph_thousand_crores_xyzw + | graph_thousand_crores_x00y_with_rem + | graph_thousand_crores_x0yz_with_rem + | graph_thousand_crores_xyzw_with_rem + ) + graph_thousand_crores.optimize() + + # Ten thousand crores (10^11): 10000-99999 ಕೋಟಿ + # Use coefficient patterns with ಸಾವಿರದ and append ಕೋಟಿ at end + + # For XY000 crore (e.g., 12000 crore = 120000000000) + graph_ten_thousand_crores_standalone = teens_and_ties + (delete_zero**10) + pynutil.insert(" ಸಾವಿರ ಕೋಟಿ") + + # For XY000 crore with sub-crore remainder + suffix_ten_thousand_crore_genitive = pynutil.insert(" ಸಾವಿರ ಕೋಟಿಯ") + graph_ten_thousand_crores_with_digit = ( + teens_and_ties + (delete_zero**9) + suffix_ten_thousand_crore_genitive + insert_space + digit + ) + graph_ten_thousand_crores_with_teens = ( + teens_and_ties + (delete_zero**8) + suffix_ten_thousand_crore_genitive + insert_space + teens_ties + ) + graph_ten_thousand_crores_with_hundreds = ( + teens_and_ties + (delete_zero**7) + suffix_ten_thousand_crore_genitive + insert_space + graph_hundreds + ) + graph_ten_thousand_crores_with_thousands = ( + teens_and_ties + (delete_zero**6) + suffix_ten_thousand_crore_genitive + insert_space + graph_thousands + ) + graph_ten_thousand_crores_with_ten_thousands = ( + teens_and_ties + (delete_zero**5) + suffix_ten_thousand_crore_genitive + insert_space + graph_ten_thousands + ) + graph_ten_thousand_crores_with_lakhs = ( + teens_and_ties + (delete_zero**4) + suffix_ten_thousand_crore_genitive + insert_space + graph_lakhs + ) + graph_ten_thousand_crores_with_ten_lakhs = ( + teens_and_ties + (delete_zero**3) + suffix_ten_thousand_crore_genitive + insert_space + graph_ten_lakhs + ) + + # For XY00Z, XY0ZW, XYZWV crore patterns (e.g., 12001, 12034, 12345 crore) + # Use "ಸಾವಿರದ" (genitive), not "ಸಾವಿರ ಕೋಟಿ", then append ಕೋಟಿ at end + # XY00Z crore standalone + graph_ten_thousand_crores_xy00z = ( + teens_and_ties + + (delete_zero**2) + + suffix_thousand_genitive + + insert_space + + digit + + (delete_zero**7) + + suffix_crore_standalone + ) + # XY0ZW crore standalone + graph_ten_thousand_crores_xy0zw = ( + teens_and_ties + + delete_zero + + suffix_thousand_genitive + + insert_space + + teens_ties + + (delete_zero**7) + + suffix_crore_standalone + ) + # XYZWV crore standalone (e.g., 12345 crore) + graph_ten_thousand_crores_xyzwv = ( + teens_and_ties + + suffix_thousand_genitive + + insert_space + + graph_hundreds + + (delete_zero**7) + + suffix_crore_standalone + ) + + # With sub-crore remainder + graph_ten_thousand_crores_xy00z_with_rem = ( + teens_and_ties + + (delete_zero**2) + + suffix_thousand_genitive + + insert_space + + digit + + suffix_crore_genitive + + insert_space + + graph_ten_lakhs + ) + graph_ten_thousand_crores_xy0zw_with_rem = ( + teens_and_ties + + delete_zero + + suffix_thousand_genitive + + insert_space + + teens_ties + + suffix_crore_genitive + + insert_space + + graph_ten_lakhs + ) + graph_ten_thousand_crores_xyzwv_with_rem = ( + teens_and_ties + + suffix_thousand_genitive + + insert_space + + graph_hundreds + + suffix_crore_genitive + + insert_space + + graph_ten_lakhs + ) + + graph_ten_thousand_crores = ( + graph_ten_thousand_crores_standalone + | graph_ten_thousand_crores_with_digit + | graph_ten_thousand_crores_with_teens + | graph_ten_thousand_crores_with_hundreds + | graph_ten_thousand_crores_with_thousands + | graph_ten_thousand_crores_with_ten_thousands + | graph_ten_thousand_crores_with_lakhs + | graph_ten_thousand_crores_with_ten_lakhs + | graph_ten_thousand_crores_xy00z + | graph_ten_thousand_crores_xy0zw + | graph_ten_thousand_crores_xyzwv + | graph_ten_thousand_crores_xy00z_with_rem + | graph_ten_thousand_crores_xy0zw_with_rem + | graph_ten_thousand_crores_xyzwv_with_rem + ) + graph_ten_thousand_crores.optimize() + + # Lakh crores (10^12): 1-9 lakh ಕೋಟಿ + # Use "ಲಕ್ಷದ" (genitive) for coefficient composition, append ಕೋಟಿ at end + + # For X00000 crore (e.g., 1 lakh crore = 1000000000000) + graph_lakh_crores_standalone = digit + (delete_zero**12) + pynutil.insert(" ಲಕ್ಷ ಕೋಟಿ") + + # For X00000 crore with sub-crore remainder (uses "ಲಕ್ಷ ಕೋಟಿಯ") + suffix_lakh_crore_genitive = pynutil.insert(" ಲಕ್ಷ ಕೋಟಿಯ") + graph_lakh_crores_with_digit = digit + (delete_zero**11) + suffix_lakh_crore_genitive + insert_space + digit + graph_lakh_crores_with_teens = ( + digit + (delete_zero**10) + suffix_lakh_crore_genitive + insert_space + teens_ties + ) + graph_lakh_crores_with_hundreds = ( + digit + (delete_zero**9) + suffix_lakh_crore_genitive + insert_space + graph_hundreds + ) + graph_lakh_crores_with_thousands = ( + digit + (delete_zero**8) + suffix_lakh_crore_genitive + insert_space + graph_thousands + ) + graph_lakh_crores_with_ten_thousands = ( + digit + (delete_zero**7) + suffix_lakh_crore_genitive + insert_space + graph_ten_thousands + ) + graph_lakh_crores_with_lakhs = ( + digit + (delete_zero**6) + suffix_lakh_crore_genitive + insert_space + graph_lakhs + ) + graph_lakh_crores_with_ten_lakhs = ( + digit + (delete_zero**5) + suffix_lakh_crore_genitive + insert_space + graph_ten_lakhs + ) + + # For X000YZ, X00YZW, etc. crore patterns (coefficient > 1 lakh) + # Use "ಲಕ್ಷದ" for coefficient, append ಕೋಟಿ at end (avoid crore duplication) + # X000Y crore (e.g., 100100 crore = 1,00,100 crore = 1 lakh 100 crore) + graph_lakh_crores_x000y = ( + digit + + (delete_zero**4) + + suffix_lakh_genitive + + insert_space + + digit + + (delete_zero**7) + + suffix_crore_standalone + ) + graph_lakh_crores_x00yz = ( + digit + + (delete_zero**3) + + suffix_lakh_genitive + + insert_space + + teens_ties + + (delete_zero**7) + + suffix_crore_standalone + ) + graph_lakh_crores_x0yzw = ( + digit + + (delete_zero**2) + + suffix_lakh_genitive + + insert_space + + graph_hundreds + + (delete_zero**7) + + suffix_crore_standalone + ) + graph_lakh_crores_xyzwv = ( + digit + + delete_zero + + suffix_lakh_genitive + + insert_space + + graph_thousands + + (delete_zero**7) + + suffix_crore_standalone + ) + graph_lakh_crores_full = ( + digit + + suffix_lakh_genitive + + insert_space + + graph_ten_thousands + + (delete_zero**7) + + suffix_crore_standalone + ) + + graph_lakh_crores = ( + graph_lakh_crores_standalone + | graph_lakh_crores_with_digit + | graph_lakh_crores_with_teens + | graph_lakh_crores_with_hundreds + | graph_lakh_crores_with_thousands + | graph_lakh_crores_with_ten_thousands + | graph_lakh_crores_with_lakhs + | graph_lakh_crores_with_ten_lakhs + | graph_lakh_crores_x000y + | graph_lakh_crores_x00yz + | graph_lakh_crores_x0yzw + | graph_lakh_crores_xyzwv + | graph_lakh_crores_full + ) + graph_lakh_crores.optimize() + + # Ten lakh crores (10^13): 10-99 lakh ಕೋಟಿ + # Use "ಲಕ್ಷದ" for coefficient composition, append ಕೋಟಿ at end + + # For XY00000 crore (e.g., 10 lakh crore) + graph_ten_lakh_crores_standalone = teens_and_ties + (delete_zero**12) + pynutil.insert(" ಲಕ್ಷ ಕೋಟಿ") + + # With sub-crore remainder + suffix_ten_lakh_crore_genitive = pynutil.insert(" ಲಕ್ಷ ಕೋಟಿಯ") + graph_ten_lakh_crores_with_digit = ( + teens_and_ties + (delete_zero**11) + suffix_ten_lakh_crore_genitive + insert_space + digit + ) + graph_ten_lakh_crores_with_teens = ( + teens_and_ties + (delete_zero**10) + suffix_ten_lakh_crore_genitive + insert_space + teens_ties + ) + graph_ten_lakh_crores_with_hundreds = ( + teens_and_ties + (delete_zero**9) + suffix_ten_lakh_crore_genitive + insert_space + graph_hundreds + ) + graph_ten_lakh_crores_with_thousands = ( + teens_and_ties + (delete_zero**8) + suffix_ten_lakh_crore_genitive + insert_space + graph_thousands + ) + graph_ten_lakh_crores_with_ten_thousands = ( + teens_and_ties + (delete_zero**7) + suffix_ten_lakh_crore_genitive + insert_space + graph_ten_thousands + ) + graph_ten_lakh_crores_with_lakhs = ( + teens_and_ties + (delete_zero**6) + suffix_ten_lakh_crore_genitive + insert_space + graph_lakhs + ) + graph_ten_lakh_crores_with_ten_lakhs = ( + teens_and_ties + (delete_zero**5) + suffix_ten_lakh_crore_genitive + insert_space + graph_ten_lakhs + ) + + # For XY000Z, XY00ZW, etc. crore patterns + graph_ten_lakh_crores_xy000z = ( + teens_and_ties + + (delete_zero**4) + + suffix_lakh_genitive + + insert_space + + digit + + (delete_zero**7) + + suffix_crore_standalone + ) + graph_ten_lakh_crores_xy00zw = ( + teens_and_ties + + (delete_zero**3) + + suffix_lakh_genitive + + insert_space + + teens_ties + + (delete_zero**7) + + suffix_crore_standalone + ) + graph_ten_lakh_crores_xy0zwv = ( + teens_and_ties + + (delete_zero**2) + + suffix_lakh_genitive + + insert_space + + graph_hundreds + + (delete_zero**7) + + suffix_crore_standalone + ) + graph_ten_lakh_crores_xyzwvu = ( + teens_and_ties + + delete_zero + + suffix_lakh_genitive + + insert_space + + graph_thousands + + (delete_zero**7) + + suffix_crore_standalone + ) + graph_ten_lakh_crores_full = ( + teens_and_ties + + suffix_lakh_genitive + + insert_space + + graph_ten_thousands + + (delete_zero**7) + + suffix_crore_standalone + ) + + graph_ten_lakh_crores = ( + graph_ten_lakh_crores_standalone + | graph_ten_lakh_crores_with_digit + | graph_ten_lakh_crores_with_teens + | graph_ten_lakh_crores_with_hundreds + | graph_ten_lakh_crores_with_thousands + | graph_ten_lakh_crores_with_ten_thousands + | graph_ten_lakh_crores_with_lakhs + | graph_ten_lakh_crores_with_ten_lakhs + | graph_ten_lakh_crores_xy000z + | graph_ten_lakh_crores_xy00zw + | graph_ten_lakh_crores_xy0zwv + | graph_ten_lakh_crores_xyzwvu + | graph_ten_lakh_crores_full + ) + graph_ten_lakh_crores.optimize() + + # ============================================================ + # FINAL GRAPH COMPOSITION + # ============================================================ + graph_without_leading_zeros = ( + digit + | zero + | teens_and_ties + | graph_hundreds + | graph_thousands + | graph_ten_thousands + | graph_lakhs + | graph_ten_lakhs + | graph_crores + | graph_ten_crores + | graph_hundred_crores + | graph_thousand_crores + | graph_ten_thousand_crores + | graph_lakh_crores + | graph_ten_lakh_crores + ) + self.graph_without_leading_zeros = graph_without_leading_zeros.optimize() + + # Handle numbers with leading zeros by reading digit-by-digit + cardinal_with_leading_zeros = pynini.compose( + NEMO_ALL_ZERO + pynini.closure(NEMO_ALL_DIGIT), self.single_digits_graph + ) + cardinal_with_leading_zeros = pynutil.add_weight(cardinal_with_leading_zeros, 0.5) + + # Full graph including leading zeros (without commas) + graph_no_commas = graph_without_leading_zeros | cardinal_with_leading_zeros + + # ============================================================ + # COMMA-SEPARATED NUMBERS (strict validation) + # ============================================================ + delete_comma = pynutil.delete(",") + + def exactly_n_digits(n): + return pynini.closure(NEMO_ALL_DIGIT, n, n) + + # Western format: 1,000 | 1,000,000 + western_format = pynini.closure(NEMO_ALL_DIGIT, 1, 3) + pynini.closure(delete_comma + exactly_n_digits(3), 1) + + # Indian format: 1,00,000 | 12,34,567 + indian_format = ( + pynini.closure(NEMO_ALL_DIGIT, 1, 2) + + pynini.closure(delete_comma + exactly_n_digits(2)) + + delete_comma + + exactly_n_digits(3) + ) + + comma_number = western_format | indian_format + cardinal_with_commas = pynini.compose(comma_number, graph_without_leading_zeros) + cardinal_with_commas = pynutil.add_weight(cardinal_with_commas, 0.1) + + # Final graph + final_graph = graph_no_commas | cardinal_with_commas + + optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + + self.final_graph = final_graph.optimize() + final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.final_graph + pynutil.insert("\"") + final_graph = self.add_tokens(final_graph) + self.fst = final_graph diff --git a/nemo_text_processing/text_normalization/kn/taggers/punctuation.py b/nemo_text_processing/text_normalization/kn/taggers/punctuation.py new file mode 100644 index 000000000..16347c129 --- /dev/null +++ b/nemo_text_processing/text_normalization/kn/taggers/punctuation.py @@ -0,0 +1,48 @@ +import sys +from unicodedata import category + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.kn.graph_utils import NEMO_NOT_SPACE, NEMO_SIGMA, GraphFst + + +class PunctuationFst(GraphFst): + """ + Finite state transducer for classifying punctuation + e.g. a, -> tokens { name: "a" } tokens { name: "," } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transductions are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="punctuation", kind="classify", deterministic=deterministic) + s = "!#%&\'()*+,-./:;<=>?@^_`{|}~\"" + + punct_symbols_to_exclude = ["[", "]"] + punct_unicode = [ + chr(i) + for i in range(sys.maxunicode) + if category(chr(i)).startswith("P") and chr(i) not in punct_symbols_to_exclude + ] + + self.punct_marks = [p for p in punct_unicode + list(s)] + + punct = pynini.union(*self.punct_marks) + punct = pynini.closure(punct, 1) + + emphasis = ( + pynini.accep("<") + + pynini.union( + (pynini.closure(NEMO_NOT_SPACE - pynini.union("<", ">"), 1) + pynini.closure(pynini.accep("/"), 0, 1)), + (pynini.accep("/") + pynini.closure(NEMO_NOT_SPACE - pynini.union("<", ">"), 1)), + ) + + pynini.accep(">") + ) + punct = plurals._priority_union(emphasis, punct, NEMO_SIGMA) + + self.graph = punct + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/nemo_text_processing/text_normalization/kn/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/kn/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..5a0a957b9 --- /dev/null +++ b/nemo_text_processing/text_normalization/kn/taggers/tokenize_and_classify.py @@ -0,0 +1,170 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.kn.graph_utils import ( + NEMO_SPACE, + NEMO_WHITE_SPACE, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.kn.taggers.cardinal import CardinalFst + +# from nemo_text_processing.text_normalization.hi.taggers.date import DateFst +# from nemo_text_processing.text_normalization.hi.taggers.decimal import DecimalFst +# from nemo_text_processing.text_normalization.hi.taggers.electronic import ElectronicFst +# from nemo_text_processing.text_normalization.hi.taggers.fraction import FractionFst +# from nemo_text_processing.text_normalization.hi.taggers.measure import MeasureFst +# from nemo_text_processing.text_normalization.hi.taggers.money import MoneyFst +# from nemo_text_processing.text_normalization.hi.taggers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.kn.taggers.punctuation import PunctuationFst + +# from nemo_text_processing.text_normalization.hi.taggers.telephone import TelephoneFst +# from nemo_text_processing.text_normalization.hi.taggers.time import TimeFst +# from nemo_text_processing.text_normalization.hi.taggers.whitelist import WhiteListFst +from nemo_text_processing.text_normalization.kn.taggers.word import WordFst + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence including punctuation. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + input_case: str, + deterministic: bool = True, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + whitelist_file = os.path.basename(whitelist) if whitelist else "" + far_file = os.path.join( + cache_dir, + f"hi_tn_{deterministic}_deterministic_{input_case}_{whitelist_file}_tokenize.far", + ) + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logging.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logging.info(f"Creating ClassifyFst grammars.") + + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + + # decimal = DecimalFst(cardinal=cardinal, deterministic=deterministic) + # decimal_graph = decimal.fst + + # fraction = FractionFst(cardinal=cardinal, deterministic=deterministic) + # fraction_graph = fraction.fst + + # date = DateFst(cardinal=cardinal) + # date_graph = date.fst + + # timefst = TimeFst(cardinal=cardinal) + # time_graph = timefst.fst + + # ordinal = OrdinalFst(cardinal=cardinal, deterministic=deterministic) + # ordinal_graph = ordinal.fst + + # measure = MeasureFst(cardinal=cardinal, decimal=decimal, ordinal=ordinal, input_case=input_case) + # measure_graph = measure.fst + + # money = MoneyFst(cardinal=cardinal) + # money_graph = money.fst + + # whitelist_graph = WhiteListFst( + # input_case=input_case, deterministic=deterministic, input_file=whitelist + # ).fst + + punctuation = PunctuationFst(deterministic=deterministic) + punct_graph = punctuation.fst + + # telephone = TelephoneFst() + # telephone_graph = telephone.fst + + # electronic = ElectronicFst(deterministic=deterministic) + # electronic_graph = electronic.fst + + classify = ( + # pynutil.add_weight(whitelist_graph, 1.01)| + pynutil.add_weight(cardinal_graph, 1.1) + # | pynutil.add_weight(decimal_graph, 1.1) + # | pynutil.add_weight(fraction_graph, 1.1) + # | pynutil.add_weight(date_graph, 1.1) + # | pynutil.add_weight(time_graph, 1.1) + # | pynutil.add_weight(measure_graph, 1.1) + # | pynutil.add_weight(money_graph, 1.1) + # | pynutil.add_weight(telephone_graph, 1.1) + # | pynutil.add_weight(ordinal_graph, 1.1) + # | pynutil.add_weight(electronic_graph, 1.1) + ) + + word_graph = WordFst(punctuation=punctuation, deterministic=deterministic).fst + + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=2.1) + pynutil.insert(" }") + punct = pynini.closure( + pynini.union( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space), + (pynutil.insert(NEMO_SPACE) + punct), + ), + 1, + ) + + classify = pynini.union(classify, pynutil.add_weight(word_graph, 100)) + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(NEMO_SPACE)) + + token + + pynini.closure(pynutil.insert(NEMO_SPACE) + punct) + ) + + graph = token_plus_punct + pynini.closure( + pynini.union( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space), + (pynutil.insert(NEMO_SPACE) + punct + pynutil.insert(NEMO_SPACE)), + ) + + token_plus_punct + ) + + graph = delete_space + graph + delete_space + graph = pynini.union(graph, punct) + + self.fst = graph.optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) + logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/kn/taggers/word.py b/nemo_text_processing/text_normalization/kn/taggers/word.py new file mode 100644 index 000000000..c5f37efae --- /dev/null +++ b/nemo_text_processing/text_normalization/kn/taggers/word.py @@ -0,0 +1,61 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.kn.graph_utils import ( + MIN_NEG_WEIGHT, + NEMO_NOT_SPACE, + GraphFst, + convert_space, +) +from nemo_text_processing.text_normalization.kn.taggers.punctuation import PunctuationFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying Kannada words. + e.g. ಚಿನ್ನ -> tokens { name: "ಚಿನ್ನ" } + + Args: + punctuation: PunctuationFst + deterministic: if True will provide a single transduction option, + for False multiple transductions are generated (used for audio-based normalization) + """ + + def __init__(self, punctuation: PunctuationFst, deterministic: bool = True): + super().__init__(name="word", kind="classify", deterministic=deterministic) + + # Define Kannada characters and symbols using pynini.union + KANNADA_CHAR = pynini.union( + *[chr(i) for i in range(0x0C85, 0x0CB9 + 1)], # Kannada vowels and consonants + *[chr(i) for i in range(0x0CBE, 0x0CCD + 1)], # More Kannada characters + *[chr(i) for i in range(0x0CE6, 0x0CEF + 1)], # Kannada diacritics + ).optimize() + + # Include punctuation in the graph + punct = punctuation.graph + default_graph = pynini.closure(pynini.difference(NEMO_NOT_SPACE, punct.project("input")), 1) + symbols_to_exclude = (pynini.union("$", "€", "₩", "£", "¥", "#", "%") | punct).optimize() + + # Use KANNADA_CHAR in the graph + graph = pynini.closure(pynini.difference(KANNADA_CHAR, symbols_to_exclude), 1) + graph = pynutil.add_weight(graph, MIN_NEG_WEIGHT) | default_graph + + # Ensure no spaces around punctuation + graph = pynini.closure(graph + pynini.closure(punct + graph, 0, 1)) + + self.graph = convert_space(graph) + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/nemo_text_processing/text_normalization/kn/utils.py b/nemo_text_processing/text_normalization/kn/utils.py new file mode 100644 index 000000000..fa0471fb6 --- /dev/null +++ b/nemo_text_processing/text_normalization/kn/utils.py @@ -0,0 +1,74 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import os +import pynini + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + label_tsv = open(abs_path, encoding="utf-8") + labels = list(csv.reader(label_tsv, delimiter="\t")) + label_tsv.close() + return labels + + +def augment_labels_with_punct_at_end(labels): + """ + augments labels: if key ends on a punctuation that value does not have, add a new label + where the value maintains the punctuation + + Args: + labels : input labels + Returns: + additional labels + """ + res = [] + for label in labels: + if len(label) > 1: + if label[0][-1] == "." and label[1][-1] != ".": + res.append([label[0], label[1] + "."] + label[2:]) + return res + + +def apply_fst(text, fst): + """Given a string input, returns the output string + produced by traversing the path with lowest weight. + If no valid path accepts input string, returns an + error. + """ + try: + print(pynini.shortestpath(text @ fst).string()) + except pynini.FstOpError: + print(f"Error: No valid output with given input: '{text}'") diff --git a/nemo_text_processing/text_normalization/kn/verbalizers/__init__.py b/nemo_text_processing/text_normalization/kn/verbalizers/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/kn/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/kn/verbalizers/cardinal.py b/nemo_text_processing/text_normalization/kn/verbalizers/cardinal.py new file mode 100644 index 000000000..847106570 --- /dev/null +++ b/nemo_text_processing/text_normalization/kn/verbalizers/cardinal.py @@ -0,0 +1,44 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.kn.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class CardinalFst(GraphFst): + """ + Verbalizes cardinals, e.g. cardinal { integer: "5" } -> ಐದು + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + + self.optional_sign = pynini.cross("negative: \"true\"", "minus ") + if not deterministic: + self.optional_sign |= pynini.cross("negative: \"true\"", "negative ") + self.optional_sign |= pynini.cross("negative: \"true\"", "dash ") + + self.optional_sign = pynini.closure(self.optional_sign + delete_space, 0, 1) + + integer = pynini.closure(NEMO_NOT_QUOTE) + + self.integer = delete_space + pynutil.delete("\"") + integer + pynutil.delete("\"") + integer = pynutil.delete("integer:") + self.integer + + self.numbers = self.optional_sign + integer + delete_tokens = self.delete_tokens(self.numbers) + + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/kn/verbalizers/post_processing.py b/nemo_text_processing/text_normalization/kn/verbalizers/post_processing.py new file mode 100644 index 000000000..561e3f869 --- /dev/null +++ b/nemo_text_processing/text_normalization/kn/verbalizers/post_processing.py @@ -0,0 +1,96 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.kn.graph_utils import ( + MIN_NEG_WEIGHT, + NEMO_CHAR, + NEMO_SIGMA, + generator_main, +) +from nemo_text_processing.text_normalization.kn.taggers.punctuation import PunctuationFst +from nemo_text_processing.utils.logging import logger + + +class PostProcessingFst: + """ + Finite state transducer that post-processing an entire sentence after verbalization is complete, e.g. + removes extra spaces around punctuation marks " ( one hundred and twenty three ) " -> "(one hundred and twenty three)" + + Args: + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, cache_dir: str = None, overwrite_cache: bool = False): + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, "kn_tn_post_processing.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["post_process_graph"] + logger.info(f'Post processing graph was restored from {far_file}.') + else: + self.fst = self.get_punct_postprocess_graph() + + if far_file: + generator_main(far_file, {"post_process_graph": self.fst}) + + def get_punct_postprocess_graph(self): + """ + Returns graph to post process punctuation marks. + + By default, spaces are removed before punctuation marks like comma, period, etc. + """ + punct_marks_all = PunctuationFst().punct_marks + + # Punctuation marks that should NOT have space before them + # (most punctuation except quotes, dashes, and opening brackets) + quotes = ["'", "\"", "«"] + dashes = ["-", "—"] + brackets = ["<", "{", "(", r"\["] + allow_space_before_punct = quotes + dashes + brackets + + no_space_before_punct = [m for m in punct_marks_all if m not in allow_space_before_punct] + # Add Hindi-specific punctuation + no_space_before_punct.extend(["।", ",", ".", ";", ":", "!", "?"]) + # Remove duplicates + no_space_before_punct = list(set(no_space_before_punct)) + no_space_before_punct = pynini.union(*no_space_before_punct) + + delete_space = pynutil.delete(" ") + + # Delete space before no_space_before_punct marks + non_punct = pynini.difference(NEMO_CHAR, no_space_before_punct).optimize() + graph = ( + pynini.closure(non_punct) + + pynini.closure( + no_space_before_punct | pynutil.add_weight(delete_space + no_space_before_punct, MIN_NEG_WEIGHT) + ) + + pynini.closure(non_punct) + ) + graph = pynini.closure(graph).optimize() + + # Remove space after opening brackets + no_space_after_punct = pynini.union(*brackets) + no_space_after_punct = pynini.cdrewrite(delete_space, no_space_after_punct, NEMO_SIGMA, NEMO_SIGMA).optimize() + graph = pynini.compose(graph, no_space_after_punct).optimize() + + return graph diff --git a/nemo_text_processing/text_normalization/kn/verbalizers/verbalize.py b/nemo_text_processing/text_normalization/kn/verbalizers/verbalize.py new file mode 100644 index 000000000..76b1e2265 --- /dev/null +++ b/nemo_text_processing/text_normalization/kn/verbalizers/verbalize.py @@ -0,0 +1,61 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.kn.graph_utils import GraphFst +from nemo_text_processing.text_normalization.kn.verbalizers.cardinal import CardinalFst + +# from nemo_text_processing.text_normalization.hi.verbalizers.date import DateFst +# from nemo_text_processing.text_normalization.hi.verbalizers.decimal import DecimalFst +# from nemo_text_processing.text_normalization.hi.verbalizers.electronic import ElectronicFst +# from nemo_text_processing.text_normalization.hi.verbalizers.fraction import FractionFst +# from nemo_text_processing.text_normalization.hi.verbalizers.measure import MeasureFst +# from nemo_text_processing.text_normalization.hi.verbalizers.money import MoneyFst +# from nemo_text_processing.text_normalization.hi.verbalizers.ordinal import OrdinalFst +# from nemo_text_processing.text_normalization.hi.verbalizers.telephone import TelephoneFst +# from nemo_text_processing.text_normalization.hi.verbalizers.time import TimeFst +# from nemo_text_processing.text_normalization.kn.verbalizers.whitelist import WhiteListFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + + graph = ( + cardinal_graph + # | decimal_graph + # | fraction_graph + # | date_graph + # | time_graph + # | measure_graph + # | money_graph + # | ordinal_graph + # | whitelist_graph + # | telephone_graph + # | electronic_graph + ) + + self.fst = graph diff --git a/nemo_text_processing/text_normalization/kn/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/kn/verbalizers/verbalize_final.py new file mode 100644 index 000000000..003b506bf --- /dev/null +++ b/nemo_text_processing/text_normalization/kn/verbalizers/verbalize_final.py @@ -0,0 +1,76 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.kn.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.kn.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.text_normalization.kn.verbalizers.word import WordFst + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence, e.g. + tokens { name: "its" } tokens { time { hours: "twelve" minutes: "thirty" } } tokens { name: "now" } tokens { name: "." } -> its twelve thirty now . + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"en_tn_{deterministic}_deterministic_verbalizer.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + logging.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + else: + verbalize = VerbalizeFst(deterministic=deterministic).fst + word = WordFst(deterministic=deterministic).fst + types = verbalize | word + + if deterministic: + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + else: + graph = delete_space + types + delete_space + + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + + self.fst = graph.optimize() + if far_file: + generator_main(far_file, {"verbalize": self.fst}) + logging.info(f"VerbalizeFinalFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/kn/verbalizers/word.py b/nemo_text_processing/text_normalization/kn/verbalizers/word.py new file mode 100644 index 000000000..4831843cf --- /dev/null +++ b/nemo_text_processing/text_normalization/kn/verbalizers/word.py @@ -0,0 +1,43 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.kn.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + + +class WordFst(GraphFst): + """ + Finite state transducer for verbalizing Kannada words. + e.g. tokens { name: "ಚಿನ್ನ" } -> ಚಿನ್ನ + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="verbalize", deterministic=deterministic) + chars = pynini.closure(NEMO_CHAR - " ", 1) + punct = pynini.union("!", "?", ".", ",", "-", ":", ";", "।") # Add other punctuation marks as needed + char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"") + + # Ensure no spaces around punctuation + graph = char + pynini.closure(delete_space + punct, 0, 1) + + # Explicitly remove spaces before punctuation + remove_space_before_punct = pynini.cdrewrite(pynini.cross(" ", ""), "", punct, NEMO_SIGMA) + graph = graph @ remove_space_before_punct + + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/normalize.py b/nemo_text_processing/text_normalization/normalize.py index d8ebf2f4d..7a3d90b4e 100644 --- a/nemo_text_processing/text_normalization/normalize.py +++ b/nemo_text_processing/text_normalization/normalize.py @@ -166,6 +166,7 @@ def __init__( if post_process: self.post_processor = PostProcessingFst(cache_dir=cache_dir, overwrite_cache=overwrite_cache) + elif lang == 'it': from nemo_text_processing.text_normalization.it.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.text_normalization.it.verbalizers.verbalize_final import VerbalizeFinalFst @@ -191,6 +192,13 @@ def __init__( elif lang == 'ko': from nemo_text_processing.text_normalization.ko.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.text_normalization.ko.verbalizers.verbalize_final import VerbalizeFinalFst + elif lang == 'kn': + from nemo_text_processing.text_normalization.kn.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.text_normalization.kn.verbalizers.post_processing import PostProcessingFst + from nemo_text_processing.text_normalization.kn.verbalizers.verbalize_final import VerbalizeFinalFst + + if post_process: + self.post_processor = PostProcessingFst(cache_dir=cache_dir, overwrite_cache=overwrite_cache) else: raise NotImplementedError(f"Language {lang} has not been supported yet.") @@ -391,7 +399,11 @@ def normalize( return text output = SPACE_DUP.sub(' ', output[1:]) - if self.lang in ["en", "hi", "vi"] and hasattr(self, 'post_processor') and self.post_processor is not None: + if ( + self.lang in ["en", "hi", "kn", "vi"] + and hasattr(self, 'post_processor') + and self.post_processor is not None + ): output = self.post_process(output) if punct_post_process: @@ -737,7 +749,7 @@ def parse_args(): parser.add_argument( "--language", help="language", - choices=["en", "de", "es", "fr", "hu", "sv", "zh", "ar", "it", "hy", "ja", "hi", "ko", "vi", "pt"], + choices=["en", "de", "es", "fr", "hu", "sv", "zh", "ar", "it", "hy", "ja", "hi", "kn", "ko", "vi", "pt"], default="en", type=str, ) diff --git a/nemo_text_processing/text_normalization/run_evaluate.py b/nemo_text_processing/text_normalization/run_evaluate.py index 3a1964bbd..28bc8fb48 100644 --- a/nemo_text_processing/text_normalization/run_evaluate.py +++ b/nemo_text_processing/text_normalization/run_evaluate.py @@ -35,7 +35,7 @@ def parse_args(): parser.add_argument( "--lang", help="language", - choices=['ar', 'de', 'en', 'es', 'fr', 'hu', 'it', 'ru', 'sv', 'zh', 'hy', 'hi', 'ko', 'vi', 'pt'], + choices=['ar', 'de', 'en', 'es', 'fr', 'hu', 'it', 'ru', 'sv', 'zh', 'hy', 'hi', 'kn', 'ko', 'vi', 'pt'], default="en", type=str, ) @@ -67,6 +67,10 @@ def parse_args(): to_lower = args.input_case == "lower_cased" training_data = load_files([file_path], to_lower=to_lower) + # print("Loading training data: " + file_path) + # to_lower = True if args.input_case == "lower_cased" else False + # training_data = load_files([file_path], to_lower=to_lower) + if args.filter: training_data = filter_loaded_data(training_data) diff --git a/tests/nemo_text_processing/kn/__init__.py b/tests/nemo_text_processing/kn/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/tests/nemo_text_processing/kn/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/nemo_text_processing/kn/data_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/kn/data_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..6ef68e31a --- /dev/null +++ b/tests/nemo_text_processing/kn/data_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,81 @@ +4 ಚೌಕೆ~ನಾಲ್ಕು ಚೌಕೆ +೬ ಆಟಗಾರ ಔಟ್~ಆರು ಆಟಗಾರ ಔಟ್ +ವನ್ಪ್ಲಸ್ 8 ಪ್ರೊ~ವನ್ಪ್ಲಸ್ ಎಂಟು ಪ್ರೊ +೫ ಚಾರ್ಜರ್~ಐದು ಚಾರ್ಜರ್ +4 ಓವರ್‌ನಲ್ಲಿ 17 ಓಟ~ನಾಲ್ಕು ಓವರ್‌ನಲ್ಲಿ ಹದಿನೇಳು ಓಟ +೫ ಚಾಕೊಲೇಟ್‌ಗಳು ೯ ಟಾಫಿಗಳು~ಐದು ಚಾಕೊಲೇಟ್‌ಗಳು ಒಂಬತ್ತು ಟಾಫಿಗಳು +10099~ಹತ್ತು ಸಾವಿರದ ತೊಂಬತ್ತೊಂಬತ್ತು +೧೦೦೦೦೧~ಒಂದು ಲಕ್ಷದ ಒಂದು +100001~ಒಂದು ಲಕ್ಷದ ಒಂದು +4 ಷಟ್ಕಗಳು 14 ಚೌಕೆಗಳು~ನಾಲ್ಕು ಷಟ್ಕಗಳು ಹದಿನಾಲ್ಕು ಚೌಕೆಗಳು +೬ ಓಟ ಗಳಿಸಿದರು~ಆರು ಓಟ ಗಳಿಸಿದರು +3 ಗೋಲು ಹೊಡೆದರು~ಮೂರು ಗೋಲು ಹೊಡೆದರು +೬ ಓಟ ಗಳಿಸಿದರು~ಆರು ಓಟ ಗಳಿಸಿದರು +3 ಗೋಲು ಹೊಡೆದರು~ಮೂರು ಗೋಲು ಹೊಡೆದರು +೫ ಚೌಕೆ~ಐದು ಚೌಕೆ +2 ಷಟ್ಕಗಳು 12 ಓಟ~ಎರಡು ಷಟ್ಕಗಳು ಹನ್ನೆರಡು ಓಟ +೯ ಅಂಕಗಳು~ಒಂಬತ್ತು ಅಂಕಗಳು +4 ಮೃತರು 18 ಗಾಯಾಳುಗಳು~ನಾಲ್ಕು ಮೃತರು ಹದಿನೆಂಟು ಗಾಯಾಳುಗಳು +೫ ಗೋಲು ಹೊಡೆ~ಐದು ಗೋಲು ಹೊಡೆ +3 ಬ್ಯಾಟ್ಸ್‌ಮನ್‌ಗಳು 12 ಆಟಗಾರರು~ಮೂರು ಬ್ಯಾಟ್ಸ್‌ಮನ್‌ಗಳು ಹನ್ನೆರಡು ಆಟಗಾರರು +೫ ಸೋಲು ೧ ಡ್ರಾ ೧೭ ಜಯ~ಐದು ಸೋಲು ಒಂದು ಡ್ರಾ ಹದಿನೇಳು ಜಯ +51022345567~ಐದು ಸಾವಿರದ ನೂರ ಎರಡು ಕೋಟಿಯ ಇಪ್ಪತ್ತಮೂರು ಲಕ್ಷದ ನಲವತ್ತೈದು ಸಾವಿರದ ಐನೂರ ಅರವತ್ತೇಳು +ಪಾಠ ೧೦ ಶ್ಲೋಕ ೨~ಪಾಠ ಹತ್ತು ಶ್ಲೋಕ ಎರಡು +೧೧೦~ನೂರ ಹತ್ತು +109~ನೂರ ಒಂಬತ್ತು +111~ನೂರ ಹನ್ನೊಂದು +೧೧೨~ನೂರ ಹನ್ನೆರಡು +113~ನೂರ ಹದಿಮೂರು +೧೧೪~ನೂರ ಹದಿನಾಲ್ಕು +115~ನೂರ ಹದಿನೈದು +೧೧೬~ನೂರ ಹದಿನಾರು +817~ಎಂಟುನೂರ ಹದಿನೇಳು +೮೧೮~ಎಂಟುನೂರ ಹದಿನೆಂಟು +741~ಏಳುನೂರ ನಲವತ್ತೊಂದು +೭೪೨~ಏಳುನೂರ ನಲವತ್ತೆರಡು +743~ಏಳುನೂರ ನಲವತ್ತಮೂರು +೭೪೪~ಏಳುನೂರ ನಲವತ್ತನಾಲ್ಕು +745~ಏಳುನೂರ ನಲವತ್ತೈದು +೪೪೬~ನಾನೂರ ನಲವತ್ತಾರು +461~ನಾನೂರ ಅರವತ್ತೊಂದು +೧೭೫~ನೂರ ಎಪ್ಪತ್ತೈದು +181~ನೂರ ಎಂಬತ್ತೊಂದು +೧೯೦~ನೂರ ತೊಂಬತ್ತು +191~ನೂರ ತೊಂಬತ್ತೊಂದು +೧೯೯~ನೂರ ತೊಂಬತ್ತೊಂಬತ್ತು +1001~ಒಂದು ಸಾವಿರದ ಒಂದು +೧೦೯೯~ಒಂದು ಸಾವಿರದ ತೊಂಬತ್ತೊಂಬತ್ತು +5551 ಬಾಳೆಹಣ್ಣುಗಳು~ಐದು ಸಾವಿರದ ಐನೂರ ಐವತ್ತೊಂದು ಬಾಳೆಹಣ್ಣುಗಳು +೫೫೫೫೨ ಸೇಬುಗಳು~ಐವತ್ತೈದು ಸಾವಿರದ ಐನೂರ ಐವತ್ತೆರಡು ಸೇಬುಗಳು +53~ಐವತ್ತಮೂರು +೫೪~ಐವತ್ತನಾಲ್ಕು +55~ಐವತ್ತೈದು +97~ತೊಂಬತ್ತೇಳು +೯೮~ತೊಂಬತ್ತೆಂಟು +99~ತೊಂಬತ್ತೊಂಬತ್ತು +೧೩೨೩~ಒಂದು ಸಾವಿರದ ಮುನ್ನೂರ ಇಪ್ಪತ್ತಮೂರು +1345~ಒಂದು ಸಾವಿರದ ಮುನ್ನೂರ ನಲವತ್ತೈದು +೧೩೪೫೬~ಹದಿಮೂರು ಸಾವಿರದ ನಾನೂರ ಐವತ್ತಾರು +12346~ಹನ್ನೆರಡು ಸಾವಿರದ ಮುನ್ನೂರ ನಲವತ್ತಾರು +೧೨೩೪೫೬~ಒಂದು ಲಕ್ಷದ ಇಪ್ಪತ್ತಮೂರು ಸಾವಿರದ ನಾನೂರ ಐವತ್ತಾರು +872987~ಎಂಟು ಲಕ್ಷದ ಎಪ್ಪತ್ತೆರಡು ಸಾವಿರದ ಒಂಬೈನೂರ ಎಂಬತ್ತೇಳು +೯೮೭೬೦೯~ಒಂಬತ್ತು ಲಕ್ಷದ ಎಂಬತ್ತೇಳು ಸಾವಿರದ ಆರುನೂರ ಒಂಬತ್ತು +9876789~ತೊಂಬತ್ತೆಂಟು ಲಕ್ಷದ ಎಪ್ಪತ್ತಾರು ಸಾವಿರದ ಏಳುನೂರ ಎಂಬತ್ತೊಂಬತ್ತು +೨೩೪೫೫೬೭~ಇಪ್ಪತ್ತಮೂರು ಲಕ್ಷದ ನಲವತ್ತೈದು ಸಾವಿರದ ಐನೂರ ಅರವತ್ತೇಳು +12345567~ಒಂದು ಕೋಟಿಯ ಇಪ್ಪತ್ತಮೂರು ಲಕ್ಷದ ನಲವತ್ತೈದು ಸಾವಿರದ ಐನೂರ ಅರವತ್ತೇಳು +೧೨೧೨೧೨೧೨~ಒಂದು ಕೋಟಿಯ ಇಪ್ಪತ್ತೊಂದು ಲಕ್ಷದ ಇಪ್ಪತ್ತೊಂದು ಸಾವಿರದ ಇನ್ನೂರ ಹನ್ನೆರಡು +೧೦೨೨೩೪೫೫೬೭~ನೂರ ಎರಡು ಕೋಟಿಯ ಇಪ್ಪತ್ತಮೂರು ಲಕ್ಷದ ನಲವತ್ತೈದು ಸಾವಿರದ ಐನೂರ ಅರವತ್ತೇಳು +2 ಅಂಕಗಳು 12 ಗೋಲ ~ಎರಡು ಅಂಕಗಳು ಹನ್ನೆರಡು ಗೋಲ +೦೫~ಸೊನ್ನೆ ಐದು +01~ಸೊನ್ನೆ ಒಂದು +೦೭೩~ಸೊನ್ನೆ ಏಳು ಮೂರು +0001~ಸೊನ್ನೆ ಸೊನ್ನೆ ಸೊನ್ನೆ ಒಂದು +೦೦೦~ಸೊನ್ನೆ ಸೊನ್ನೆ ಸೊನ್ನೆ +500~ಐನೂರು +೧೦೦೦೦೦೩~ಹತ್ತು ಲಕ್ಷದ ಮೂರು +ಪುಟ ೧೯೯ ಪಾಠ ೬~ಪುಟ ನೂರ ತೊಂಬತ್ತೊಂಬತ್ತು ಪಾಠ ಆರು +೧೯ ರಲ್ಲಿ ೩ ಗ್ಲಾಸ್~ಹತ್ತೊಂಬತ್ತು ರಲ್ಲಿ ಮೂರು ಗ್ಲಾಸ್ +ರುಕು ೪ ಆಯತ್ ೬ ರ ಸಾಲು ೩೩~ರುಕು ನಾಲ್ಕು ಆಯತ್ ಆರು ರ ಸಾಲು ಮೂವತ್ತಮೂರು +೧೦೦೦ ರ ೨ ನೋಟು~ಒಂದು ಸಾವಿರ ರ ಎರಡು ನೋಟು +ಅಧ್ಯಾಯ ೧೨ ಶ್ಲೋಕ ೮~ಅಧ್ಯಾಯ ಹನ್ನೆರಡು ಶ್ಲೋಕ ಎಂಟು +೮ ಓವರ್‌ನಲ್ಲಿ ೪೫ ಓಟ~ಎಂಟು ಓವರ್‌ನಲ್ಲಿ ನಲವತ್ತೈದು ಓಟ \ No newline at end of file diff --git a/tests/nemo_text_processing/kn/test_cardinal.py b/tests/nemo_text_processing/kn/test_cardinal.py new file mode 100644 index 000000000..e758fdeda --- /dev/null +++ b/tests/nemo_text_processing/kn/test_cardinal.py @@ -0,0 +1,45 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestCardinal: + normalizer = Normalizer( + input_case='cased', lang='kn', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + # inverse_normalizer = InverseNormalizer(lang='kn', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('kn/data_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() + + +''' + @parameterized.expand(parse_test_case_file('kn/data_inverse_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected +'''