Hi TN: implement natural cardinal verbalization for up to 3-digit numbers in address

Jenkinsfile

@@ -28,7 +28,7 @@ pipeline {
     MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1'
     JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1'
     KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/04-23-26-0'
-    HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-25-26-0'
+    HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-08-26-0'
     DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
   }
   stages {

nemo_text_processing/text_normalization/hi/data/address/cities.tsv

@@ -34,3 +34,4 @@
 कारगिल
 कवरत्ती
 पुडुचेरी
+बैंगलोर

nemo_text_processing/text_normalization/hi/data/address/context.tsv

@@ -45,4 +45,7 @@
 सामने
 पीछे
 वीया
-आर डी
+हिल
+पिनकोड
+पार्कहर्स्ट
+स्टे

nemo_text_processing/text_normalization/hi/taggers/measure.py

@@ -30,6 +30,7 @@
     HYPHEN,
     INPUT_LOWER_CASED,
     LOWERCASE_X,
+    NEMO_ALL_DIGIT,
     NEMO_CHAR,
     NEMO_DIGIT,
     NEMO_HI_DIGIT,
@@ -50,7 +51,6 @@
 from nemo_text_processing.text_normalization.hi.utils import get_abs_path
 
 digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
-# Load both Hindi (Devanagari) and English (Arabic) number mappings
 teens_ties_hi = pynini.string_file(get_abs_path("data/numbers/teens_and_ties.tsv"))
 teens_ties_en = pynini.string_file(get_abs_path("data/numbers/teens_and_ties_en.tsv"))
 teens_ties = pynini.union(teens_ties_hi, teens_ties_en)
@@ -67,49 +67,40 @@ class MeasureFst(GraphFst):
     Args:
         cardinal: CardinalFst
         decimal: DecimalFst
-        deterministic: if True will provide a single transduction option,
-            for False multiple transduction are generated (used for audio-based normalization)
+        ordinal: OrdinalFst
+        input_case: str
     """
 
     def get_structured_address_graph(self, ordinal: GraphFst, input_case: str):
         """
         Minimal address tagger for state/city + pincode patterns only.
-        Highly optimized for performance.
 
         Examples:
             "मुंबई ८८४४०४" -> "मुंबई आठ आठ चार चार शून्य चार"
             "गोवा १२३४५६" -> "गोवा एक दो तीन चार पाँच छह"
         """
-        # State/city keywords
         states = pynini.string_file(get_abs_path("data/address/states.tsv"))
         cities = pynini.string_file(get_abs_path("data/address/cities.tsv"))
         state_city_names = pynini.union(states, cities).optimize()
 
-        # Digit mappings
         num_token = (
             digit
             | pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
             | pynini.string_file(get_abs_path("data/telephone/number.tsv"))
         ).optimize()
 
-        # Pincode (6 digits)
         pincode = (num_token + pynini.closure(insert_space + num_token, 5, 5)).optimize()
-
-        # Street number (1-4 digits)
         street_num = (num_token + pynini.closure(insert_space + num_token, 0, 3)).optimize()
 
-        # Text: words with trailing separator (comma? + space)
         any_digit = pynini.union(NEMO_HI_DIGIT, NEMO_DIGIT).optimize()
         punctuation = pynini.union(COMMA, PERIOD, HI_PERIOD).optimize()
         word_char = pynini.difference(NEMO_NOT_SPACE, pynini.union(any_digit, punctuation)).optimize()
         word = pynini.closure(word_char, 1)
 
-        # Separator: optional comma followed by mandatory space
         sep = pynini.closure(pynini.accep(COMMA), 0, 1) + pynini.accep(NEMO_SPACE)
         word_with_sep = word + sep
         text = pynini.closure(word_with_sep, 0, 5).optimize()
 
-        # Pattern: [street_num + sep]? text state/city [space pincode]
         pattern = (
             pynini.closure(street_num + sep, 0, 1)
             + text
@@ -124,78 +115,135 @@ def get_structured_address_graph(self, ordinal: GraphFst, input_case: str):
         )
         return pynutil.add_weight(graph, 1.0).optimize()
 
-    def get_address_graph(self, ordinal: GraphFst, input_case: str):
+    def get_address_graph(self, cardinal: GraphFst, ordinal: GraphFst, input_case: str):
         """
-        Address tagger that converts digits/hyphens/slashes character-by-character
-        when address context keywords are present.
-        English words and ordinals are converted to Hindi transliterations.
+        Address tagger with natural Hindi verbalization for street numbers.
+        1-3 digit numbers are verbalized as cardinals, 4+ digit numbers and
+        PIN codes as digit-by-digit.
 
         Examples:
-            "७०० ओक स्ट्रीट" -> "सात शून्य शून्य ओक स्ट्रीट"
-            "६६-४ पार्क रोड" -> "छह छह हाइफ़न चार पार्क रोड"
+            "७०० ओक स्ट्रीट"  -> "सात सौ ओक स्ट्रीट"
+            "६६-४ पार्क रोड"  -> "छियासठ हाइफ़न चार पार्क रोड"
+            "14/3 मथुरा रोड"  -> "चौदह बटा तीन मथुरा रोड"
         """
-        ordinal_graph = ordinal.graph
-        # Alphanumeric to word mappings (digits, special characters, telephone digits)
-        char_to_word = (
-            digit
-            | pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
-            | pynini.string_file(get_abs_path("data/address/special_characters.tsv"))
-            | pynini.string_file(get_abs_path("data/telephone/number.tsv"))
+        single_digit_word = (cardinal.digit | cardinal.zero).optimize()
+        single_digit_input = NEMO_ALL_DIGIT
+
+        digit_run = (single_digit_word + pynini.closure(insert_space + single_digit_word)).optimize()
+
+        num_1digit = pynini.compose(single_digit_input**1, (cardinal.digit | cardinal.zero)).optimize()
+
+        num_2digit = pynini.compose(single_digit_input**2, cardinal.teens_and_ties).optimize()
+
+        num_3digit = pynini.compose(single_digit_input**3, cardinal.graph_hundreds).optimize()
+
+        num_as_cardinal = (
+            pynutil.add_weight(num_3digit, -2.0)
+            | pynutil.add_weight(num_2digit, -1.0)
+            | pynutil.add_weight(num_1digit, 0.0)
         ).optimize()
-        letter_to_word = pynini.string_file(get_abs_path("data/address/letters.tsv"))
-        letter_to_word = capitalized_input_graph(letter_to_word)
-        address_keywords_hi = pynini.string_file(get_abs_path("data/address/context.tsv"))
 
-        # English address keywords with Hindi translation (case-insensitive)
+        digit_run_4_plus = pynini.compose(
+            single_digit_input**4 + pynini.closure(single_digit_input), digit_run
+        ).optimize()
+
+        any_street_num = (num_as_cardinal | pynutil.add_weight(digit_run_4_plus, -5.0)).optimize()
+
+        hyphen_word = pynini.cross(HYPHEN, "हाइफ़न")
+        hyphenated_num = (any_street_num + insert_space + hyphen_word + insert_space + any_street_num).optimize()
+
+        slash_word = pynini.cross(SLASH, "बटा")
+        slashed_num = (any_street_num + insert_space + slash_word + insert_space + any_street_num).optimize()
+
+        pincode = pynini.compose(
+            single_digit_input**6, single_digit_word + pynini.closure(insert_space + single_digit_word, 5, 5)
+        ).optimize()
+
+        street_num_1_to_3 = num_as_cardinal
+
+        street_num_4_digit = pynini.compose(single_digit_input**4, digit_run).optimize()
+
+        street_num_5_digit = pynini.compose(single_digit_input**5, digit_run).optimize()
+
+        ordinal_processor = pynutil.add_weight(insert_space + ordinal.graph, -5.0)
+
         en_to_hi_map = pynini.string_file(get_abs_path("data/address/en_to_hi_mapping.tsv"))
         if input_case != INPUT_LOWER_CASED:
             en_to_hi_map = capitalized_input_graph(en_to_hi_map)
-        address_keywords_en = pynini.project(en_to_hi_map, "input")
-        address_keywords = pynini.union(address_keywords_hi, address_keywords_en)
+        english_word_processor = pynutil.add_weight(insert_space + en_to_hi_map, -3.0)
 
-        # Alphanumeric processing: treat digits, letters, and -/ as convertible tokens
-        single_digit = pynini.union(NEMO_DIGIT, NEMO_HI_DIGIT).optimize()
-        special_chars = pynini.union(HYPHEN, SLASH).optimize()
+        letter_to_word = pynini.string_file(get_abs_path("data/address/letters.tsv"))
+        letter_to_word = capitalized_input_graph(letter_to_word)
         single_letter = pynini.project(letter_to_word, "input").optimize()
-        convertible_char = pynini.union(single_digit, special_chars, single_letter)
+        letter_processor = pynutil.add_weight(insert_space + pynini.compose(single_letter, letter_to_word), 0.5)
+
+        special_chars = pynini.union(HYPHEN, SLASH).optimize()
+        convertible_char = pynini.union(single_digit_input, special_chars, single_letter)
         non_space_char = pynini.difference(
             NEMO_CHAR, pynini.union(NEMO_WHITE_SPACE, convertible_char, pynini.accep(COMMA))
         ).optimize()
 
-        # Token processors with weights: prefer ordinals and known English→Hindi words
-        # Delete space before comma to avoid Sparrowhawk "sil" issue
+        right_side_letter = (
+            pynini.compose(single_letter, letter_to_word) | pynini.closure(non_space_char, 1)
+        ).optimize()
+
+        slashed_num_letter = (any_street_num + insert_space + slash_word + insert_space + right_side_letter).optimize()
+
+        slashed_num = (slashed_num | slashed_num_letter).optimize()
+
         comma_processor = pynutil.add_weight(delete_space + pynini.accep(COMMA), 0.0)
-        ordinal_processor = pynutil.add_weight(insert_space + ordinal_graph, -5.0)
-        english_word_processor = pynutil.add_weight(insert_space + en_to_hi_map, -3.0)
-        letter_processor = pynutil.add_weight(insert_space + pynini.compose(single_letter, letter_to_word), 0.5)
-        digit_char_processor = pynutil.add_weight(insert_space + pynini.compose(convertible_char, char_to_word), 0.0)
+
+        standalone_hyphen = pynutil.add_weight(pynini.cross(HYPHEN, "हाइफ़न"), 0.2)
+
         other_word_processor = pynutil.add_weight(insert_space + pynini.closure(non_space_char, 1), 0.1)
 
         token_processor = (
             ordinal_processor
             | english_word_processor
+            | pynutil.add_weight(insert_space + hyphenated_num, -4.0)
+            | pynutil.add_weight(insert_space + slashed_num, -4.0)
+            | pynutil.add_weight(insert_space + pincode, -5.0)
+            | pynutil.add_weight(insert_space + street_num_5_digit, -3.5)
+            | pynutil.add_weight(insert_space + street_num_4_digit, -3.5)
+            | pynutil.add_weight(insert_space + street_num_1_to_3, 0.0)
             | letter_processor
-            | digit_char_processor
             | pynini.accep(NEMO_SPACE)
             | comma_processor
+            | pynutil.add_weight(insert_space + standalone_hyphen, 0.2)
             | other_word_processor
         ).optimize()
+
         full_string_processor = pynini.closure(token_processor, 1).optimize()
 
-        # Window-based context matching around address keywords for robust detection
+        address_keywords_hi = pynini.string_file(get_abs_path("data/address/context.tsv"))
+        address_keywords_en = pynini.project(en_to_hi_map, "input")
+        states_graph = pynini.string_file(get_abs_path("data/address/states.tsv"))
+        cities_graph = pynini.string_file(get_abs_path("data/address/cities.tsv"))
+        state_city_keywords = pynini.union(
+            pynini.project(states_graph, "input"),
+            pynini.project(cities_graph, "input"),
+        ).optimize()
+        address_keywords = pynini.union(address_keywords_hi, address_keywords_en, state_city_keywords)
+
         word_boundary = pynini.union(
-            NEMO_WHITE_SPACE, pynini.accep(COMMA), pynini.accep(HI_PERIOD), pynini.accep(PERIOD)
+            NEMO_WHITE_SPACE,
+            pynini.accep(COMMA),
+            pynini.accep(HI_PERIOD),
+            pynini.accep(PERIOD),
         ).optimize()
         non_boundary_char = pynini.difference(NEMO_CHAR, word_boundary)
         word = pynini.closure(non_boundary_char, 1).optimize()
         word_with_boundary = word + pynini.closure(word_boundary)
         window = pynini.closure(word_with_boundary, 0, 5).optimize()
         boundary = pynini.closure(word_boundary, 1).optimize()
+
         input_pattern = pynini.union(
             address_keywords + boundary + window,
             window + boundary + address_keywords + pynini.closure(boundary + window, 0, 1),
         ).optimize()
+
         address_graph = pynini.compose(input_pattern, full_string_processor).optimize()
+
         graph = (
             pynutil.insert('units: "address" cardinal { integer: "')
             + address_graph
@@ -221,15 +269,12 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, inp
         decimal_graph = decimal_integers + point + insert_space + decimal.graph_fractional
         unit_graph = pynini.string_file(get_abs_path("data/measure/unit.tsv"))
 
-        # Year unit variants for formal/informal handling
         year_informal = pynini.string_map([("yr", "साल")])
         year_formal = pynini.string_file(get_abs_path("data/measure/unit_year_formal.tsv"))
 
-        # All units EXCEPT year
         unit_inputs_except_yr = pynini.difference(pynini.project(unit_graph, "input"), pynini.accep("yr"))
         unit_graph_no_year = pynini.compose(unit_inputs_except_yr, unit_graph)
 
-        # Load quarterly units from separate files: map (FST) and list (FSA)
         quarterly_units_map = pynini.string_file(get_abs_path("data/measure/quarterly_units_map.tsv"))
         quarterly_units_list = pynini.string_file(get_abs_path("data/measure/quarterly_units_list.tsv"))
         quarterly_units_graph = pynini.union(quarterly_units_map, quarterly_units_list)
@@ -240,15 +285,6 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, inp
             1,
         )
 
-        # Define the quarterly measurements - support both Devanagari and Arabic digits
-        quarter = pynini.union(
-            pynini.cross(POINT_FIVE, HI_SADHE),
-            pynini.cross(ONE_POINT_FIVE, HI_DEDH),
-            pynini.cross(TWO_POINT_FIVE, HI_DHAI),
-        )
-        quarter_graph = pynutil.insert("integer_part: \"") + quarter + pynutil.insert("\"")
-
-        # Define the unit handling
         unit = (
             pynutil.insert(NEMO_SPACE)
             + pynutil.insert("units: \"")
@@ -264,7 +300,6 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, inp
             + pynutil.insert(NEMO_SPACE)
         )
 
-        # Year-specific unit wrappers
         unit_year_informal = (
             pynutil.insert(NEMO_SPACE)
             + pynutil.insert("units: \"")
@@ -280,11 +315,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, inp
             + pynutil.insert(NEMO_SPACE)
         )
 
-        # Cardinal >= 1000 -> formal year (वर्ष)
-        # Use graph_without_leading_zeros which covers all number ranges (thousands to shankhs)
         cardinal_large = cardinal.graph_without_leading_zeros
-
-        # Cardinal < 1000 -> informal year (साल)
         cardinal_small = cardinal.zero | cardinal.digit | cardinal.teens_and_ties | cardinal.graph_hundreds
 
         symbol_graph = pynini.string_map(
@@ -304,14 +335,12 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, inp
             + unit
         )
 
-        # Support both Devanagari and Arabic digits for dedh/dhai patterns
         dedh_dhai = pynini.union(
             pynini.cross(ONE_POINT_FIVE, HI_DEDH),
             pynini.cross(TWO_POINT_FIVE, HI_DHAI),
         )
         dedh_dhai_graph = pynutil.insert("integer: \"") + dedh_dhai + pynutil.insert("\"")
 
-        # Support both Devanagari and Arabic digits for savva pattern
         savva_numbers = cardinal_graph + pynini.cross(DECIMAL_25, "")
         savva_graph = (
             pynutil.insert("integer: \"")
@@ -321,7 +350,6 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, inp
             + pynutil.insert("\"")
         )
 
-        # Support both Devanagari and Arabic digits for sadhe pattern
         sadhe_numbers = cardinal_graph + pynini.cross(POINT_FIVE, "")
         sadhe_graph = (
             pynutil.insert("integer: \"")
@@ -332,7 +360,6 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, inp
         )
 
         paune = pynini.string_file(get_abs_path("data/whitelist/paune_mappings.tsv"))
-        # Support both Devanagari and Arabic digits for paune pattern
         paune_numbers = paune + pynini.cross(DECIMAL_75, "")
         paune_graph = (
             pynutil.insert("integer: \"")
@@ -393,7 +420,6 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, inp
             + unit
         )
 
-        # Large numbers (>=1000) + yr -> formal (वर्ष)
         graph_cardinal_year_formal = (
             pynutil.insert("cardinal { ")
             + optional_graph_negative
@@ -406,7 +432,6 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, inp
             + unit_year_formal
         )
 
-        # Small numbers (<1000) + yr -> informal (साल)
         graph_cardinal_year_informal = (
             pynutil.insert("cardinal { ")
             + optional_graph_negative
@@ -419,7 +444,6 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, inp
             + unit_year_informal
         )
 
-        # Regular decimals (e.g., 16.07) + yr -> formal (वर्ष)
         graph_decimal_year_formal = (
             pynutil.insert("decimal { ")
             + optional_graph_negative
@@ -429,7 +453,6 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, inp
             + unit_year_formal
         )
 
-        # Handling cardinal clubbed with symbol as single token
         graph_exceptions = (
             pynutil.insert("cardinal { ")
             + optional_graph_negative
@@ -451,15 +474,15 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, inp
             + pynutil.insert("\"")
         )
 
-        address_graph = self.get_address_graph(ordinal, input_case)
+        address_graph = self.get_address_graph(cardinal, ordinal, input_case)
         structured_address_graph = self.get_structured_address_graph(ordinal, input_case)
 
         graph = (
             pynutil.add_weight(graph_decimal, 0.1)
             | pynutil.add_weight(graph_decimal_year_formal, 0.1)
             | pynutil.add_weight(graph_cardinal, 0.1)
             | pynutil.add_weight(graph_cardinal_year_formal, 0.1)
-            | pynutil.add_weight(graph_cardinal_year_informal, -0.1)  # Higher priority for small numbers
+            | pynutil.add_weight(graph_cardinal_year_informal, -0.1)
             | pynutil.add_weight(graph_exceptions, 0.1)
             | pynutil.add_weight(graph_dedh_dhai, -0.2)
             | pynutil.add_weight(graph_savva, -0.1)