Ar TN/TN

Jenkinsfile

@@ -11,7 +11,7 @@ pipeline {
   }
   environment {
 
-    AR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/02-15-23-0'
+    AR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/04-27-23-0'
     DE_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/02-15-23-0'
     EN_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/03-27-23-1'
     ES_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/04-05-23-0'

nemo_text_processing/inverse_text_normalization/ar/taggers/date.py

@@ -0,0 +1,113 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pynini
+from nemo_text_processing.text_normalization.ar.graph_utils import (
+    NEMO_DIGIT,
+    NEMO_NOT_QUOTE,
+    NEMO_SIGMA,
+    GraphFst,
+    convert_space,
+)
+from nemo_text_processing.text_normalization.ar.utils import get_abs_path, load_labels
+from pynini.lib import pynutil
+
+
+class DateFst(GraphFst):
+    """
+    Finite state transducer for classifying spoken Arabic date into a digit form,
+    by inverting the text-normalization date tagger and verbalizer, e.g.
+        الأول من شهر نوفمبر لعام ألفين وعشرة -> tokens { name: "01/11/2010" }
+        الأول من شهر نوفمبر -> tokens { name: "1 نوفمبر" }
+        الأول من شهر ربيع الثاني لعام ... هجري -> tokens { name: "01/04/1445 هـ" }
+
+    Args:
+        itn_cardinal_tagger: ITN cardinal tagger
+        tn_date_tagger: TN date tagger
+        tn_date_verbalizer: TN date verbalizer
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(
+        self,
+        itn_cardinal_tagger: GraphFst,
+        tn_date_tagger: GraphFst,
+        tn_date_verbalizer: GraphFst,
+        deterministic: bool = True,
+    ):
+        super().__init__(name="date", kind="classify", deterministic=deterministic)
+
+        add_leading_zero_to_double_digit = (NEMO_DIGIT + NEMO_DIGIT) | (pynutil.insert("0") + NEMO_DIGIT)
+        optional_delete_space = pynini.closure(NEMO_SIGMA | pynutil.delete(" ", weight=0.0001))
+        ordinals = pynini.string_file(get_abs_path("data/ordinal/ordinals_date.tsv")).invert().optimize()
+        hijri_suffixes = pynini.string_file(get_abs_path("data/months/hijri_suffixes_inverse.tsv")).optimize()
+        months_names = [x[0] for x in load_labels(get_abs_path("data/months/months_name.tsv"))]
+        months_names = pynini.union(*months_names)
+        month_to_number = tn_date_tagger.number_to_month.invert().optimize()
+        month_to_number_hijri = tn_date_tagger.month_hijri.invert().optimize()
+        tagger = tn_date_verbalizer.graph.invert().optimize()
+
+        delete_day_marker = (
+            pynutil.delete("day: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"")
+        ) @ ordinals
+
+        month_as_number = pynutil.delete("month: \"") + itn_cardinal_tagger.graph + pynutil.delete("\"")
+        month_as_string = pynutil.delete("month: \"") + months_names + pynutil.delete("\"")
+        month_name_to_number = (
+            pynutil.delete("month: \"") + (month_to_number | month_to_number_hijri) + pynutil.delete("\"")
+        )
+
+        convert_year = (tn_date_tagger.year @ optional_delete_space).invert().optimize()
+        convert_year |= (
+            (tn_date_tagger.year @ optional_delete_space).invert().optimize() + pynini.accep(" ") + hijri_suffixes
+        )
+        delete_year_marker = (
+            pynutil.delete("year: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"")
+        ) @ convert_year
+
+        # day month as string (year)
+        verbalizer = pynutil.add_weight(
+            (
+                pynini.closure(delete_day_marker + pynini.accep(" "), 0, 1)
+                + month_as_string
+                + pynini.closure(pynini.accep(" ") + delete_year_marker, 0, 1)
+            ),
+            weight=0.0001,
+        )
+
+        # day month as number (year); trailing "/" only when a year is present
+        verbalizer |= (
+            delete_day_marker @ add_leading_zero_to_double_digit
+            + pynutil.insert("/")
+            + pynutil.delete(" ")
+            + month_as_number @ add_leading_zero_to_double_digit
+            + pynini.closure(pynutil.insert("/") + pynutil.delete(" ") + delete_year_marker, 0, 1)
+        )
+        # 02/03/2022 هـ
+        verbalizer |= (
+            delete_day_marker @ add_leading_zero_to_double_digit
+            + pynutil.insert("/")
+            + pynutil.delete(" ")
+            + month_name_to_number @ add_leading_zero_to_double_digit
+            + pynini.closure(pynutil.insert("/") + pynutil.delete(" ") + delete_year_marker, 0, 1)
+        )
+
+        # year
+        verbalizer |= delete_year_marker
+
+        final_graph = tagger @ verbalizer
+
+        graph = pynutil.insert("name: \"") + convert_space(final_graph) + pynutil.insert("\"")
+        self.fst = graph.optimize()

nemo_text_processing/inverse_text_normalization/ar/taggers/decimal.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -53,7 +53,7 @@ def __init__(self, tn_decimal):
         self.final_graph_wo_sign = (
             graph_integer + pynini.accep(NEMO_SPACE) + graph_fractional + optional_graph_quantity
         )
-        final_graph = optional_graph_negative + self.final_graph_wo_sign
+        self.final_graph_wo_negative = optional_graph_negative + self.final_graph_wo_sign
 
-        final_graph = self.add_tokens(final_graph)
+        final_graph = self.add_tokens(self.final_graph_wo_negative)
         self.fst = final_graph.optimize()

nemo_text_processing/inverse_text_normalization/ar/taggers/fraction.py

@@ -45,7 +45,7 @@ def __init__(self, tn_cardinal: GraphFst):
         # create unions for special cases
         denominator_singular = pynini.union("نصف", "ثلث", "ربع", "خمس", "سدس", "سبع", "ثمن", "تسع", "عشر")
         denominator_dual = pynini.union(
-            "نصفين", "ثلثين", "ربعين", "خمسين", "سدسين", "سبعين", "ثمنين", "تسعين", "عشرين"
+            "نصفي", "ثلثي", "ربعي", "خمسي", "سدسي", "سبعي", "ثمني", "تسعي", "عشري"
         )
         denominator_plural = pynini.union("أخماس", "أرباع", "أثلاث", "أسداس", "أسباع", "أثمان", "أتساع", "أعشار")
         numerator_three_to_ten = pynini.union("خمسة", "سبعة", "عشرة", "ثلاثة", "أربعة", "ستة", "ثمانية", "تسعة")

nemo_text_processing/inverse_text_normalization/ar/taggers/measure.py

@@ -0,0 +1,83 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pynini
+from nemo_text_processing.text_normalization.ar.graph_utils import GraphFst, convert_space, delete_extra_space
+from nemo_text_processing.text_normalization.ar.taggers.measure import unit_singular
+from pynini.lib import pynutil
+
+
+class MeasureFst(GraphFst):
+    """
+    Finite state transducer for classifying measure. Allows for plural form for unit.
+        e.g. "عشرون  في المائة" -> measure { cardinal { integer: "20" } units: "%" }
+
+    Args:
+        itn_cardinal_tagger: ITN Cardinal tagger
+        itn_decimal_tagger: ITN Decimal tagger
+        itn_fraction_tagger: ITN Fraction tagger
+    """
+
+    def __init__(
+        self,
+        itn_cardinal_tagger: GraphFst,
+        itn_decimal_tagger: GraphFst,
+        itn_fraction_tagger: GraphFst,
+        deterministic: bool = True,
+    ):
+        super().__init__(name="measure", kind="classify", deterministic=deterministic)
+
+        cardinal_graph = itn_cardinal_tagger.graph
+
+        graph_unit_singular = pynini.invert(unit_singular)
+        unit = convert_space(graph_unit_singular)
+
+        optional_graph_negative = pynini.closure(
+            pynutil.insert("negative: ") + pynini.cross("سالب", "\"true\"") + delete_extra_space, 0, 1
+        )
+
+        unit = pynutil.insert("units: \"") + (unit) + pynutil.insert("\"")
+
+        subgraph_decimal = (
+            pynutil.insert("decimal { ")
+            + optional_graph_negative
+            + itn_decimal_tagger.final_graph_wo_negative
+            + pynutil.insert(" }")
+            + delete_extra_space
+            + unit
+        )
+
+        subgraph_fraction = (
+            pynutil.insert("decimal { ")
+            + optional_graph_negative
+            + pynutil.insert("integer_part: \"")
+            + itn_fraction_tagger.graph
+            + pynutil.insert("\" }")
+            + delete_extra_space
+            + unit
+        )
+
+        subgraph_cardinal = (
+            pynutil.insert("cardinal { ")
+            + optional_graph_negative
+            + pynutil.insert("integer: \"")
+            + cardinal_graph
+            + pynutil.insert("\"")
+            + pynutil.insert(" }")
+            + delete_extra_space
+            + unit
+        )
+        final_graph = subgraph_cardinal | subgraph_decimal | subgraph_fraction
+        final_graph = self.add_tokens(final_graph)
+        self.fst = final_graph.optimize()

nemo_text_processing/inverse_text_normalization/ar/taggers/time.py

@@ -0,0 +1,41 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import pynini
+from nemo_text_processing.text_normalization.ar.graph_utils import NEMO_SIGMA, GraphFst
+from pynini.lib import pynutil
+
+
+class TimeFst(GraphFst):
+    """
+    Finite state transducer for classifying spoken Arabic time by inverting the
+    text-normalization time verbalizer, e.g.
+        الثالثة وخمس عشرة دقيقة -> time { hours: "3" minutes: "15" }
+        الخامسة وعشر دقائق وثانيتان -> time { hours: "5" minutes: "10" seconds: "2" }
+        التاسعة صباحًا -> time { hours: "9" suffix: "صباحًا" }
+
+    Args:
+        tn_time_verbalizer: TN time verbalizer, whose .graph maps the tagged fields
+            (hours/minutes/seconds/suffix/zone) to their spoken Arabic form.
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, tn_time_verbalizer: GraphFst, deterministic: bool = True):
+        super().__init__(name="time", kind="classify", deterministic=deterministic)
+        # allow the spoken form to be matched with flexible spacing
+        optional_delete_space = pynini.closure(NEMO_SIGMA | pynutil.delete(" ", weight=0.0001))
+        graph = (tn_time_verbalizer.graph @ optional_delete_space).invert().optimize()
+        self.fst = self.add_tokens(graph).optimize()

nemo_text_processing/inverse_text_normalization/ar/taggers/tokenize_and_classify.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,18 +17,24 @@
 
 import pynini
 from nemo_text_processing.inverse_text_normalization.ar.taggers.cardinal import CardinalFst
+from nemo_text_processing.inverse_text_normalization.ar.taggers.date import DateFst
 from nemo_text_processing.inverse_text_normalization.ar.taggers.decimal import DecimalFst
 from nemo_text_processing.inverse_text_normalization.ar.taggers.fraction import FractionFst
+from nemo_text_processing.inverse_text_normalization.ar.taggers.measure import MeasureFst
 from nemo_text_processing.inverse_text_normalization.ar.taggers.money import MoneyFst
 from nemo_text_processing.inverse_text_normalization.ar.taggers.punctuation import PunctuationFst
+from nemo_text_processing.inverse_text_normalization.ar.taggers.time import TimeFst
 from nemo_text_processing.inverse_text_normalization.ar.taggers.word import WordFst
 from nemo_text_processing.text_normalization.ar.graph_utils import (
     GraphFst,
     delete_extra_space,
     delete_space,
     generator_main,
 )
+from nemo_text_processing.text_normalization.ar.taggers.date import DateFst as TNDateTagger
 from nemo_text_processing.text_normalization.ar.taggers.tokenize_and_classify import ClassifyFst as TNClassifyFst
+from nemo_text_processing.text_normalization.ar.verbalizers.date import DateFst as TNDateVerbalizer
+from nemo_text_processing.text_normalization.ar.verbalizers.time import TimeFst as TNTimeVerbalizer
 from nemo_text_processing.text_normalization.en.graph_utils import INPUT_LOWER_CASED
 from pynini.lib import pynutil
 
@@ -76,6 +82,24 @@ def __init__(
             fraction_graph = fraction.fst
             money = MoneyFst(itn_cardinal_tagger=cardinal)
             money_graph = money.fst
+            measure = MeasureFst(
+                itn_cardinal_tagger=cardinal,
+                itn_decimal_tagger=decimal,
+                itn_fraction_tagger=fraction,
+                deterministic=True,
+            )
+            measure_graph = measure.fst
+            tn_time_verbalizer = TNTimeVerbalizer(cardinal_tagger=tn_classify.cardinal, deterministic=True)
+            time = TimeFst(tn_time_verbalizer=tn_time_verbalizer)
+            time_graph = time.fst
+            tn_date_tagger = TNDateTagger(cardinal=tn_classify.cardinal, deterministic=True)
+            tn_date_verbalizer = TNDateVerbalizer(deterministic=True)
+            date = DateFst(
+                itn_cardinal_tagger=cardinal,
+                tn_date_tagger=tn_date_tagger,
+                tn_date_verbalizer=tn_date_verbalizer,
+            )
+            date_graph = date.fst
             word_graph = WordFst().fst
             punct_graph = PunctuationFst().fst
 
@@ -84,6 +108,9 @@ def __init__(
                 | pynutil.add_weight(decimal_graph, 1.1)
                 | pynutil.add_weight(fraction_graph, 1.1)
                 | pynutil.add_weight(money_graph, 1.1)
+                | pynutil.add_weight(measure_graph, 1.1)
+                | pynutil.add_weight(time_graph, 1.1)
+                | pynutil.add_weight(date_graph, 1.1)
                 | pynutil.add_weight(word_graph, 100)
             )