From d717f371d534be7db090ef58cbf8df4b35669c6b Mon Sep 17 00:00:00 2001 From: ealbasiri Date: Thu, 27 Apr 2023 16:27:01 -0700 Subject: [PATCH 01/12] add measure coverage for TN and ITN Signed-off-by: ealbasiri --- Jenkinsfile | 2 +- .../ar/taggers/decimal.py | 6 +- .../ar/taggers/measure.py | 94 +++++++++++ .../ar/taggers/tokenize_and_classify.py | 6 +- .../ar/verbalizers/measure.py | 62 +++++++ .../ar/verbalizers/verbalize.py | 5 +- .../ar/data/measure/__init__.py | 13 ++ .../ar/data/measure/measurements.tsv | 14 ++ .../text_normalization/ar/taggers/measure.py | 156 ++++++++++++++++++ .../ar/taggers/tokenize_and_classify.py | 6 +- .../ar/verbalizers/measure.py | 57 +++++++ .../ar/verbalizers/verbalize.py | 7 +- tests/nemo_text_processing/ar/__init__.py | 13 ++ .../test_cases_measure.txt | 15 ++ .../test_cases_measure.txt | 14 ++ tests/nemo_text_processing/ar/test_measure.py | 74 +++++++++ 16 files changed, 535 insertions(+), 9 deletions(-) create mode 100644 nemo_text_processing/inverse_text_normalization/ar/taggers/measure.py create mode 100644 nemo_text_processing/inverse_text_normalization/ar/verbalizers/measure.py create mode 100644 nemo_text_processing/text_normalization/ar/data/measure/__init__.py create mode 100644 nemo_text_processing/text_normalization/ar/data/measure/measurements.tsv create mode 100644 nemo_text_processing/text_normalization/ar/taggers/measure.py create mode 100644 nemo_text_processing/text_normalization/ar/verbalizers/measure.py create mode 100644 tests/nemo_text_processing/ar/__init__.py create mode 100644 tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_measure.txt create mode 100644 tests/nemo_text_processing/ar/data_text_normalization/test_cases_measure.txt create mode 100644 tests/nemo_text_processing/ar/test_measure.py diff --git a/Jenkinsfile b/Jenkinsfile index 1337deee1..b182aceaa 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,7 +11,7 @@ pipeline { } environment { - AR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/02-15-23-0' + AR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/04-27-23-0' DE_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/02-15-23-0' EN_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/03-27-23-1' ES_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/04-05-23-0' diff --git a/nemo_text_processing/inverse_text_normalization/ar/taggers/decimal.py b/nemo_text_processing/inverse_text_normalization/ar/taggers/decimal.py index 87718a13e..fa14c4629 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/taggers/decimal.py +++ b/nemo_text_processing/inverse_text_normalization/ar/taggers/decimal.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -53,7 +53,7 @@ def __init__(self, tn_decimal): self.final_graph_wo_sign = ( graph_integer + pynini.accep(NEMO_SPACE) + graph_fractional + optional_graph_quantity ) - final_graph = optional_graph_negative + self.final_graph_wo_sign + self.final_graph_wo_negative = optional_graph_negative + self.final_graph_wo_sign - final_graph = self.add_tokens(final_graph) + final_graph = self.add_tokens(self.final_graph_wo_negative) self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ar/taggers/measure.py b/nemo_text_processing/inverse_text_normalization/ar/taggers/measure.py new file mode 100644 index 000000000..19b7b3071 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ar/taggers/measure.py @@ -0,0 +1,94 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.ar.taggers.measure import unit_singular +from nemo_text_processing.text_normalization.ar.graph_utils import ( + NEMO_SIGMA, + GraphFst, + convert_space, + delete_extra_space, + delete_space, +) +from pynini.lib import pynutil + + +class MeasureFst(GraphFst): + """ + Finite state transducer for classifying measure. Allows for plural form for unit. + e.g. "عشرون في المائة" -> measure { cardinal { integer: "20" } units: "%" } + + Args: + itn_cardinal_tagger: ITN Cardinal tagger + itn_decimal_tagger: ITN Decimal tagger + itn_fraction_tagger: ITN Fraction tagger + """ + + def __init__( + self, + itn_cardinal_tagger: GraphFst, + itn_decimal_tagger: GraphFst, + itn_fraction_tagger: GraphFst, + deterministic: bool = True, + ): + super().__init__(name="measure", kind="classify", deterministic=deterministic) + + cardinal_graph = itn_cardinal_tagger.graph + + graph_unit_singular = pynini.invert(unit_singular) + unit = convert_space(graph_unit_singular) + graph_unit_singular = convert_space(graph_unit_singular) + + optional_graph_negative = pynini.closure( + pynutil.insert("negative: ") + pynini.cross("سالب", "\"true\"") + delete_extra_space, 0, 1 + ) + + unit = ( + pynutil.insert("units: \"") + + (unit) + + pynutil.insert("\"") + ) + + subgraph_decimal = ( + pynutil.insert("decimal { ") + + optional_graph_negative + + itn_decimal_tagger.final_graph_wo_negative + + pynutil.insert(" }") + + delete_extra_space + + unit + ) + + subgraph_fraction = ( + pynutil.insert("decimal { ") + + optional_graph_negative + + pynutil.insert("integer_part: \"") + + itn_fraction_tagger.graph + + pynutil.insert("\" }") + + delete_extra_space + + unit + ) + + subgraph_cardinal = ( + pynutil.insert("cardinal { ") + + optional_graph_negative + + pynutil.insert("integer: \"") + + cardinal_graph + + pynutil.insert("\"") + + pynutil.insert(" }") + + delete_extra_space + + unit + ) + final_graph = subgraph_cardinal | subgraph_decimal | subgraph_fraction + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ar/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/ar/taggers/tokenize_and_classify.py index 6c1715a6c..cf1c03b47 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/inverse_text_normalization/ar/taggers/tokenize_and_classify.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ from nemo_text_processing.inverse_text_normalization.ar.taggers.decimal import DecimalFst from nemo_text_processing.inverse_text_normalization.ar.taggers.fraction import FractionFst from nemo_text_processing.inverse_text_normalization.ar.taggers.money import MoneyFst +from nemo_text_processing.inverse_text_normalization.ar.taggers.measure import MeasureFst from nemo_text_processing.inverse_text_normalization.ar.taggers.punctuation import PunctuationFst from nemo_text_processing.inverse_text_normalization.ar.taggers.word import WordFst from nemo_text_processing.text_normalization.ar.graph_utils import ( @@ -76,6 +77,8 @@ def __init__( fraction_graph = fraction.fst money = MoneyFst(itn_cardinal_tagger=cardinal) money_graph = money.fst + measure = MeasureFst(itn_cardinal_tagger=cardinal, itn_decimal_tagger=decimal, itn_fraction_tagger= fraction,deterministic= True) + measure_graph = measure.fst word_graph = WordFst().fst punct_graph = PunctuationFst().fst @@ -84,6 +87,7 @@ def __init__( | pynutil.add_weight(decimal_graph, 1.1) | pynutil.add_weight(fraction_graph, 1.1) | pynutil.add_weight(money_graph, 1.1) + | pynutil.add_weight(measure_graph, 1.1) | pynutil.add_weight(word_graph, 100) ) diff --git a/nemo_text_processing/inverse_text_normalization/ar/verbalizers/measure.py b/nemo_text_processing/inverse_text_normalization/ar/verbalizers/measure.py new file mode 100644 index 000000000..78299091b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ar/verbalizers/measure.py @@ -0,0 +1,62 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.ar.graph_utils import NEMO_CHAR, GraphFst, delete_space +from pynini.lib import pynutil + + +class MeasureFst(GraphFst): + """ + Finite state transducer for verbalizing measure, e.g. + measure { cardinal { integer: "20" } units: "%" } -> 20% + + Args: + decimal: ITN Decimal verbalizer + cardinal: ITN Cardinal verbalizer + """ + + def __init__(self, decimal: GraphFst, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="measure", kind="verbalize", deterministic=deterministic) + optional_sign = pynini.closure(pynini.cross("negative: \"true\"", "-"), 0, 1) + unit = ( + pynutil.delete("units:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_CHAR - " ", 1) + + pynutil.delete("\"") + + delete_space + ) + graph_decimal = ( + pynutil.delete("decimal {") + + delete_space + + optional_sign + + delete_space + + decimal.numbers + + delete_space + + pynutil.delete("}") + ) + graph_cardinal = ( + pynutil.delete("cardinal {") + + delete_space + + optional_sign + + delete_space + + cardinal.numbers + + delete_space + + pynutil.delete("}") + ) + + graph = (graph_cardinal | graph_decimal) + delete_space + pynutil.insert("") + unit + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ar/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/ar/verbalizers/verbalize.py index b14cf67f5..b49ea0948 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/verbalizers/verbalize.py +++ b/nemo_text_processing/inverse_text_normalization/ar/verbalizers/verbalize.py @@ -17,6 +17,7 @@ from nemo_text_processing.inverse_text_normalization.ar.verbalizers.decimal import DecimalFst from nemo_text_processing.inverse_text_normalization.ar.verbalizers.fraction import FractionFst from nemo_text_processing.inverse_text_normalization.ar.verbalizers.money import MoneyFst +from nemo_text_processing.inverse_text_normalization.ar.verbalizers.measure import MeasureFst from nemo_text_processing.text_normalization.ar.graph_utils import GraphFst @@ -37,5 +38,7 @@ def __init__(self): fraction_graph = fraction.fst money = MoneyFst(decimal, deterministic=True) money_graph = money.fst - graph = cardinal_graph | decimal_graph | fraction_graph | money_graph + measure = MeasureFst(decimal=decimal, cardinal=cardinal, deterministic=True) + measure_graph = measure.fst + graph = cardinal_graph | decimal_graph | fraction_graph | money_graph | measure_graph self.fst = graph diff --git a/nemo_text_processing/text_normalization/ar/data/measure/__init__.py b/nemo_text_processing/text_normalization/ar/data/measure/__init__.py new file mode 100644 index 000000000..7d200df57 --- /dev/null +++ b/nemo_text_processing/text_normalization/ar/data/measure/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ar/data/measure/measurements.tsv b/nemo_text_processing/text_normalization/ar/data/measure/measurements.tsv new file mode 100644 index 000000000..b41e2ef1a --- /dev/null +++ b/nemo_text_processing/text_normalization/ar/data/measure/measurements.tsv @@ -0,0 +1,14 @@ +% في المائة +f فهرنهايت +°C درجة مئوية +°F درجة فهرنهايت +كم كيلومتر +كلم كيلومتر +م متر +سم سنتيمتر +ملم مليمتر +هم هكتومتر +م² متر مربع +كم² كيلو متر مربع +مم² مليمتر مربع +سم² سنتيمتر مربع \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ar/taggers/measure.py b/nemo_text_processing/text_normalization/ar/taggers/measure.py new file mode 100644 index 000000000..352b1c7a5 --- /dev/null +++ b/nemo_text_processing/text_normalization/ar/taggers/measure.py @@ -0,0 +1,156 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.ar.utils import get_abs_path +from nemo_text_processing.text_normalization.ar.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_NON_BREAKING_SPACE, + NEMO_SIGMA, + GraphFst, + convert_space, + insert_space, +) +from pynini.lib import pynutil + +unit_singular = pynini.string_file(get_abs_path("data/measure/measurements.tsv")) + +class MeasureFst(GraphFst): + """ + Finite state transducer for classifying measure, e.g. + "20%" -> measure { cardinal { integer: "20" } units: "%" } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, deterministic: bool = True): + super().__init__(name="measure", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.cardinal_numbers + graph_unit = pynini.string_file(get_abs_path("data/measure/measurements.tsv")) + + graph_unit_singular = convert_space(graph_unit) + optional_graph_negative = pynini.closure("-", 0, 1) + + graph_unit_denominator = ( + pynini.cross("/", "في") + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit_singular | graph_unit_singular + pynutil.insert(NEMO_NON_BREAKING_SPACE) + pynini.cross("/", "في") + ) + + optional_unit_denominator = pynini.closure( + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit_denominator, 0, 1, + ) + + unit_plural = ( + pynutil.insert("units: \"") + (graph_unit_singular + (optional_unit_denominator) | graph_unit_denominator) + pynutil.insert("\"") + ) + + unit_singular_graph = ( + pynutil.insert("units: \"") + + ((graph_unit_singular + optional_unit_denominator) | graph_unit_denominator) + + pynutil.insert("\"") + ) + + subgraph_decimal = decimal.fst + insert_space + pynini.closure(pynutil.delete(" "), 0, 1) + unit_plural | unit_plural + pynini.closure(pynutil.delete(" "), 0, 1) + insert_space + decimal.fst + + subgraph_cardinal = ( + (optional_graph_negative + (pynini.closure(NEMO_DIGIT) - "1")) @ cardinal.fst + + insert_space + + pynini.closure(pynutil.delete(" "), 0, 1) + + unit_plural | unit_plural + pynini.closure(pynutil.delete(" "), 0, 1) + insert_space + (optional_graph_negative + (pynini.closure(NEMO_DIGIT) - "1")) @ cardinal.fst + ) + + subgraph_cardinal |= ( + (optional_graph_negative + pynini.accep("1")) + @ cardinal.fst + # @ pynini.cdrewrite(pynini.cross("واحد", ""), "", "", NEMO_SIGMA) + + insert_space + + pynini.closure(pynutil.delete(" "), 0, 1) + + unit_singular_graph + ) + + subgraph_fraction = fraction.fst + insert_space + pynini.closure(pynutil.delete(" "), 0, 1) + unit_plural + subgraph_fraction |= unit_plural + pynini.closure(pynutil.delete(" "), 0, 1) + insert_space + fraction.fst + + cardinal_dash_alpha = ( + pynutil.insert("cardinal { integer: \"") + + cardinal_graph + + pynutil.delete('-') + + pynutil.insert("\" } units: \"") + + pynini.closure(NEMO_ALPHA, 1) + + pynutil.insert("\"") + ) + + alpha_dash_cardinal = ( + pynutil.insert("units: \"") + + pynini.closure(NEMO_ALPHA, 1) + + pynutil.delete('-') + + pynutil.insert("\"") + + pynutil.insert(" cardinal { integer: \"") + + cardinal_graph + + pynutil.insert("\" }") + ) + + decimal_dash_alpha = ( + pynutil.insert("decimal { ") + + decimal.final_graph_decimal + + pynutil.delete('-') + + pynutil.insert(" } units: \"") + + pynini.closure(NEMO_ALPHA, 1) + + pynutil.insert("\"") + ) + + decimal_times = ( + pynutil.insert("decimal { ") + + decimal.final_graph_decimal + + pynutil.insert(" } units: \"") + + pynini.union('x', 'X') + + pynutil.insert("\"") + ) + + cardinal_times = ( + pynutil.insert("cardinal { integer: \"") + + cardinal_graph + + pynutil.insert("\" } units: \"") + + pynini.union('x', 'X') + + pynutil.insert("\"") + ) + + alpha_dash_decimal = ( + pynutil.insert("units: \"") + + pynini.closure(NEMO_ALPHA, 1) + + pynutil.delete('-') + + pynutil.insert("\"") + + pynutil.insert(" decimal { ") + + decimal.final_graph_decimal + + pynutil.insert(" }") + ) + + final_graph = ( + subgraph_decimal + | subgraph_cardinal + | cardinal_dash_alpha + | alpha_dash_cardinal + | decimal_dash_alpha + | decimal_times + | alpha_dash_decimal + | subgraph_fraction + | cardinal_times + ) + final_graph = self.add_tokens(final_graph) + + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/ar/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/ar/taggers/tokenize_and_classify.py index bb1d1b986..b7a28d642 100644 --- a/nemo_text_processing/text_normalization/ar/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/text_normalization/ar/taggers/tokenize_and_classify.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -29,6 +29,7 @@ from nemo_text_processing.text_normalization.ar.taggers.fraction import FractionFst from nemo_text_processing.text_normalization.ar.taggers.money import MoneyFst from nemo_text_processing.text_normalization.ar.taggers.word import WordFst +from nemo_text_processing.text_normalization.ar.taggers.measure import MeasureFst from nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst from pynini.lib import pynutil @@ -80,6 +81,8 @@ def __init__( fraction_graph = self.fraction.fst self.money = MoneyFst(cardinal=self.cardinal) money_graph = self.money.fst + self.measure = MeasureFst(cardinal=self.cardinal, decimal=self.decimal, fraction=self.fraction, deterministic=deterministic) + measure_graph = self.measure.fst word_graph = WordFst(deterministic=deterministic).fst punct_graph = PunctuationFst(deterministic=deterministic).fst @@ -88,6 +91,7 @@ def __init__( | pynutil.add_weight(decimal_graph, 1.1) | pynutil.add_weight(fraction_graph, 1.0) | pynutil.add_weight(money_graph, 1.0) + | pynutil.add_weight(measure_graph, 1.0) ) classify |= pynutil.add_weight(word_graph, 100) diff --git a/nemo_text_processing/text_normalization/ar/verbalizers/measure.py b/nemo_text_processing/text_normalization/ar/verbalizers/measure.py new file mode 100644 index 000000000..d1907a482 --- /dev/null +++ b/nemo_text_processing/text_normalization/ar/verbalizers/measure.py @@ -0,0 +1,57 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.ar.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_extra_space, + delete_space, + delete_preserve_order, + delete_zero_or_one_space +) +from pynini.lib import pynutil + + +class MeasureFst(GraphFst): + """ + Finite state transducer for verbalizing measure, e.g. + measure { cardinal { integer: "20" } units: "%" } -> "عشرون في المائة" + + Args: + decimal: decimal GraphFst + cardinal: cardinal GraphFst + fraction: fraction GraphFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, decimal: GraphFst, cardinal: GraphFst, fraction: GraphFst, deterministic: bool): + super().__init__(name="measure", kind="verbalize", deterministic=deterministic) + unit = pynutil.delete("units: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + + graph_decimal = decimal.fst + graph_cardinal = cardinal.fst + graph_fraction = fraction.fst + + graph = (graph_cardinal | graph_decimal | graph_fraction) + pynini.accep(" ") + unit + + graph |= (graph_cardinal | graph_decimal | graph_fraction) + delete_extra_space + unit + + #graph |= unit + delete_extra_space + (graph_cardinal | graph_decimal) + + graph += delete_preserve_order + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/ar/verbalizers/verbalize.py b/nemo_text_processing/text_normalization/ar/verbalizers/verbalize.py index 14b548b59..d114b211d 100644 --- a/nemo_text_processing/text_normalization/ar/verbalizers/verbalize.py +++ b/nemo_text_processing/text_normalization/ar/verbalizers/verbalize.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ from nemo_text_processing.text_normalization.ar.verbalizers.decimal import DecimalFst from nemo_text_processing.text_normalization.ar.verbalizers.fraction import FractionFst from nemo_text_processing.text_normalization.ar.verbalizers.money import MoneyFst +from nemo_text_processing.text_normalization.ar.verbalizers.measure import MeasureFst class VerbalizeFst(GraphFst): @@ -42,6 +43,8 @@ def __init__(self, deterministic: bool = True): fraction_graph = fraction.fst money = MoneyFst() money_graph = money.fst + measure = MeasureFst(decimal=decimal, cardinal=cardinal, fraction=fraction, deterministic=deterministic) + measure_graph = measure.fst - graph = cardinal_graph | decimal_graph | fraction_graph | money_graph + graph = cardinal_graph | decimal_graph | fraction_graph | money_graph | measure_graph self.fst = graph diff --git a/tests/nemo_text_processing/ar/__init__.py b/tests/nemo_text_processing/ar/__init__.py new file mode 100644 index 000000000..4fc50543f --- /dev/null +++ b/tests/nemo_text_processing/ar/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_measure.txt new file mode 100644 index 000000000..bb8e3a15a --- /dev/null +++ b/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_measure.txt @@ -0,0 +1,15 @@ +سبعة في المائة~7% +تسعة فهرنهايت~9f +اثنا عشر في المائة~12% +تسعة في المائة~9% +عشرون في المائة~20% +ستة وثمانية وثمانون من مئة في المائة~6.88% +سبعة وتسعة وتسعون من مئة في المائة~7.99% +أربعة وتسعة اَلاف وثمان مئة وستة وسبعون من عشرة آلاف في المائة~4.9876% +ستة وثمانية اَلاف وتسع مئة وأربعة وأربعون من عشرة آلاف في المائة~6.8944% +ستة اَلاف وسبع مئة وثمانية وتسعون وتسعة من عشرة في المائة~6798.9% +ست مئة وثمانية وخمسون وثمان مئة وثمانية وثمانون من ألف في المائة~658.888% +اثنان وثمانية من عشرة في المائة~2.8% +سبعة وتسعة من عشرة في المائة~7.9% +واحد في المائة~1% +ألفان وتسعة في المائة~2009% diff --git a/tests/nemo_text_processing/ar/data_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/ar/data_text_normalization/test_cases_measure.txt new file mode 100644 index 000000000..a382d7a9f --- /dev/null +++ b/tests/nemo_text_processing/ar/data_text_normalization/test_cases_measure.txt @@ -0,0 +1,14 @@ +سبعة في المائة~%7 +تسعة فهرنهايت~f9 +اثنا عشر في المائة~12% +تسعة في المائة~09% +عشرون في المائة~20% +ستة وثمانية وثمانون من مئة في المائة~%6.88 +سبعة وتسعة وتسعون من مئة في المائة~%7,99 +أربعة وتسعة اَلاف وثمان مئة وستة وسبعون من عشرة آلاف في المائة~%4.9876 +ستة وثمانية اَلاف وتسع مئة وأربعة وأربعون من عشرة آلاف في المائة~%6,8944 +ستة اَلاف وسبع مئة وثمانية وتسعون وتسعة من عشرة في المائة~%6798,9 +ست مئة وثمانية وخمسون وثمان مئة وثمانية وثمانون من ألف في المائة~%658,888 +اثنان وثمانية من عشرة في المائة~2.8% +سبعة وتسعة من عشرة في المائة~%7.9 +واحد في المائة~1% diff --git a/tests/nemo_text_processing/ar/test_measure.py b/tests/nemo_text_processing/ar/test_measure.py new file mode 100644 index 000000000..79644f80d --- /dev/null +++ b/tests/nemo_text_processing/ar/test_measure.py @@ -0,0 +1,74 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +from parameterized import parameterized + +from tests.nemo_text_processing.utils import CACHE_DIR, parse_test_case_file + +try: + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + from nemo_text_processing.text_normalization.normalize import Normalizer + from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio + + PYNINI_AVAILABLE = True +except (ImportError, ModuleNotFoundError): + PYNINI_AVAILABLE = False + + +class TestMeasure: + inverse_normalizer = ( + InverseNormalizer(lang='ar', cache_dir=CACHE_DIR, overwrite_cache=False) if PYNINI_AVAILABLE else None + ) + + @parameterized.expand(parse_test_case_file('ar/data_inverse_text_normalization/test_cases_measure.txt')) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, + reason="`pynini` not installed, please install via nemo_text_processing/pynini_install.sh", + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + normalizer = ( + Normalizer(input_case='cased', lang='ar', cache_dir=CACHE_DIR, overwrite_cache=False) + if PYNINI_AVAILABLE + else None + ) + + normalizer_with_audio = ( + NormalizerWithAudio(input_case='cased', lang='ar', cache_dir=CACHE_DIR, overwrite_cache=False) + if CACHE_DIR and RUN_AUDIO_BASED_TESTS and PYNINI_AVAILABLE + else None + ) + + @parameterized.expand(parse_test_case_file('ar/data_text_normalization/test_cases_measure.txt')) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, + reason="`pynini` not installed, please install via nemo_text_processing/pynini_install.sh", + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, expected, test_input): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected + + if self.normalizer_with_audio: + pred_non_deterministic = self.normalizer_with_audio.normalize( + test_input, n_tagged=1000, punct_post_process=False + ) + assert expected in pred_non_deterministic From 25e485fc2c250a39af45501c42e5469fdacbb022 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 28 Apr 2023 00:03:46 +0000 Subject: [PATCH 02/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../ar/taggers/measure.py | 10 ++----- .../ar/taggers/tokenize_and_classify.py | 9 ++++-- .../ar/verbalizers/verbalize.py | 2 +- .../text_normalization/ar/taggers/measure.py | 29 +++++++++++++------ .../ar/taggers/tokenize_and_classify.py | 6 ++-- .../ar/verbalizers/measure.py | 8 ++--- .../ar/verbalizers/verbalize.py | 2 +- 7 files changed, 40 insertions(+), 26 deletions(-) diff --git a/nemo_text_processing/inverse_text_normalization/ar/taggers/measure.py b/nemo_text_processing/inverse_text_normalization/ar/taggers/measure.py index 19b7b3071..b5395fd40 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/taggers/measure.py +++ b/nemo_text_processing/inverse_text_normalization/ar/taggers/measure.py @@ -13,7 +13,6 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.ar.taggers.measure import unit_singular from nemo_text_processing.text_normalization.ar.graph_utils import ( NEMO_SIGMA, GraphFst, @@ -21,6 +20,7 @@ delete_extra_space, delete_space, ) +from nemo_text_processing.text_normalization.ar.taggers.measure import unit_singular from pynini.lib import pynutil @@ -46,7 +46,7 @@ def __init__( cardinal_graph = itn_cardinal_tagger.graph - graph_unit_singular = pynini.invert(unit_singular) + graph_unit_singular = pynini.invert(unit_singular) unit = convert_space(graph_unit_singular) graph_unit_singular = convert_space(graph_unit_singular) @@ -54,11 +54,7 @@ def __init__( pynutil.insert("negative: ") + pynini.cross("سالب", "\"true\"") + delete_extra_space, 0, 1 ) - unit = ( - pynutil.insert("units: \"") - + (unit) - + pynutil.insert("\"") - ) + unit = pynutil.insert("units: \"") + (unit) + pynutil.insert("\"") subgraph_decimal = ( pynutil.insert("decimal { ") diff --git a/nemo_text_processing/inverse_text_normalization/ar/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/ar/taggers/tokenize_and_classify.py index cf1c03b47..8f6dd0782 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/inverse_text_normalization/ar/taggers/tokenize_and_classify.py @@ -19,8 +19,8 @@ from nemo_text_processing.inverse_text_normalization.ar.taggers.cardinal import CardinalFst from nemo_text_processing.inverse_text_normalization.ar.taggers.decimal import DecimalFst from nemo_text_processing.inverse_text_normalization.ar.taggers.fraction import FractionFst -from nemo_text_processing.inverse_text_normalization.ar.taggers.money import MoneyFst from nemo_text_processing.inverse_text_normalization.ar.taggers.measure import MeasureFst +from nemo_text_processing.inverse_text_normalization.ar.taggers.money import MoneyFst from nemo_text_processing.inverse_text_normalization.ar.taggers.punctuation import PunctuationFst from nemo_text_processing.inverse_text_normalization.ar.taggers.word import WordFst from nemo_text_processing.text_normalization.ar.graph_utils import ( @@ -77,7 +77,12 @@ def __init__( fraction_graph = fraction.fst money = MoneyFst(itn_cardinal_tagger=cardinal) money_graph = money.fst - measure = MeasureFst(itn_cardinal_tagger=cardinal, itn_decimal_tagger=decimal, itn_fraction_tagger= fraction,deterministic= True) + measure = MeasureFst( + itn_cardinal_tagger=cardinal, + itn_decimal_tagger=decimal, + itn_fraction_tagger=fraction, + deterministic=True, + ) measure_graph = measure.fst word_graph = WordFst().fst punct_graph = PunctuationFst().fst diff --git a/nemo_text_processing/inverse_text_normalization/ar/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/ar/verbalizers/verbalize.py index b49ea0948..e9f1758f8 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/verbalizers/verbalize.py +++ b/nemo_text_processing/inverse_text_normalization/ar/verbalizers/verbalize.py @@ -16,8 +16,8 @@ from nemo_text_processing.inverse_text_normalization.ar.verbalizers.cardinal import CardinalFst from nemo_text_processing.inverse_text_normalization.ar.verbalizers.decimal import DecimalFst from nemo_text_processing.inverse_text_normalization.ar.verbalizers.fraction import FractionFst -from nemo_text_processing.inverse_text_normalization.ar.verbalizers.money import MoneyFst from nemo_text_processing.inverse_text_normalization.ar.verbalizers.measure import MeasureFst +from nemo_text_processing.inverse_text_normalization.ar.verbalizers.money import MoneyFst from nemo_text_processing.text_normalization.ar.graph_utils import GraphFst diff --git a/nemo_text_processing/text_normalization/ar/taggers/measure.py b/nemo_text_processing/text_normalization/ar/taggers/measure.py index 352b1c7a5..a9e53cbc4 100644 --- a/nemo_text_processing/text_normalization/ar/taggers/measure.py +++ b/nemo_text_processing/text_normalization/ar/taggers/measure.py @@ -13,7 +13,6 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.ar.utils import get_abs_path from nemo_text_processing.text_normalization.ar.graph_utils import ( NEMO_ALPHA, NEMO_DIGIT, @@ -23,10 +22,12 @@ convert_space, insert_space, ) +from nemo_text_processing.text_normalization.ar.utils import get_abs_path from pynini.lib import pynutil unit_singular = pynini.string_file(get_abs_path("data/measure/measurements.tsv")) + class MeasureFst(GraphFst): """ Finite state transducer for classifying measure, e.g. @@ -47,8 +48,10 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, de graph_unit_singular = convert_space(graph_unit) optional_graph_negative = pynini.closure("-", 0, 1) - graph_unit_denominator = ( - pynini.cross("/", "في") + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit_singular | graph_unit_singular + pynutil.insert(NEMO_NON_BREAKING_SPACE) + pynini.cross("/", "في") + graph_unit_denominator = pynini.cross("/", "في") + pynutil.insert( + NEMO_NON_BREAKING_SPACE + ) + graph_unit_singular | graph_unit_singular + pynutil.insert(NEMO_NON_BREAKING_SPACE) + pynini.cross( + "/", "في" ) optional_unit_denominator = pynini.closure( @@ -56,7 +59,9 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, de ) unit_plural = ( - pynutil.insert("units: \"") + (graph_unit_singular + (optional_unit_denominator) | graph_unit_denominator) + pynutil.insert("\"") + pynutil.insert("units: \"") + + (graph_unit_singular + (optional_unit_denominator) | graph_unit_denominator) + + pynutil.insert("\"") ) unit_singular_graph = ( @@ -65,18 +70,24 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, de + pynutil.insert("\"") ) - subgraph_decimal = decimal.fst + insert_space + pynini.closure(pynutil.delete(" "), 0, 1) + unit_plural | unit_plural + pynini.closure(pynutil.delete(" "), 0, 1) + insert_space + decimal.fst + subgraph_decimal = ( + decimal.fst + insert_space + pynini.closure(pynutil.delete(" "), 0, 1) + unit_plural + | unit_plural + pynini.closure(pynutil.delete(" "), 0, 1) + insert_space + decimal.fst + ) subgraph_cardinal = ( (optional_graph_negative + (pynini.closure(NEMO_DIGIT) - "1")) @ cardinal.fst + insert_space + pynini.closure(pynutil.delete(" "), 0, 1) - + unit_plural | unit_plural + pynini.closure(pynutil.delete(" "), 0, 1) + insert_space + (optional_graph_negative + (pynini.closure(NEMO_DIGIT) - "1")) @ cardinal.fst + + unit_plural + | unit_plural + + pynini.closure(pynutil.delete(" "), 0, 1) + + insert_space + + (optional_graph_negative + (pynini.closure(NEMO_DIGIT) - "1")) @ cardinal.fst ) subgraph_cardinal |= ( - (optional_graph_negative + pynini.accep("1")) - @ cardinal.fst + (optional_graph_negative + pynini.accep("1")) @ cardinal.fst # @ pynini.cdrewrite(pynini.cross("واحد", ""), "", "", NEMO_SIGMA) + insert_space + pynini.closure(pynutil.delete(" "), 0, 1) @@ -84,7 +95,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, de ) subgraph_fraction = fraction.fst + insert_space + pynini.closure(pynutil.delete(" "), 0, 1) + unit_plural - subgraph_fraction |= unit_plural + pynini.closure(pynutil.delete(" "), 0, 1) + insert_space + fraction.fst + subgraph_fraction |= unit_plural + pynini.closure(pynutil.delete(" "), 0, 1) + insert_space + fraction.fst cardinal_dash_alpha = ( pynutil.insert("cardinal { integer: \"") diff --git a/nemo_text_processing/text_normalization/ar/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/ar/taggers/tokenize_and_classify.py index b7a28d642..833652d76 100644 --- a/nemo_text_processing/text_normalization/ar/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/text_normalization/ar/taggers/tokenize_and_classify.py @@ -27,9 +27,9 @@ from nemo_text_processing.text_normalization.ar.taggers.cardinal import CardinalFst from nemo_text_processing.text_normalization.ar.taggers.decimal import DecimalFst from nemo_text_processing.text_normalization.ar.taggers.fraction import FractionFst +from nemo_text_processing.text_normalization.ar.taggers.measure import MeasureFst from nemo_text_processing.text_normalization.ar.taggers.money import MoneyFst from nemo_text_processing.text_normalization.ar.taggers.word import WordFst -from nemo_text_processing.text_normalization.ar.taggers.measure import MeasureFst from nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst from pynini.lib import pynutil @@ -81,7 +81,9 @@ def __init__( fraction_graph = self.fraction.fst self.money = MoneyFst(cardinal=self.cardinal) money_graph = self.money.fst - self.measure = MeasureFst(cardinal=self.cardinal, decimal=self.decimal, fraction=self.fraction, deterministic=deterministic) + self.measure = MeasureFst( + cardinal=self.cardinal, decimal=self.decimal, fraction=self.fraction, deterministic=deterministic + ) measure_graph = self.measure.fst word_graph = WordFst(deterministic=deterministic).fst punct_graph = PunctuationFst(deterministic=deterministic).fst diff --git a/nemo_text_processing/text_normalization/ar/verbalizers/measure.py b/nemo_text_processing/text_normalization/ar/verbalizers/measure.py index d1907a482..6134925f4 100644 --- a/nemo_text_processing/text_normalization/ar/verbalizers/measure.py +++ b/nemo_text_processing/text_normalization/ar/verbalizers/measure.py @@ -17,9 +17,9 @@ NEMO_NOT_QUOTE, GraphFst, delete_extra_space, - delete_space, delete_preserve_order, - delete_zero_or_one_space + delete_space, + delete_zero_or_one_space, ) from pynini.lib import pynutil @@ -47,9 +47,9 @@ def __init__(self, decimal: GraphFst, cardinal: GraphFst, fraction: GraphFst, de graph = (graph_cardinal | graph_decimal | graph_fraction) + pynini.accep(" ") + unit - graph |= (graph_cardinal | graph_decimal | graph_fraction) + delete_extra_space + unit + graph |= (graph_cardinal | graph_decimal | graph_fraction) + delete_extra_space + unit - #graph |= unit + delete_extra_space + (graph_cardinal | graph_decimal) + # graph |= unit + delete_extra_space + (graph_cardinal | graph_decimal) graph += delete_preserve_order diff --git a/nemo_text_processing/text_normalization/ar/verbalizers/verbalize.py b/nemo_text_processing/text_normalization/ar/verbalizers/verbalize.py index d114b211d..d693abf28 100644 --- a/nemo_text_processing/text_normalization/ar/verbalizers/verbalize.py +++ b/nemo_text_processing/text_normalization/ar/verbalizers/verbalize.py @@ -17,8 +17,8 @@ from nemo_text_processing.text_normalization.ar.verbalizers.cardinal import CardinalFst from nemo_text_processing.text_normalization.ar.verbalizers.decimal import DecimalFst from nemo_text_processing.text_normalization.ar.verbalizers.fraction import FractionFst -from nemo_text_processing.text_normalization.ar.verbalizers.money import MoneyFst from nemo_text_processing.text_normalization.ar.verbalizers.measure import MeasureFst +from nemo_text_processing.text_normalization.ar.verbalizers.money import MoneyFst class VerbalizeFst(GraphFst): From 4717919619f71590ec707b9b48f0ef2de20dbb19 Mon Sep 17 00:00:00 2001 From: anand-nv <105917641+anand-nv@users.noreply.github.com> Date: Thu, 1 Jun 2023 10:36:52 +0530 Subject: [PATCH 03/12] Remove unused imports Signed-off-by: anand-nv <105917641+anand-nv@users.noreply.github.com> --- .../inverse_text_normalization/ar/taggers/measure.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nemo_text_processing/inverse_text_normalization/ar/taggers/measure.py b/nemo_text_processing/inverse_text_normalization/ar/taggers/measure.py index b5395fd40..cf277bbc8 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/taggers/measure.py +++ b/nemo_text_processing/inverse_text_normalization/ar/taggers/measure.py @@ -14,11 +14,9 @@ import pynini from nemo_text_processing.text_normalization.ar.graph_utils import ( - NEMO_SIGMA, GraphFst, convert_space, - delete_extra_space, - delete_space, + delete_extra_space ) from nemo_text_processing.text_normalization.ar.taggers.measure import unit_singular from pynini.lib import pynutil From f684fca0ec57c8b75b56f06892ade0a992a618fe Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 1 Jun 2023 05:07:05 +0000 Subject: [PATCH 04/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../inverse_text_normalization/ar/taggers/measure.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/nemo_text_processing/inverse_text_normalization/ar/taggers/measure.py b/nemo_text_processing/inverse_text_normalization/ar/taggers/measure.py index cf277bbc8..ea74c7863 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/taggers/measure.py +++ b/nemo_text_processing/inverse_text_normalization/ar/taggers/measure.py @@ -13,11 +13,7 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.ar.graph_utils import ( - GraphFst, - convert_space, - delete_extra_space -) +from nemo_text_processing.text_normalization.ar.graph_utils import GraphFst, convert_space, delete_extra_space from nemo_text_processing.text_normalization.ar.taggers.measure import unit_singular from pynini.lib import pynutil From fe435fbc53aa441fd63ece80c901030112a8821a Mon Sep 17 00:00:00 2001 From: anand-nv <105917641+anand-nv@users.noreply.github.com> Date: Thu, 1 Jun 2023 10:42:31 +0530 Subject: [PATCH 05/12] Remove unused imports Signed-off-by: anand-nv <105917641+anand-nv@users.noreply.github.com> --- nemo_text_processing/text_normalization/ar/taggers/measure.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nemo_text_processing/text_normalization/ar/taggers/measure.py b/nemo_text_processing/text_normalization/ar/taggers/measure.py index a9e53cbc4..2aca51201 100644 --- a/nemo_text_processing/text_normalization/ar/taggers/measure.py +++ b/nemo_text_processing/text_normalization/ar/taggers/measure.py @@ -17,7 +17,6 @@ NEMO_ALPHA, NEMO_DIGIT, NEMO_NON_BREAKING_SPACE, - NEMO_SIGMA, GraphFst, convert_space, insert_space, From 4f6f38c9b0bac16583d061d0fcdab1c9566d6903 Mon Sep 17 00:00:00 2001 From: anand-nv <105917641+anand-nv@users.noreply.github.com> Date: Thu, 1 Jun 2023 10:43:44 +0530 Subject: [PATCH 06/12] Remove unused imports Signed-off-by: anand-nv <105917641+anand-nv@users.noreply.github.com> --- .../text_normalization/ar/verbalizers/measure.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nemo_text_processing/text_normalization/ar/verbalizers/measure.py b/nemo_text_processing/text_normalization/ar/verbalizers/measure.py index 6134925f4..328461f09 100644 --- a/nemo_text_processing/text_normalization/ar/verbalizers/measure.py +++ b/nemo_text_processing/text_normalization/ar/verbalizers/measure.py @@ -17,9 +17,7 @@ NEMO_NOT_QUOTE, GraphFst, delete_extra_space, - delete_preserve_order, - delete_space, - delete_zero_or_one_space, + delete_preserve_order ) from pynini.lib import pynutil From 59afc4428fad914a4a91704f8a6d6a873dc803e9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 1 Jun 2023 05:13:57 +0000 Subject: [PATCH 07/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../text_normalization/ar/verbalizers/measure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo_text_processing/text_normalization/ar/verbalizers/measure.py b/nemo_text_processing/text_normalization/ar/verbalizers/measure.py index 328461f09..e2f3d19f8 100644 --- a/nemo_text_processing/text_normalization/ar/verbalizers/measure.py +++ b/nemo_text_processing/text_normalization/ar/verbalizers/measure.py @@ -17,7 +17,7 @@ NEMO_NOT_QUOTE, GraphFst, delete_extra_space, - delete_preserve_order + delete_preserve_order, ) from pynini.lib import pynutil From d7007ae9bbe8ceb40c83496a52c5f094e8046c19 Mon Sep 17 00:00:00 2001 From: anand-nv <105917641+anand-nv@users.noreply.github.com> Date: Thu, 1 Jun 2023 10:45:58 +0530 Subject: [PATCH 08/12] Update measure.py Signed-off-by: anand-nv <105917641+anand-nv@users.noreply.github.com> --- .../inverse_text_normalization/ar/taggers/measure.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nemo_text_processing/inverse_text_normalization/ar/taggers/measure.py b/nemo_text_processing/inverse_text_normalization/ar/taggers/measure.py index ea74c7863..ac3fc7fbe 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/taggers/measure.py +++ b/nemo_text_processing/inverse_text_normalization/ar/taggers/measure.py @@ -42,7 +42,6 @@ def __init__( graph_unit_singular = pynini.invert(unit_singular) unit = convert_space(graph_unit_singular) - graph_unit_singular = convert_space(graph_unit_singular) optional_graph_negative = pynini.closure( pynutil.insert("negative: ") + pynini.cross("سالب", "\"true\"") + delete_extra_space, 0, 1 From bc46f66f4eaeb9acde6bd3ee9da78c5ec5a59352 Mon Sep 17 00:00:00 2001 From: Enas Albasiri Date: Thu, 2 Jul 2026 16:14:13 +0000 Subject: [PATCH 09/12] =?UTF-8?q?feat(ar):=20expand=20cardinal=20to=20mill?= =?UTF-8?q?ions,=20fix=20orthography,=20agreement,=20and=20fraction=20homo?= =?UTF-8?q?graph=20Cardinal=20(TN/ITN):=20-=20Expand=20to=20millions=20wit?= =?UTF-8?q?h=20place-value=203-digit=20period=20decomposition=20-=20Orthog?= =?UTF-8?q?raphy:=20=D9=85=D8=A6=D8=A9=20(drop=20silent=20alif),=20=D9=85?= =?UTF-8?q?=D8=A6=D8=AA=D9=8A=D9=86,=20separated=20compound=20hundreds=20?= =?UTF-8?q?=20=20(=D8=AB=D9=84=D8=A7=D8=AB=20=D9=85=D8=A6=D8=A9),=20=D9=85?= =?UTF-8?q?=D8=A6=D8=AA=D9=8A=20construct-state=20before=20counted=20nouns?= =?UTF-8?q?=20-=20Use=20genitive=20tens=20(=D8=B9=D8=B4=D8=B1=D9=8A=D9=86)?= =?UTF-8?q?=20and=20=D8=A7=D8=AB=D9=86=D9=8A=D9=86=20by=20default=20-=20Fi?= =?UTF-8?q?x=20counted-noun=20(=D8=AA=D9=85=D9=8A=D9=8A=D8=B2)=20agreement?= =?UTF-8?q?:=20trailing=203-10=20->=20plural=20=20=20(110000=20->=20=D9=85?= =?UTF-8?q?=D8=A6=D8=A9=20=D9=88=D8=B9=D8=B4=D8=B1=D8=A9=20=D8=A2=D9=84?= =?UTF-8?q?=D8=A7=D9=81),=20otherwise=20singular;=20applied=20to=20=D8=A2?= =?UTF-8?q?=D9=84=D8=A7=D9=81=20and=20=D9=85=D9=84=D8=A7=D9=8A=D9=8A=D9=86?= =?UTF-8?q?=20Fraction=20(TN/ITN):=20-=20Resolve=20homograph=20collision?= =?UTF-8?q?=20between=20dual=20fractions=20and=20cardinals=20by=20using=20?= =?UTF-8?q?=20=20construct-state=20dual=20denominators=20without=20final?= =?UTF-8?q?=20nun=20(=D8=B3=D8=A8=D8=B9=D9=8A=3D2/7=20vs=20=D8=B3=D8=A8?= =?UTF-8?q?=D8=B9=D9=8A=D9=86=3D70);=20=20=20both=20directions=20now=20rou?= =?UTF-8?q?nd-trip=20losslessly=20-=20Verbalizer=20accepts=20=D8=A7=D8=AB?= =?UTF-8?q?=D9=86=D9=8A=D9=86=20numerator=20Tests:=20-=20Fix=20pre-existin?= =?UTF-8?q?g=20NameError=20(RUN=5FAUDIO=5FBASED=5FTESTS)=20in=20ar/test=5F?= =?UTF-8?q?measure.py=20-=20Regenerate=20cardinal/fraction/decimal/money/m?= =?UTF-8?q?easure=20test=20data=20(TN+ITN)=20to=20the=20=20=20new=20canoni?= =?UTF-8?q?cal=20spelling;=20decimal/money/measure=20grammars=20unchanged?= =?UTF-8?q?=20-=20Full=20ar=20suite:=20306=20passed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ar/taggers/fraction.py | 2 +- .../ar/data/number/digit.tsv | 2 +- .../ar/data/number/fraction_dual.tsv | 18 +- .../ar/data/number/tens_gen.tsv | 10 ++ .../ar/data/number/{tens.tsv => tens_nom.tsv} | 0 .../ar/data/number/zero.tsv | 2 +- .../text_normalization/ar/taggers/cardinal.py | 157 +++++++++++------- .../ar/verbalizers/fraction.py | 10 +- .../test_cases_cardinal.txt | 89 ++++++---- .../test_cases_decimal.txt | 48 +++--- .../test_cases_fraction.txt | 16 +- .../test_cases_measure.txt | 18 +- .../test_cases_money.txt | 24 +-- .../test_cases_cardinal.txt | 82 ++++++--- .../test_cases_decimal.txt | 60 +++---- .../test_cases_fraction.txt | 16 +- .../test_cases_measure.txt | 16 +- .../test_cases_money.txt | 24 +-- tests/nemo_text_processing/ar/test_measure.py | 2 +- 19 files changed, 354 insertions(+), 242 deletions(-) create mode 100644 nemo_text_processing/text_normalization/ar/data/number/tens_gen.tsv rename nemo_text_processing/text_normalization/ar/data/number/{tens.tsv => tens_nom.tsv} (100%) diff --git a/nemo_text_processing/inverse_text_normalization/ar/taggers/fraction.py b/nemo_text_processing/inverse_text_normalization/ar/taggers/fraction.py index be18ac16f..34ae30400 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/taggers/fraction.py +++ b/nemo_text_processing/inverse_text_normalization/ar/taggers/fraction.py @@ -45,7 +45,7 @@ def __init__(self, tn_cardinal: GraphFst): # create unions for special cases denominator_singular = pynini.union("نصف", "ثلث", "ربع", "خمس", "سدس", "سبع", "ثمن", "تسع", "عشر") denominator_dual = pynini.union( - "نصفين", "ثلثين", "ربعين", "خمسين", "سدسين", "سبعين", "ثمنين", "تسعين", "عشرين" + "نصفي", "ثلثي", "ربعي", "خمسي", "سدسي", "سبعي", "ثمني", "تسعي", "عشري" ) denominator_plural = pynini.union("أخماس", "أرباع", "أثلاث", "أسداس", "أسباع", "أثمان", "أتساع", "أعشار") numerator_three_to_ten = pynini.union("خمسة", "سبعة", "عشرة", "ثلاثة", "أربعة", "ستة", "ثمانية", "تسعة") diff --git a/nemo_text_processing/text_normalization/ar/data/number/digit.tsv b/nemo_text_processing/text_normalization/ar/data/number/digit.tsv index 5892463f2..47fc27378 100644 --- a/nemo_text_processing/text_normalization/ar/data/number/digit.tsv +++ b/nemo_text_processing/text_normalization/ar/data/number/digit.tsv @@ -1,5 +1,5 @@ 1 واحد -2 اثنان +2 اثنين 3 ثلاثة 4 أربعة 5 خمسة diff --git a/nemo_text_processing/text_normalization/ar/data/number/fraction_dual.tsv b/nemo_text_processing/text_normalization/ar/data/number/fraction_dual.tsv index b81f4643b..49d5bb29b 100644 --- a/nemo_text_processing/text_normalization/ar/data/number/fraction_dual.tsv +++ b/nemo_text_processing/text_normalization/ar/data/number/fraction_dual.tsv @@ -1,9 +1,9 @@ -2 نصفين -3 ثلثين -4 ربعين -5 خمسين -6 سدسين -7 سبعين -8 ثمنين -9 تسعين -10 عشرين +2 نصفي +3 ثلثي +4 ربعي +5 خمسي +6 سدسي +7 سبعي +8 ثمني +9 تسعي +10 عشري diff --git a/nemo_text_processing/text_normalization/ar/data/number/tens_gen.tsv b/nemo_text_processing/text_normalization/ar/data/number/tens_gen.tsv new file mode 100644 index 000000000..397e6c933 --- /dev/null +++ b/nemo_text_processing/text_normalization/ar/data/number/tens_gen.tsv @@ -0,0 +1,10 @@ +1 عشرة +2 عشرين +3 ثلاثين +4 أربعين +5 خمسين +6 ستين +7 سبعين +8 ثمانين +9 تسعين +10 مئة diff --git a/nemo_text_processing/text_normalization/ar/data/number/tens.tsv b/nemo_text_processing/text_normalization/ar/data/number/tens_nom.tsv similarity index 100% rename from nemo_text_processing/text_normalization/ar/data/number/tens.tsv rename to nemo_text_processing/text_normalization/ar/data/number/tens_nom.tsv diff --git a/nemo_text_processing/text_normalization/ar/data/number/zero.tsv b/nemo_text_processing/text_normalization/ar/data/number/zero.tsv index 70f51fb93..84ac6b35e 100644 --- a/nemo_text_processing/text_normalization/ar/data/number/zero.tsv +++ b/nemo_text_processing/text_normalization/ar/data/number/zero.tsv @@ -1 +1 @@ -0 صفر \ No newline at end of file +0 صفر diff --git a/nemo_text_processing/text_normalization/ar/taggers/cardinal.py b/nemo_text_processing/text_normalization/ar/taggers/cardinal.py index 1ac137e78..bf81f66ab 100644 --- a/nemo_text_processing/text_normalization/ar/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/ar/taggers/cardinal.py @@ -21,7 +21,7 @@ class CardinalFst(GraphFst): """ Finite state transducer for classifying cardinals, e.g. - "9837" -> cardinal { integer: "تسعة اَلاف وثمان مئة وسبعة وثلاثون" } + "9837" -> cardinal { integer: "تسعة آلاف وثمان مئة وسبعة وثلاثين" } Args: deterministic: if True will provide a single transduction option, @@ -35,24 +35,32 @@ def __init__(self): graph_zero = pynini.string_file(get_abs_path("data/number/zero.tsv")) # cardinals data files - graph_digit = pynini.string_file(get_abs_path("data/number/digit.tsv")) - digit_100 = pynini.string_file(get_abs_path("data/number/digit_100.tsv")) - digit_1000 = pynini.string_file(get_abs_path("data/number/digit_1000.tsv")) - teens = pynini.string_file(get_abs_path("data/number/teens.tsv")) - tens = pynini.string_file(get_abs_path("data/number/tens.tsv")) + graph_digit = pynini.string_file(get_abs_path("data/number/digit.tsv")).optimize() + digit_100 = pynini.string_file(get_abs_path("data/number/digit_100.tsv")).optimize() + digit_1000 = pynini.string_file(get_abs_path("data/number/digit_1000.tsv")).optimize() + teens = pynini.string_file(get_abs_path("data/number/teens.tsv")).optimize() + tens_nom = pynutil.add_weight( + pynini.string_file(get_abs_path("data/number/tens_nom.tsv")), weight=0.001 + ).optimize() + tens_gen = pynini.string_file(get_abs_path("data/number/tens_gen.tsv")).optimize() # Grammar for cardinals 10_20_30 etc - tens_zero = tens + pynutil.delete("0") + # add weight to prefer genetive case over nominative + tens_zero_nom = tens_nom + pynutil.delete("0") + tens_zero_nom = pynutil.add_weight(tens_zero_nom, weight=0.001) + tens_zero_gen = tens_gen + pynutil.delete("0") # Creating flops for two digit cardinals 34->43 reverse_digits = pynini.string_file(get_abs_path("data/number/flops.tsv")) # Grammar for two digitcardinals graph_flops = flop_digits @ reverse_digits - graph_tens_plus1 = graph_digit + insert_space + insert_and + tens - graph_tens_plus = graph_flops @ graph_tens_plus1 - graph_all = graph_digit | teens | tens_zero | graph_tens_plus - graph_two_digits = teens | tens_zero | graph_tens_plus + # 34-- أربعة وثلاثون + graph_tens_plus = graph_digit + insert_space + insert_and + (tens_nom | tens_gen) + # flop + graph_tens_plus_flop = graph_flops @ graph_tens_plus + graph_all = graph_digit | teens | tens_zero_nom | graph_tens_plus_flop | tens_zero_gen + graph_two_digits = teens | tens_zero_nom | graph_tens_plus_flop | tens_zero_gen # Grammar for cardinals hundreds one_hundred = pynini.cross("1", "مئة") @@ -67,7 +75,7 @@ def __init__(self): + insert_and + graph_digit ) - two_hundreds = pynini.cross("2", "مئتان") + two_hundreds = pynini.cross("2", "مئتين") graph_one_hundred = one_hundred + pynutil.delete("00", weight=0.001) graph_one_hundred_plus = ( one_hundred + insert_space + insert_and + graph_two_digits @@ -83,61 +91,92 @@ def __init__(self): graph_all_hundreds = graph_all_one_hundred | graph_two_hundreds | hundreds_zero | hundreds_plus - # Grammar for thousands - one_thousand = pynini.cross("1", "ألف") - thousands_zero = digit_1000 + insert_space + pynutil.insert("اَلاف") + pynutil.delete("000", weight=0.001) - thousands_plus = ( - digit_1000 + insert_space + pynutil.insert("اَلاف") + insert_space + insert_and + graph_all_hundreds + # ---- counted-noun (تمييز) agreement for a 3-digit multiplier count (100-999) ---- + # The thousand/million word agrees with the *trailing* element of the count: + # trailing 3-10 -> plural (آلاف / ملايين), otherwise -> singular (ألف / مليون). + # e.g. 110 -> "مئة وعشرة آلاف", but 123 -> "مئة وثلاثة وعشرين ألف". + _h = pynini.union("1", "2", "3", "4", "5", "6", "7", "8", "9") + _digit_any = pynini.union("0", "1", "2", "3", "4", "5", "6", "7", "8", "9") + # trailing value 3..10 + _trailing_plural = pynini.accep("0") + pynini.union("3", "4", "5", "6", "7", "8", "9") | pynini.accep("10") + # trailing value 0,1,2 or 11..99 + _trailing_singular = ( + pynini.accep("0") + pynini.union("0", "1", "2") + | pynini.accep("1") + pynini.union("1", "2", "3", "4", "5", "6", "7", "8", "9") + | pynini.union("2", "3", "4", "5", "6", "7", "8", "9") + _digit_any ) - thousands_skip_hundreds = ( - digit_1000 - + insert_space - + pynutil.insert("اَلاف") - + pynutil.delete("0") - + insert_space - + insert_and - + graph_two_digits - | digit_1000 - + insert_space - + pynutil.insert("اَلاف") - + pynutil.delete("00") - + insert_space - + insert_and - + graph_digit + hundreds_count_plural = ((_h + _trailing_plural) @ graph_all_hundreds).optimize() + hundreds_count_singular = ((_h + _trailing_singular) @ graph_all_hundreds).optimize() + + # ---- reusable building blocks (values 1-999) ---- + # a full 3-digit period (001-999) with internal leading zeros removed + period_nonzero = ( + pynutil.delete("00") + graph_digit + | pynutil.delete("0") + graph_two_digits + | graph_all_hundreds ) - two_thousands = pynini.cross("2", "ألفان") - graph_one_thousand = one_thousand + pynutil.delete("000", weight=0.001) - graph_one_thousand_plus = ( - one_thousand + insert_space + insert_and + graph_all_hundreds - | one_thousand + pynutil.delete("0") + insert_space + insert_and + graph_two_digits - | one_thousand + pynutil.delete("00") + insert_space + insert_and + graph_digit + # trailing 3-digit remainder: either all zeros (nothing) or " و" + units_remainder = pynutil.delete("000") | (insert_space + insert_and + period_nonzero) + + # ---- thousands: 1_000 .. 999_999 ---- + # maps the thousand-count to " " with correct agreement: + # 1 -> ألف, 2 -> ألفين, 3-10 -> آلاف, 11-999 -> ألف + thousand_group = ( + pynini.cross("1", "ألف") + | pynini.cross("2", "ألفين") + | (digit_1000 + pynutil.insert(" آلاف")) + | pynini.cross("10", "عشرة آلاف") + | pynini.cross("200", "مئتي ألف") # dual construct-state (drops nun before counted noun) + | pynutil.add_weight(graph_two_digits + pynutil.insert(" ألف"), 0.01) + | pynutil.add_weight(hundreds_count_plural + pynutil.insert(" آلاف"), 0.02) + | pynutil.add_weight(hundreds_count_singular + pynutil.insert(" ألف"), 0.02) ) - graph_two_thousands = two_thousands + pynutil.delete("000", weight=0.001) - graph_two_thousands_plus = ( - two_thousands + insert_space + insert_and + graph_all_hundreds - | two_thousands + pynutil.delete("0") + insert_space + insert_and + graph_two_digits - | two_thousands + pynutil.delete("00") + insert_space + insert_and + graph_digit + graph_thousands = thousand_group + units_remainder + + # ---- millions: 1_000_000 .. 999_999_999 ---- + # same agreement pattern as thousands (مليون / مليونين / ملايين / مليون) + million_group = ( + pynini.cross("1", "مليون") + | pynini.cross("2", "مليونين") + | (digit_1000 + pynutil.insert(" ملايين")) + | pynini.cross("10", "عشرة ملايين") + | pynini.cross("200", "مئتي مليون") # dual construct-state (drops nun before counted noun) + | pynutil.add_weight(graph_two_digits + pynutil.insert(" مليون"), 0.01) + | pynutil.add_weight(hundreds_count_plural + pynutil.insert(" ملايين"), 0.02) + | pynutil.add_weight(hundreds_count_singular + pynutil.insert(" مليون"), 0.02) ) - - graph_all_one_thousand = graph_one_thousand | graph_one_thousand_plus - graph_all_two_thousands = graph_two_thousands | graph_two_thousands_plus - - graph_all_thousands = ( - graph_all_one_thousand - | graph_two_thousands - | thousands_zero - | thousands_plus - | thousands_skip_hundreds - | graph_all_two_thousands + # zero-padded thousand-count (001-999) used inside a 6-digit remainder block + thousand_group_padded = ( + pynini.cross("001", "ألف") + | pynini.cross("002", "ألفين") + | (pynutil.delete("00") + digit_1000 + pynutil.insert(" آلاف")) + | pynini.cross("010", "عشرة آلاف") + | pynini.cross("200", "مئتي ألف") # dual construct-state (drops nun before counted noun) + | pynutil.add_weight(pynutil.delete("0") + graph_two_digits + pynutil.insert(" ألف"), 0.01) + | pynutil.add_weight(hundreds_count_plural + pynutil.insert(" آلاف"), 0.02) + | pynutil.add_weight(hundreds_count_singular + pynutil.insert(" ألف"), 0.02) + ) + # 6-digit remainder after the millions group (000001 .. 999999) + block6_nonzero = ( + pynutil.delete("000") + period_nonzero + | thousand_group_padded + units_remainder + ) + million_remainder = pynutil.delete("000000") | (insert_space + insert_and + block6_nonzero) + graph_millions = million_group + million_remainder + + self.graph = ( + graph_zero + | graph_all + | graph_all_hundreds + | graph_thousands + | graph_millions ) - - graph = graph_all | graph_all_hundreds | graph_all_thousands | graph_zero - - self.cardinal_numbers = (graph).optimize() # remove leading zeros leading_zeros = pynini.closure(pynini.cross("0", "")) - self.cardinal_numbers_with_leading_zeros = (leading_zeros + self.cardinal_numbers).optimize() + self.cardinal_numbers_with_leading_zeros = (leading_zeros + self.graph).optimize() + + self.cardinal_numbers = (self.graph | self.cardinal_numbers_with_leading_zeros).optimize() self.optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1) diff --git a/nemo_text_processing/text_normalization/ar/verbalizers/fraction.py b/nemo_text_processing/text_normalization/ar/verbalizers/fraction.py index 1b3330cec..7d68fb8bc 100644 --- a/nemo_text_processing/text_normalization/ar/verbalizers/fraction.py +++ b/nemo_text_processing/text_normalization/ar/verbalizers/fraction.py @@ -32,7 +32,7 @@ def __init__(self): # create unions for special cases denominator_singular = pynini.union("نصف", "ثلث", "ربع", "خمس", "سدس", "سبع", "ثمن", "تسع", "عشر") denominator_dual = pynini.union( - "نصفين", "ثلثين", "ربعين", "خمسين", "سدسين", "سبعين", "ثمنين", "تسعين", "عشرين" + "نصفي", "ثلثي", "ربعي", "خمسي", "سدسي", "سبعي", "ثمني", "تسعي", "عشري" ) denominator_plural = pynini.union("أخماس", "أرباع", "أثلاث", "أسداس", "أسباع", "أثمان", "أتساع", "أعشار") numerator_three_to_ten = pynini.union("خمسة", "سبعة", "عشرة", "ثلاثة", "أربعة", "ستة", "ثمانية", "تسعة") @@ -63,7 +63,13 @@ def __init__(self): # verbalize from integer and one over half --> integer and half e.g واحد ونصف numerator_one = pynutil.delete("numerator: \"واحد\"") + pynutil.delete(" ") + graph_denominator_singular # verbalize from integer and two over half(dual) --> integer and half(dual) e.g. واحد وثلثين - numerator_two = pynutil.delete("numerator: \"اثنان\"") + pynutil.delete(" ") + graph_denominator_dual + numerator_two = ( + pynutil.delete("numerator: \"") + + pynutil.delete(pynini.union("اثنان", "اثنين")) + + pynutil.delete("\"") + + pynutil.delete(" ") + + graph_denominator_dual + ) # verbalize from integer and three over thirds(plural) --> integer and three thirds(plural) e.g. واحد وثلاثة أرباع numerator_three_to_ten = ( pynutil.delete("numerator: \"") diff --git a/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_cardinal.txt index 893e3f44a..7a5a3b3f8 100644 --- a/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_cardinal.txt +++ b/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_cardinal.txt @@ -1,34 +1,59 @@ -ألفان وتسعة~2009 -ألف~1000 -ثلاثة اَلاف وستة~3006 -ألف وسبعة وسبعون~1077 -تسعة اَلاف وثمان مئة وسبعة وثلاثون~9837 -ثلاثة اَلاف وتسع مئة وسبعة عشر~3917 -ألفان~2000 -ثلاثة اَلاف وستة~3006 -ألف وسبعة وسبعون~1077 -تسعة اَلاف وثمان مئة وسبعة وثلاثون~9837 -ثلاثة اَلاف وتسع مئة وسبعة عشر~3917 -خمسة اَلاف وخمس مئة وعشرون~5520 -ثمانية اَلاف وسبع مئة وواحد وعشرون~8721 -ثمانية اَلاف وتسع مئة~8900 -ثمان مئة وتسعة~809 -مئة وسبعة~107 -ثمان مئة وسبعة وثلاثون~837 -تسع مئة وسبعة عشر~917 -خمس مئة وعشرون~520 -اثنان~2 -ثلاثة وثلاثون~33 -ستون~60 -ثمانية عشر~18 -ستة عشر~16 -اثنا عشر~12 +صفر~0 واحد~1 -عشرون~20 -واحد وعشرون~21 -مئة~100 -أربعة وثلاثون~34 -ثمانية وسبعون~78 -تسعة وتسعون~99 +اثنين~2 +ألفين وتسعة~2009 +ألفين~2000 عشرة~10 -سبعون~70 \ No newline at end of file +اثنا عشر~12 +ستة عشر~16 +ثمانية عشر~18 +عشرين~20 +واحد وعشرين~21 +اثنين وعشرين~22 +ثلاثة وثلاثين~33 +أربعة وثلاثين~34 +خمسين~50 +ستين~60 +سبعين~70 +ثمانية وسبعين~78 +تسعين~90 +تسعة وتسعين~99 +مئة~100 +مئة وسبعة~107 +مئتين واثنين~202 +خمس مئة وعشرين~520 +ثمان مئة وتسعة~809 +ثمان مئة وسبعة وثلاثين~837 +تسع مئة وسبعة عشر~917 +ألف وسبعة وسبعين~1077 +ثلاثة آلاف وستة~3006 +ثلاثة آلاف وتسع مئة وسبعة عشر~3917 +خمسة آلاف وخمس مئة وعشرين~5520 +ثمانية آلاف وسبع مئة وواحد وعشرين~8721 +ثمانية آلاف وتسع مئة~8900 +تسعة آلاف وثمان مئة وسبعة وثلاثين~9837 +عشرة آلاف~10000 +احد عشر ألف~11000 +عشرين ألف~20000 +واحد وعشرين ألف~21000 +خمسة وعشرين ألف~25000 +تسعة وتسعين ألف وتسع مئة وتسعة وتسعين~99999 +مئة ألف~100000 +مئة وثلاثة آلاف~103000 +مئة وخمسة آلاف~105000 +مئة وعشرة آلاف~110000 +مئة وعشرين ألف~120000 +مئة وثلاثة وعشرين ألف وأربع مئة وستة وخمسين~123456 +مئتي ألف~200000 +مئتين وعشرة آلاف~210000 +مئتين وخمسين ألف~250000 +ثلاث مئة وخمسة آلاف~305000 +خمس مئة ألف~500000 +سبع مئة وخمسين ألف~750000 +تسع مئة وتسعة وتسعين ألف وتسع مئة وتسعة وتسعين~999999 +مليون~1000000 +مليون وخمس مئة ألف~1500000 +مليونين~2000000 +مليونين ومئتي ألف~2200000 +اثنا عشر مليون وثلاث مئة وخمسة وأربعين ألف وست مئة وثمانية وسبعين~12345678 +مئة وعشرة ملايين~110000000 diff --git a/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_decimal.txt b/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_decimal.txt index 2666dd952..b6d4d6923 100644 --- a/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_decimal.txt +++ b/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_decimal.txt @@ -1,30 +1,30 @@ -ستة وثمانية وثمانون من مئة~6.88 -سبعة وتسعة وتسعون من مئة~7.99 -أربعة وتسعة اَلاف وثمان مئة وستة وسبعون من عشرة آلاف~4.9876 -ستة وثمانية اَلاف وتسع مئة وأربعة وأربعون من عشرة آلاف~6.8944 -ستة اَلاف وسبع مئة وثمانية وتسعون وتسعة من عشرة~6798.9 -ست مئة وثمانية وخمسون وثمان مئة وثمانية وثمانون من ألف~658.888 -اثنان وثمانية من عشرة~2.8 +ستة وثمانية وثمانين من مئة~6.88 +سبعة وتسعة وتسعين من مئة~7.99 +أربعة وتسعة آلاف وثمان مئة وستة وسبعين من عشرة آلاف~4.9876 +ستة وثمانية آلاف وتسع مئة وأربعة وأربعين من عشرة آلاف~6.8944 +ستة آلاف وسبع مئة وثمانية وتسعين وتسعة من عشرة~6798.9 +ست مئة وثمانية وخمسين وثمان مئة وثمانية وثمانين من ألف~658.888 +اثنين وثمانية من عشرة~2.8 سبعة وتسعة من عشرة~7.9 -أربعون وتسعة اَلاف وثمان مئة وستة وسبعون من عشرة آلاف~40.9876 -مئتان وواحد وسبعة من عشرة~201.7 -مئة وواحد وخمسون من مئة~101.50 -ثلاث مئة وأربعون من مئة~300.40 -سبعة وسبعون وثمانية اَلاف وتسع مئة وأربعة وأربعون من عشرة آلاف~77.8944 -تسعة اَلاف وثمان مئة وستة وسبعون وتسع مئة من ألف~9876.900 -ألفان وسبعة وثمان مئة وثمانية وثمانون من ألف~2007.888 -سبعون وتسع مئة وتسعون من ألف~70.990 -واحد وعشرون وثمان مئة واثنان وثلاثون من ألف~21.832 +أربعين وتسعة آلاف وثمان مئة وستة وسبعين من عشرة آلاف~40.9876 +مئتين وواحد وسبعة من عشرة~201.7 +مئة وواحد وخمسين من مئة~101.50 +ثلاث مئة وأربعين من مئة~300.40 +سبعة وسبعين وثمانية آلاف وتسع مئة وأربعة وأربعين من عشرة آلاف~77.8944 +تسعة آلاف وثمان مئة وستة وسبعين وتسع مئة من ألف~9876.900 +ألفين وسبعة وثمان مئة وثمانية وثمانين من ألف~2007.888 +سبعين وتسع مئة وتسعين من ألف~70.990 +واحد وعشرين وثمان مئة واثنين وثلاثين من ألف~21.832 سبعة عشر وسبعة عشر من مئة~17.17 -ستة وثمانون واثنا عشر من مئة~86.12 -عشرة ومئتان من ألف~10.200 -سبع مئة وتسعة وستون وثمان مئة من ألف~769.800 -خمسة اَلاف وست مئة وتسعة وسبعون وثمانية اَلاف وتسع مئة وثلاثة من عشرة آلاف~5679.8903 -ستة اَلاف وتسعون من مئة~6000.90 +ستة وثمانين واثنا عشر من مئة~86.12 +عشرة ومئتين من ألف~10.200 +سبع مئة وتسعة وستين وثمان مئة من ألف~769.800 +خمسة آلاف وست مئة وتسعة وسبعين وثمانية آلاف وتسع مئة وثلاثة من عشرة آلاف~5679.8903 +ستة آلاف وتسعين من مئة~6000.90 واحد وثمانية عشر من مئة~1.18 -اثنان واثنان من عشرة~2.2 +اثنين واثنين من عشرة~2.2 ثلاثة وخمسة من عشرة~3.5 خمسة وأربعة من عشرة~5.4 -اثنان وثمانون وثمان مئة من ألف~82.800 -مئتان وسبعة ومئتان من ألف~207.200 +اثنين وثمانين وثمان مئة من ألف~82.800 +مئتين وسبعة ومئتين من ألف~207.200 عشرة وواحد من عشرة~10.1 diff --git a/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_fraction.txt b/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_fraction.txt index 82476ff54..a95db6f31 100644 --- a/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_fraction.txt +++ b/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_fraction.txt @@ -1,20 +1,20 @@ واحد و نصف~1 1/2 سبعة و عشرة أخماس~7 10/5 -خمسة و ثلثين~5 2/3 +خمسة و ثلثي~5 2/3 عشرة و نصف~10 1/2 -ثلاثون و تسعة أعشار~30 9/10 -سبعة وسبعون و ثلاثة أسداس~77 3/6 +ثلاثين و تسعة أعشار~30 9/10 +سبعة وسبعين و ثلاثة أسداس~77 3/6 ستة عشر و سبعة عشر على ثلاثة~16 17/3 -مئتان و واحد على ثلاث مئة وسبعة وثمانون~200 1/387 -عشرون و ثلث~20 1/3 +مئتين و واحد على ثلاث مئة وسبعة وثمانين~200 1/387 +عشرين و ثلث~20 1/3 ربع~1/4 نصف~1/2 ثلث~1/3 تسعة أثلاث~9/3 -ستة و ثلثين~6 2/3 -ثلثين~2/3 +ستة و ثلثي~6 2/3 +ثلثي~2/3 خمس~1/5 سدس~1/6 سبع~1/7 ثمن~1/8 -سبعين~2/7 +سبعي~2/7 diff --git a/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_measure.txt index bb8e3a15a..b5a68ca4c 100644 --- a/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_measure.txt +++ b/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_measure.txt @@ -2,14 +2,14 @@ تسعة فهرنهايت~9f اثنا عشر في المائة~12% تسعة في المائة~9% -عشرون في المائة~20% -ستة وثمانية وثمانون من مئة في المائة~6.88% -سبعة وتسعة وتسعون من مئة في المائة~7.99% -أربعة وتسعة اَلاف وثمان مئة وستة وسبعون من عشرة آلاف في المائة~4.9876% -ستة وثمانية اَلاف وتسع مئة وأربعة وأربعون من عشرة آلاف في المائة~6.8944% -ستة اَلاف وسبع مئة وثمانية وتسعون وتسعة من عشرة في المائة~6798.9% -ست مئة وثمانية وخمسون وثمان مئة وثمانية وثمانون من ألف في المائة~658.888% -اثنان وثمانية من عشرة في المائة~2.8% +عشرين في المائة~20% +ستة وثمانية وثمانين من مئة في المائة~6.88% +سبعة وتسعة وتسعين من مئة في المائة~7.99% +أربعة وتسعة آلاف وثمان مئة وستة وسبعين من عشرة آلاف في المائة~4.9876% +ستة وثمانية آلاف وتسع مئة وأربعة وأربعين من عشرة آلاف في المائة~6.8944% +ستة آلاف وسبع مئة وثمانية وتسعين وتسعة من عشرة في المائة~6798.9% +ست مئة وثمانية وخمسين وثمان مئة وثمانية وثمانين من ألف في المائة~658.888% +اثنين وثمانية من عشرة في المائة~2.8% سبعة وتسعة من عشرة في المائة~7.9% واحد في المائة~1% -ألفان وتسعة في المائة~2009% +ألفين وتسعة في المائة~2009% diff --git a/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_money.txt index 25ee8b5d6..bbf5e894e 100644 --- a/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_money.txt +++ b/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_money.txt @@ -1,21 +1,21 @@ سبعة دولار~$7 -سبعة دولار وخمسون سنت~$7.50 +سبعة دولار وخمسين سنت~$7.50 سبعة دولار وخمسة سنتات~$7.05 -سبعة دولار وتسعة وتسعون سنت~$7.99 +سبعة دولار وتسعة وتسعين سنت~$7.99 سبعة دولار وعشرة سنتات~$7.10 -تسعة وتسعون سنت~$0.99 -خمسة وتسعون سنت~$0.95 +تسعة وتسعين سنت~$0.99 +خمسة وتسعين سنت~$0.95 تسعة يورو~€9 تسعة دولار وتسعة سنتات~$9.09 -خمسة دينار كويتي~د.ك5 -تسعة دينار كويتي~د.ك9 -خمسة ريال سعودي~ر.س5 -سبعون دولار وستون سنت~$70.60 +د.ك5~د.ك5 +د.ك9~د.ك9 +ر.س5~ر.س5 +سبعين دولار وستين سنت~$70.60 اثنا عشر دولار~$12 تسعة دولار~$9 -عشرون دولار~$20 -عشرون دولار وخمسون سنت~$20.50 -عشرون دولار وعشرة سنتات~$20.10 +عشرين دولار~$20 +عشرين دولار وخمسين سنت~$20.50 +عشرين دولار وعشرة سنتات~$20.10 عشرة سنتات~$0.10 ثلاثة سنتات~$0.03 -عشرون سنت~$0.20 +عشرين سنت~$0.20 diff --git a/tests/nemo_text_processing/ar/data_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/ar/data_text_normalization/test_cases_cardinal.txt index 248c2033c..0c7e05c6a 100644 --- a/tests/nemo_text_processing/ar/data_text_normalization/test_cases_cardinal.txt +++ b/tests/nemo_text_processing/ar/data_text_normalization/test_cases_cardinal.txt @@ -1,27 +1,59 @@ -3006~ثلاثة اَلاف وستة -1077~ألف وسبعة وسبعون -9837~تسعة اَلاف وثمان مئة وسبعة وثلاثون -3917~ثلاثة اَلاف وتسع مئة وسبعة عشر -5520~خمسة اَلاف وخمس مئة وعشرون -8721~ثمانية اَلاف وسبع مئة وواحد وعشرون -8900~ثمانية اَلاف وتسع مئة -809~ثمان مئة وتسعة -107~مئة وسبعة -837~ثمان مئة وسبعة وثلاثون -917~تسع مئة وسبعة عشر -520~خمس مئة وعشرون -2~اثنان -33~ثلاثة وثلاثون -60~ستون -18~ثمانية عشر -16~ستة عشر -12~اثنا عشر +0~صفر 1~واحد -20~عشرون -21~واحد وعشرون -100~مئة -34~أربعة وثلاثون -78~ثمانية وسبعون -99~تسعة وتسعون +2~اثنين +2009~ألفين وتسعة +2000~ألفين 10~عشرة -70~سبعون \ No newline at end of file +12~اثنا عشر +16~ستة عشر +18~ثمانية عشر +20~عشرين +21~واحد وعشرين +22~اثنين وعشرين +33~ثلاثة وثلاثين +34~أربعة وثلاثين +50~خمسين +60~ستين +70~سبعين +78~ثمانية وسبعين +90~تسعين +99~تسعة وتسعين +100~مئة +107~مئة وسبعة +202~مئتين واثنين +520~خمس مئة وعشرين +809~ثمان مئة وتسعة +837~ثمان مئة وسبعة وثلاثين +917~تسع مئة وسبعة عشر +1077~ألف وسبعة وسبعين +3006~ثلاثة آلاف وستة +3917~ثلاثة آلاف وتسع مئة وسبعة عشر +5520~خمسة آلاف وخمس مئة وعشرين +8721~ثمانية آلاف وسبع مئة وواحد وعشرين +8900~ثمانية آلاف وتسع مئة +9837~تسعة آلاف وثمان مئة وسبعة وثلاثين +10000~عشرة آلاف +11000~احد عشر ألف +20000~عشرين ألف +21000~واحد وعشرين ألف +25000~خمسة وعشرين ألف +99999~تسعة وتسعين ألف وتسع مئة وتسعة وتسعين +100000~مئة ألف +103000~مئة وثلاثة آلاف +105000~مئة وخمسة آلاف +110000~مئة وعشرة آلاف +120000~مئة وعشرين ألف +123456~مئة وثلاثة وعشرين ألف وأربع مئة وستة وخمسين +200000~مئتي ألف +210000~مئتين وعشرة آلاف +250000~مئتين وخمسين ألف +305000~ثلاث مئة وخمسة آلاف +500000~خمس مئة ألف +750000~سبع مئة وخمسين ألف +999999~تسع مئة وتسعة وتسعين ألف وتسع مئة وتسعة وتسعين +1000000~مليون +1500000~مليون وخمس مئة ألف +2000000~مليونين +2200000~مليونين ومئتي ألف +12345678~اثنا عشر مليون وثلاث مئة وخمسة وأربعين ألف وست مئة وثمانية وسبعين +110000000~مئة وعشرة ملايين diff --git a/tests/nemo_text_processing/ar/data_text_normalization/test_cases_decimal.txt b/tests/nemo_text_processing/ar/data_text_normalization/test_cases_decimal.txt index 2c13484ca..ebda13bbb 100644 --- a/tests/nemo_text_processing/ar/data_text_normalization/test_cases_decimal.txt +++ b/tests/nemo_text_processing/ar/data_text_normalization/test_cases_decimal.txt @@ -1,30 +1,30 @@ -6.88~ستة وثمانية وثمانون من مئة -7,99~سبعة وتسعة وتسعون من مئة -4.9876~أربعة وتسعة اَلاف وثمان مئة وستة وسبعون من عشرة آلاف -6,8944~ستة وثمانية اَلاف وتسع مئة وأربعة وأربعون من عشرة آلاف -6798,9~ستة اَلاف وسبع مئة وثمانية وتسعون وتسعة من عشرة -658,888~ست مئة وثمانية وخمسون وثمان مئة وثمانية وثمانون من ألف -2.8~اثنان وثمانية من عشرة -7.9~سبعة وتسعة من عشرة -40.9876~أربعون وتسعة اَلاف وثمان مئة وستة وسبعون من عشرة آلاف -201.7~مئتان وواحد وسبعة من عشرة -101.50~مئة وواحد وخمسون من مئة -300.40~ثلاث مئة وأربعون من مئة -77,8944~سبعة وسبعون وثمانية اَلاف وتسع مئة وأربعة وأربعون من عشرة آلاف -9876,900~تسعة اَلاف وثمان مئة وستة وسبعون وتسع مئة من ألف -2007,888~ألفان وسبعة وثمان مئة وثمانية وثمانون من ألف -70,990~سبعون وتسع مئة وتسعون من ألف -21.832~واحد وعشرون وثمان مئة واثنان وثلاثون من ألف -17.17~سبعة عشر وسبعة عشر من مئة -86,12~ستة وثمانون واثنا عشر من مئة -10.200~عشرة ومئتان من ألف -769.800~سبع مئة وتسعة وستون وثمان مئة من ألف -5679,8903~خمسة اَلاف وست مئة وتسعة وسبعون وثمانية اَلاف وتسع مئة وثلاثة من عشرة آلاف -6000.90~ستة اَلاف وتسعون من مئة -1,18~واحد وثمانية عشر من مئة -2,2~اثنان واثنان من عشرة -3.5~ثلاثة وخمسة من عشرة -5.4~خمسة وأربعة من عشرة -82.800~اثنان وثمانون وثمان مئة من ألف -207.200~مئتان وسبعة ومئتان من ألف -10,1~عشرة وواحد من عشرة +6.88~ستة وثمانية وثمانين من مئة +7,99~سبعة وتسعة وتسعين من مئة +4.9876~أربعة وتسعة آلاف وثمان مئة وستة وسبعين من عشرة آلاف +6,8944~ستة وثمانية آلاف وتسع مئة وأربعة وأربعين من عشرة آلاف +6798,9~ستة آلاف وسبع مئة وثمانية وتسعين وتسعة من عشرة +658,888~ست مئة وثمانية وخمسين وثمان مئة وثمانية وثمانين من ألف +2.8~اثنين وثمانية من عشرة +7.9~سبعة وتسعة من عشرة +40.9876~أربعين وتسعة آلاف وثمان مئة وستة وسبعين من عشرة آلاف +201.7~مئتين وواحد وسبعة من عشرة +101.50~مئة وواحد وخمسين من مئة +300.40~ثلاث مئة وأربعين من مئة +77,8944~سبعة وسبعين وثمانية آلاف وتسع مئة وأربعة وأربعين من عشرة آلاف +9876,900~تسعة آلاف وثمان مئة وستة وسبعين وتسع مئة من ألف +2007,888~ألفين وسبعة وثمان مئة وثمانية وثمانين من ألف +70,990~سبعين وتسع مئة وتسعين من ألف +21.832~واحد وعشرين وثمان مئة واثنين وثلاثين من ألف +17.17~سبعة عشر وسبعة عشر من مئة +86,12~ستة وثمانين واثنا عشر من مئة +10.200~عشرة ومئتين من ألف +769.800~سبع مئة وتسعة وستين وثمان مئة من ألف +5679,8903~خمسة آلاف وست مئة وتسعة وسبعين وثمانية آلاف وتسع مئة وثلاثة من عشرة آلاف +6000.90~ستة آلاف وتسعين من مئة +1,18~واحد وثمانية عشر من مئة +2,2~اثنين واثنين من عشرة +3.5~ثلاثة وخمسة من عشرة +5.4~خمسة وأربعة من عشرة +82.800~اثنين وثمانين وثمان مئة من ألف +207.200~مئتين وسبعة ومئتين من ألف +10,1~عشرة وواحد من عشرة diff --git a/tests/nemo_text_processing/ar/data_text_normalization/test_cases_fraction.txt b/tests/nemo_text_processing/ar/data_text_normalization/test_cases_fraction.txt index cad1bc6a3..a95db6f31 100644 --- a/tests/nemo_text_processing/ar/data_text_normalization/test_cases_fraction.txt +++ b/tests/nemo_text_processing/ar/data_text_normalization/test_cases_fraction.txt @@ -1,20 +1,20 @@ واحد و نصف~1 1/2 سبعة و عشرة أخماس~7 10/5 -خمسة و ثلثين~5 2/3 +خمسة و ثلثي~5 2/3 عشرة و نصف~10 1/2 -ثلاثون و تسعة أعشار~30 9/10 -سبعة وسبعون و ثلاثة أسداس~77 3/6 +ثلاثين و تسعة أعشار~30 9/10 +سبعة وسبعين و ثلاثة أسداس~77 3/6 ستة عشر و سبعة عشر على ثلاثة~16 17/3 -مئتان و واحد على ثلاث مئة وسبعة وثمانون~200 1/387 -عشرون و ثلث~20 1/3 +مئتين و واحد على ثلاث مئة وسبعة وثمانين~200 1/387 +عشرين و ثلث~20 1/3 ربع~1/4 نصف~1/2 ثلث~1/3 تسعة أثلاث~9/3 -ستة و ثلثين~6 2/3 -ثلثين~2/3 +ستة و ثلثي~6 2/3 +ثلثي~2/3 خمس~1/5 سدس~1/6 سبع~1/7 ثمن~1/8 -سبعين~2/7 \ No newline at end of file +سبعي~2/7 diff --git a/tests/nemo_text_processing/ar/data_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/ar/data_text_normalization/test_cases_measure.txt index a382d7a9f..17b58ea64 100644 --- a/tests/nemo_text_processing/ar/data_text_normalization/test_cases_measure.txt +++ b/tests/nemo_text_processing/ar/data_text_normalization/test_cases_measure.txt @@ -2,13 +2,13 @@ تسعة فهرنهايت~f9 اثنا عشر في المائة~12% تسعة في المائة~09% -عشرون في المائة~20% -ستة وثمانية وثمانون من مئة في المائة~%6.88 -سبعة وتسعة وتسعون من مئة في المائة~%7,99 -أربعة وتسعة اَلاف وثمان مئة وستة وسبعون من عشرة آلاف في المائة~%4.9876 -ستة وثمانية اَلاف وتسع مئة وأربعة وأربعون من عشرة آلاف في المائة~%6,8944 -ستة اَلاف وسبع مئة وثمانية وتسعون وتسعة من عشرة في المائة~%6798,9 -ست مئة وثمانية وخمسون وثمان مئة وثمانية وثمانون من ألف في المائة~%658,888 -اثنان وثمانية من عشرة في المائة~2.8% +عشرين في المائة~20% +ستة وثمانية وثمانين من مئة في المائة~%6.88 +سبعة وتسعة وتسعين من مئة في المائة~%7,99 +أربعة وتسعة آلاف وثمان مئة وستة وسبعين من عشرة آلاف في المائة~%4.9876 +ستة وثمانية آلاف وتسع مئة وأربعة وأربعين من عشرة آلاف في المائة~%6,8944 +ستة آلاف وسبع مئة وثمانية وتسعين وتسعة من عشرة في المائة~%6798,9 +ست مئة وثمانية وخمسين وثمان مئة وثمانية وثمانين من ألف في المائة~%658,888 +اثنين وثمانية من عشرة في المائة~2.8% سبعة وتسعة من عشرة في المائة~%7.9 واحد في المائة~1% diff --git a/tests/nemo_text_processing/ar/data_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/ar/data_text_normalization/test_cases_money.txt index 162e15799..24a7eee84 100644 --- a/tests/nemo_text_processing/ar/data_text_normalization/test_cases_money.txt +++ b/tests/nemo_text_processing/ar/data_text_normalization/test_cases_money.txt @@ -1,26 +1,26 @@ $7~سبعة دولار -$7.50~سبعة دولار وخمسون سنت +$7.50~سبعة دولار وخمسين سنت $7.05~سبعة دولار وخمسة سنتات -$7.99~سبعة دولار وتسعة وتسعون سنت +$7.99~سبعة دولار وتسعة وتسعين سنت $7.10~سبعة دولار وعشرة سنتات -$0.99~تسعة وتسعون سنت -€0.99~تسعة وتسعون سنت +$0.99~تسعة وتسعين سنت +€0.99~تسعة وتسعين سنت €9~تسعة يورو $9.09~تسعة دولار وتسعة سنتات 5د.ك~خمسة دينار كويتي 9د.ك~تسعة دينار كويتي 5ر.س~خمسة ريال سعودي -$70.60~سبعون دولار وستون سنت +$70.60~سبعين دولار وستين سنت $1.00~دولار واحد $1.00~دولار واحد $12~اثنا عشر دولار $09~تسعة دولار -$20~عشرون دولار -$20.50~عشرون دولار وخمسون سنت -$20.500~عشرون دولار وخمسون سنت -$20.5~عشرون دولار وخمسون سنت -$20.1~عشرون دولار وعشرة سنتات -$20.10~عشرون دولار وعشرة سنتات +$20~عشرين دولار +$20.50~عشرين دولار وخمسين سنت +$20.500~عشرين دولار وخمسين سنت +$20.5~عشرين دولار وخمسين سنت +$20.1~عشرين دولار وعشرة سنتات +$20.10~عشرين دولار وعشرة سنتات $0.10~عشرة سنتات $0.03~ثلاثة سنتات -$0.20~عشرون سنت \ No newline at end of file +$0.20~عشرين سنت diff --git a/tests/nemo_text_processing/ar/test_measure.py b/tests/nemo_text_processing/ar/test_measure.py index 79644f80d..9d47d0edb 100644 --- a/tests/nemo_text_processing/ar/test_measure.py +++ b/tests/nemo_text_processing/ar/test_measure.py @@ -16,7 +16,7 @@ import pytest from parameterized import parameterized -from tests.nemo_text_processing.utils import CACHE_DIR, parse_test_case_file +from tests.nemo_text_processing.utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file try: from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer From 65395bad9e31aee4707f94af959907fea4c1545b Mon Sep 17 00:00:00 2001 From: Enas Albasiri Date: Thu, 2 Jul 2026 21:42:47 +0000 Subject: [PATCH 10/12] feat(ar): add time normalization and inverse normalization (TN + ITN) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TN (text_normalization/ar): - Add time tagger + verbalizer for HH:MM, HH:MM:SS, and H inputs - Formal/regular style: hour as feminine ordinal (24h; midnight 0 -> الثانية عشرة), minutes/seconds as دقيقة/دقائق - Counted-noun agreement: * teens via 13_19 data (11 -> إحدى عشرة, 15 -> خمس عشرة) * 3-10 plural (خمس دقائق), 1 -> دقيقة واحدة, 2 -> دقيقتان * 21-59 ones digit feminized by polarity (خمس/ثمان/إحدى/اثنتان ...) * minutes/seconds number is nominative (عشرون/ثلاثون/أربعون/خمسون) - Constrain suffix verbalization to suffix.tsv values (required for a clean inverse) - Fix suffix.tsv tab separators - Wire TimeFst into tokenize_and_classify.py and verbalize.py - Add data: time/{suffix,time_zone}.tsv, ordinal/ordinals.tsv, number/{3_10mas,13_19}.tsv ITN (inverse_text_normalization/ar): - Add time tagger (inverts the TN time verbalizer) + verbalizer (-> HH:MM[:SS]) - Wire into tokenize_and_classify.py and verbalize.py Tests: - test_time.py: 30 TN (test_norm) + 19 ITN (test_denorm) hand-written cases - Full ar suite: 355 passed --- .../ar/taggers/time.py | 41 ++++ .../ar/taggers/tokenize_and_classify.py | 6 + .../ar/verbalizers/time.py | 57 ++++++ .../ar/verbalizers/verbalize.py | 5 +- .../ar/data/number/13_19.tsv | 9 + .../ar/data/number/3_10mas.tsv | 8 + .../ar/data/ordinal/ordinals.tsv | 24 +++ .../ar/data/time/suffix.tsv | 10 + .../ar/data/time/time_zone.tsv | 14 ++ .../text_normalization/ar/taggers/time.py | 105 ++++++++++ .../ar/taggers/tokenize_and_classify.py | 4 + .../text_normalization/ar/verbalizers/time.py | 192 ++++++++++++++++++ .../ar/verbalizers/verbalize.py | 5 +- .../test_cases_time.txt | 19 ++ .../test_cases_time.txt | 30 +++ tests/nemo_text_processing/ar/test_time.py | 40 ++++ 16 files changed, 567 insertions(+), 2 deletions(-) create mode 100644 nemo_text_processing/inverse_text_normalization/ar/taggers/time.py create mode 100644 nemo_text_processing/inverse_text_normalization/ar/verbalizers/time.py create mode 100644 nemo_text_processing/text_normalization/ar/data/number/13_19.tsv create mode 100644 nemo_text_processing/text_normalization/ar/data/number/3_10mas.tsv create mode 100644 nemo_text_processing/text_normalization/ar/data/ordinal/ordinals.tsv create mode 100644 nemo_text_processing/text_normalization/ar/data/time/suffix.tsv create mode 100644 nemo_text_processing/text_normalization/ar/data/time/time_zone.tsv create mode 100644 nemo_text_processing/text_normalization/ar/taggers/time.py create mode 100644 nemo_text_processing/text_normalization/ar/verbalizers/time.py create mode 100644 tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_time.txt create mode 100644 tests/nemo_text_processing/ar/data_text_normalization/test_cases_time.txt create mode 100644 tests/nemo_text_processing/ar/test_time.py diff --git a/nemo_text_processing/inverse_text_normalization/ar/taggers/time.py b/nemo_text_processing/inverse_text_normalization/ar/taggers/time.py new file mode 100644 index 000000000..962942e10 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ar/taggers/time.py @@ -0,0 +1,41 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from nemo_text_processing.text_normalization.ar.graph_utils import NEMO_SIGMA, GraphFst +from pynini.lib import pynutil + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying spoken Arabic time by inverting the + text-normalization time verbalizer, e.g. + الثالثة وخمس عشرة دقيقة -> time { hours: "3" minutes: "15" } + الخامسة وعشر دقائق وثانيتان -> time { hours: "5" minutes: "10" seconds: "2" } + التاسعة صباحًا -> time { hours: "9" suffix: "صباحًا" } + + Args: + tn_time_verbalizer: TN time verbalizer, whose .graph maps the tagged fields + (hours/minutes/seconds/suffix/zone) to their spoken Arabic form. + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, tn_time_verbalizer: GraphFst, deterministic: bool = True): + super().__init__(name="time", kind="classify", deterministic=deterministic) + # allow the spoken form to be matched with flexible spacing + optional_delete_space = pynini.closure(NEMO_SIGMA | pynutil.delete(" ", weight=0.0001)) + graph = (tn_time_verbalizer.graph @ optional_delete_space).invert().optimize() + self.fst = self.add_tokens(graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ar/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/ar/taggers/tokenize_and_classify.py index 8f6dd0782..285bfcef7 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/inverse_text_normalization/ar/taggers/tokenize_and_classify.py @@ -22,6 +22,7 @@ from nemo_text_processing.inverse_text_normalization.ar.taggers.measure import MeasureFst from nemo_text_processing.inverse_text_normalization.ar.taggers.money import MoneyFst from nemo_text_processing.inverse_text_normalization.ar.taggers.punctuation import PunctuationFst +from nemo_text_processing.inverse_text_normalization.ar.taggers.time import TimeFst from nemo_text_processing.inverse_text_normalization.ar.taggers.word import WordFst from nemo_text_processing.text_normalization.ar.graph_utils import ( GraphFst, @@ -30,6 +31,7 @@ generator_main, ) from nemo_text_processing.text_normalization.ar.taggers.tokenize_and_classify import ClassifyFst as TNClassifyFst +from nemo_text_processing.text_normalization.ar.verbalizers.time import TimeFst as TNTimeVerbalizer from nemo_text_processing.text_normalization.en.graph_utils import INPUT_LOWER_CASED from pynini.lib import pynutil @@ -84,6 +86,9 @@ def __init__( deterministic=True, ) measure_graph = measure.fst + tn_time_verbalizer = TNTimeVerbalizer(cardinal_tagger=tn_classify.cardinal, deterministic=True) + time = TimeFst(tn_time_verbalizer=tn_time_verbalizer) + time_graph = time.fst word_graph = WordFst().fst punct_graph = PunctuationFst().fst @@ -93,6 +98,7 @@ def __init__( | pynutil.add_weight(fraction_graph, 1.1) | pynutil.add_weight(money_graph, 1.1) | pynutil.add_weight(measure_graph, 1.1) + | pynutil.add_weight(time_graph, 1.1) | pynutil.add_weight(word_graph, 100) ) diff --git a/nemo_text_processing/inverse_text_normalization/ar/verbalizers/time.py b/nemo_text_processing/inverse_text_normalization/ar/verbalizers/time.py new file mode 100644 index 000000000..84875142b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ar/verbalizers/time.py @@ -0,0 +1,57 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.ar.graph_utils import ( + NEMO_DIGIT, + NEMO_NOT_QUOTE, + GraphFst, + delete_space, +) +from pynini.lib import pynutil + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing a time token into digits, e.g. + time { hours: "8" minutes: "30" } -> 08:30 + time { hours: "8" minutes: "30" seconds: "10" } -> 08:30:10 + time { hours: "9" suffix: "صباحًا" } -> 9 صباحًا + time { hours: "8" } -> 8 + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="time", kind="verbalize", deterministic=deterministic) + + add_leading_zero_to_double_digit = (NEMO_DIGIT + NEMO_DIGIT) | (pynutil.insert("0") + NEMO_DIGIT) + hour = pynutil.delete("hours: \"") + pynini.closure(NEMO_DIGIT, 1) + pynutil.delete("\"") + minute = pynutil.delete("minutes: \"") + pynini.closure(NEMO_DIGIT, 1) + pynutil.delete("\"") + second = pynutil.delete("seconds: \"") + pynini.closure(NEMO_DIGIT, 1) + pynutil.delete("\"") + zone = pynutil.delete("zone: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + optional_zone = pynini.closure(pynini.accep(" ") + zone, 0, 1) + suffix = pynutil.delete("suffix: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + optional_suffix = pynini.closure(pynini.accep(" ") + suffix, 0, 1) + + graph_minutes_seconds = ( + delete_space + + pynutil.insert(":") + + (minute @ add_leading_zero_to_double_digit) + + pynini.closure( + delete_space + pynutil.insert(":") + (second @ add_leading_zero_to_double_digit), 0, 1 + ) + ) + graph_h = hour + graph_hms = hour @ add_leading_zero_to_double_digit + graph_minutes_seconds + final_graph = (graph_hms | graph_h) + optional_suffix + optional_zone + self.fst = self.delete_tokens(final_graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ar/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/ar/verbalizers/verbalize.py index e9f1758f8..9b56a42aa 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/verbalizers/verbalize.py +++ b/nemo_text_processing/inverse_text_normalization/ar/verbalizers/verbalize.py @@ -18,6 +18,7 @@ from nemo_text_processing.inverse_text_normalization.ar.verbalizers.fraction import FractionFst from nemo_text_processing.inverse_text_normalization.ar.verbalizers.measure import MeasureFst from nemo_text_processing.inverse_text_normalization.ar.verbalizers.money import MoneyFst +from nemo_text_processing.inverse_text_normalization.ar.verbalizers.time import TimeFst from nemo_text_processing.text_normalization.ar.graph_utils import GraphFst @@ -40,5 +41,7 @@ def __init__(self): money_graph = money.fst measure = MeasureFst(decimal=decimal, cardinal=cardinal, deterministic=True) measure_graph = measure.fst - graph = cardinal_graph | decimal_graph | fraction_graph | money_graph | measure_graph + time = TimeFst() + time_graph = time.fst + graph = cardinal_graph | decimal_graph | fraction_graph | money_graph | measure_graph | time_graph self.fst = graph diff --git a/nemo_text_processing/text_normalization/ar/data/number/13_19.tsv b/nemo_text_processing/text_normalization/ar/data/number/13_19.tsv new file mode 100644 index 000000000..236d6234a --- /dev/null +++ b/nemo_text_processing/text_normalization/ar/data/number/13_19.tsv @@ -0,0 +1,9 @@ +11 إحدى عشرة +12 اثنتي عشرة +13 ثلاث عشرة +14 أربع عشرة +15 خمس عشرة +16 ست عشرة +17 سبع عشرة +18 ثمان عشرة +19 تسع عشرة diff --git a/nemo_text_processing/text_normalization/ar/data/number/3_10mas.tsv b/nemo_text_processing/text_normalization/ar/data/number/3_10mas.tsv new file mode 100644 index 000000000..6130e3b06 --- /dev/null +++ b/nemo_text_processing/text_normalization/ar/data/number/3_10mas.tsv @@ -0,0 +1,8 @@ +3 ثلاث +4 أربع +5 خمس +6 ست +7 سبع +8 ثمان +9 تسع +10 عشر diff --git a/nemo_text_processing/text_normalization/ar/data/ordinal/ordinals.tsv b/nemo_text_processing/text_normalization/ar/data/ordinal/ordinals.tsv new file mode 100644 index 000000000..5b1c70b5a --- /dev/null +++ b/nemo_text_processing/text_normalization/ar/data/ordinal/ordinals.tsv @@ -0,0 +1,24 @@ +1 الواحدة +2 الثانية +3 الثالثة +4 الرابعة +5 الخامسة +6 السادسة +7 السابعة +8 الثامنة +9 التاسعة +10 العاشرة +11 الحادية عشرة +12 الثانية عشرة +13 الثالثة عشرة +14 الرابعة عشرة +15 الخامسة عشرة +16 السادسة عشرة +17 السابعة عشرة +18 الثامنة عشرة +19 التاسعة عشرة +20 العشرين +21 الحادية والعشرين +22 الثانية والعشرين +23 الثالثة والعشرين +24 الرابعة والعشرين \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ar/data/time/suffix.tsv b/nemo_text_processing/text_normalization/ar/data/time/suffix.tsv new file mode 100644 index 000000000..14330befb --- /dev/null +++ b/nemo_text_processing/text_normalization/ar/data/time/suffix.tsv @@ -0,0 +1,10 @@ +صباحاً صباحًا +صباحا صباحًا +مساءً مساءً +مساء مساءً +ظهراً ظهراً +ظهرا ظهراً +عصراً عصراً +عصرا عصراً +ص صباحًا +م مساءً diff --git a/nemo_text_processing/text_normalization/ar/data/time/time_zone.tsv b/nemo_text_processing/text_normalization/ar/data/time/time_zone.tsv new file mode 100644 index 000000000..0fda04208 --- /dev/null +++ b/nemo_text_processing/text_normalization/ar/data/time/time_zone.tsv @@ -0,0 +1,14 @@ +cst CST +c.s.t CST +cet CET +c.e.t CET +pst PST +p.s.t PST +est EST +e.s.t EST +pt PT +p.t PT +et ET +e.t ET +gmt GMT +g.m.t GMT diff --git a/nemo_text_processing/text_normalization/ar/taggers/time.py b/nemo_text_processing/text_normalization/ar/taggers/time.py new file mode 100644 index 000000000..642f6feda --- /dev/null +++ b/nemo_text_processing/text_normalization/ar/taggers/time.py @@ -0,0 +1,105 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from nemo_text_processing.text_normalization.ar.graph_utils import ( + NEMO_DIGIT, + GraphFst, + convert_space, + delete_space, + insert_space, +) +from nemo_text_processing.text_normalization.ar.utils import get_abs_path +from pynini.lib import pynutil + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying time, e.g. + "02:15" -> time { hours: "2" minutes: "15" } + "9 صباحاً" -> time { hours: "9" suffix: "صباحاً" } + "09:00" -> time { hours: "9" } + "02:15:10" -> time { hours: "2" minutes: "15" seconds: "10" } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="time", kind="classify", deterministic=deterministic) + suffix_graph = pynini.string_file(get_abs_path("data/time/suffix.tsv")) + + final_suffix = pynutil.insert("suffix: \"") + suffix_graph + pynutil.insert("\"") + final_suffix_optional = pynini.closure(delete_space + insert_space + final_suffix, 0, 1) + time_zone_graph = pynini.string_file(get_abs_path("data/time/time_zone.tsv")) + + labels_hour = [str(x) for x in range(0, 25)] + labels_minute_single = [str(x) for x in range(1, 10)] + labels_minute_double = [str(x) for x in range(10, 60)] + + delete_leading_zero_to_double_digit = (pynini.closure(pynutil.delete("0"), 0, 1) | (NEMO_DIGIT - "0")) + NEMO_DIGIT + + graph_hour = pynini.union(*labels_hour) + + graph_minute_single = pynini.union(*labels_minute_single) + graph_minute_double = pynini.union(*labels_minute_double) + + final_graph_hour_only = pynutil.insert("hours: \"") + graph_hour + pynutil.insert("\"") + final_graph_hour = ( + pynutil.insert("hours: \"") + delete_leading_zero_to_double_digit @ graph_hour + pynutil.insert("\"") + ) + final_graph_minute = ( + pynutil.insert("minutes: \"") + + (pynutil.delete("0") + graph_minute_single | graph_minute_double) + + pynutil.insert("\"") + ) + final_graph_second = ( + pynutil.insert("seconds: \"") + + (pynutil.delete("0") + graph_minute_single | graph_minute_double) + + pynutil.insert("\"") + ) + final_time_zone_optional = pynini.closure( + pynini.accep(" ") + pynutil.insert("zone: \"") + convert_space(time_zone_graph) + pynutil.insert("\""), + 0, + 1, + ) + + # 02:30 + graph_hm = ( + final_graph_hour + + pynutil.delete(":") + + (pynutil.delete("00") | (insert_space + final_graph_minute)) + + final_suffix_optional + + final_time_zone_optional + ) + + # 10:30:05 + graph_hms = ( + final_graph_hour + + pynutil.delete(":") + + (pynini.cross("00", " minutes: \"0\"") | (insert_space + final_graph_minute)) + + pynutil.delete(":") + + (pynini.cross("00", " seconds: \"0\"") | (insert_space + final_graph_second)) + + final_suffix_optional + + final_time_zone_optional + + pynutil.insert(" preserve_order: true") + ) + + # 9 صباحاً + graph_h = final_graph_hour_only + final_suffix + final_graph = (graph_hm | graph_h | graph_hms).optimize() + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/ar/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/ar/taggers/tokenize_and_classify.py index 833652d76..d5ca7e6c8 100644 --- a/nemo_text_processing/text_normalization/ar/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/text_normalization/ar/taggers/tokenize_and_classify.py @@ -29,6 +29,7 @@ from nemo_text_processing.text_normalization.ar.taggers.fraction import FractionFst from nemo_text_processing.text_normalization.ar.taggers.measure import MeasureFst from nemo_text_processing.text_normalization.ar.taggers.money import MoneyFst +from nemo_text_processing.text_normalization.ar.taggers.time import TimeFst from nemo_text_processing.text_normalization.ar.taggers.word import WordFst from nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst from pynini.lib import pynutil @@ -85,6 +86,8 @@ def __init__( cardinal=self.cardinal, decimal=self.decimal, fraction=self.fraction, deterministic=deterministic ) measure_graph = self.measure.fst + self.time = TimeFst(deterministic=deterministic) + time_graph = self.time.fst word_graph = WordFst(deterministic=deterministic).fst punct_graph = PunctuationFst(deterministic=deterministic).fst @@ -94,6 +97,7 @@ def __init__( | pynutil.add_weight(fraction_graph, 1.0) | pynutil.add_weight(money_graph, 1.0) | pynutil.add_weight(measure_graph, 1.0) + | pynutil.add_weight(time_graph, 1.0) ) classify |= pynutil.add_weight(word_graph, 100) diff --git a/nemo_text_processing/text_normalization/ar/verbalizers/time.py b/nemo_text_processing/text_normalization/ar/verbalizers/time.py new file mode 100644 index 000000000..0419ab279 --- /dev/null +++ b/nemo_text_processing/text_normalization/ar/verbalizers/time.py @@ -0,0 +1,192 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.ar.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + GraphFst, + convert_space, + delete_preserve_order, +) +from nemo_text_processing.text_normalization.ar.utils import get_abs_path, load_labels +from pynini.lib import pynutil + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing time in a regular (formal) style, e.g. + 'time { hours: "7" minutes: "12" }' -> السابعة واثنتي عشرة دقيقة + 'time { hours: "1" minutes: "1" }' -> الواحدة ودقيقة واحدة + 'time { hours: "2" minutes: "2" }' -> الثانية ودقيقتان + 'time { hours: "3" minutes: "30" }' -> الثالثة وثلاثين دقيقة + 'time { hours: "18" minutes: "30" }' -> الثامنة عشرة وثلاثين دقيقة + 'time { hours: "5" minutes: "10" seconds: "2" preserve_order: true }' -> الخامسة وعشر دقائق وثانيتان + 'time { hours: "9" suffix: "صباحًا" }' -> التاسعة صباحًا + + The hour is a feminine ordinal (24-hour), the minutes/seconds are regular + cardinals with number agreement (دقيقة/دقيقتان/دقائق). No colloquial + ربع/نصف/إلا forms are produced, so the mapping is deterministic and consistent + across h:m and h:m:s. + + Args: + cardinal_tagger: cardinal tagger GraphFst (provides the number verbalization graph) + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal_tagger: GraphFst, deterministic: bool = True): + super().__init__(name="time", kind="verbalize", deterministic=deterministic) + + n3_10 = pynini.union(*[str(x) for x in range(2, 11)]) + n13_19 = pynini.union(*[str(x) for x in range(11, 20)]) + + ordinals = pynini.string_file(get_abs_path("data/ordinal/ordinals.tsv")) + mas_3_10 = pynini.string_file(get_abs_path("data/number/3_10mas.tsv")) + graph_13_19 = pynini.string_file(get_abs_path("data/number/13_19.tsv")) + time_zone_graph = pynini.invert( + convert_space( + pynini.union(*[x[1] for x in load_labels(get_abs_path("data/time/time_zone.tsv"))]) + ) + ) + number_verbalization = cardinal_tagger.graph + + # hour -> feminine ordinal; midnight hour "0" reads as الثانية عشرة + hour = pynutil.delete('hours: "') + (pynini.cross("0", "12") | pynini.closure(NEMO_DIGIT, 1)) + pynutil.delete('"') + hour_verbalized = pynutil.add_weight(hour @ ordinals, weight=0.001) + + # minutes: 3-10 -> plural دقائق, 12-19 -> teens + دقيقة, else -> cardinal + دقيقة + minute_singular = ( + pynutil.delete('minutes: "') + + pynini.closure(NEMO_DIGIT, 1) + @ number_verbalization + @ pynini.cdrewrite(pynini.cross("اثنا عشر", "اثنتا عشرة"), "", "", NEMO_SIGMA) + @ pynini.cdrewrite(pynini.cross("احد عشر", "احدى عشرة"), "", "", NEMO_SIGMA) + + pynutil.insert(" دقيقة") + + pynutil.delete('"') + ) + minute_plural = ( + pynutil.delete('minutes: "') + + n3_10 @ pynini.closure(NEMO_DIGIT, 1) @ mas_3_10 + + pynutil.insert(" دقائق") + + pynutil.delete('"') + ) + minute_13_19 = ( + pynutil.delete('minutes: "') + + n13_19 @ pynini.closure(NEMO_DIGIT, 1) @ graph_13_19 + + pynutil.insert(" دقيقة") + + pynutil.delete('"') + ) + # Reverse (polarity) agreement for the ones digit of a 21-59 compound before a + # feminine counted noun (دقيقة/ثانية): the cardinal graph emits the masculine + # citation form (خمسة وأربعين), but a feminine noun requires the feminine form + # (خمس وأربعين). Fires only when the ones word is immediately followed by " و" + # (the tens connector), so standalone/teens/plural forms are left untouched. + feminize_unit = pynini.cdrewrite( + pynini.cross("واحد", "إحدى") + | pynini.cross("اثنين", "اثنتان") + | pynini.cross("ثلاثة", "ثلاث") + | pynini.cross("أربعة", "أربع") + | pynini.cross("خمسة", "خمس") + | pynini.cross("ستة", "ست") + | pynini.cross("سبعة", "سبع") + | pynini.cross("ثمانية", "ثمان") + | pynini.cross("تسعة", "تسع"), + "", + " و", + NEMO_SIGMA, + ) + + # The minutes/seconds number is the conjoined predicate of the time phrase and is + # therefore nominative (عشرون/ثلاثون/...), while the cardinal graph defaults to the + # genitive/accusative form (عشرين/ثلاثين/...). Applied to minutes/seconds only, not + # the hour ordinal (e.g. الثالثة والعشرين for the 23rd hour stays unchanged). + nominative_tens = pynini.cdrewrite( + pynini.cross("عشرين", "عشرون") + | pynini.cross("ثلاثين", "ثلاثون") + | pynini.cross("أربعين", "أربعون") + | pynini.cross("خمسين", "خمسون"), + "", + "", + NEMO_SIGMA, + ) + + minute_verbalized = ( + (minute_plural | minute_13_19 | pynutil.add_weight(minute_singular, 0.001)) @ feminize_unit @ nominative_tens + ) + + # constrain suffix to the known suffix.tsv values (not an open NEMO_NOT_QUOTE + # catch-all) so that inverting this verbalizer for ITN does not greedily absorb + # the minutes/seconds text as a "suffix". + suffix_values = pynini.union(*[x[1] for x in load_labels(get_abs_path("data/time/suffix.tsv"))]) + zone = pynutil.delete('zone: "') + time_zone_graph + pynutil.delete('"') + suffix = pynutil.delete('suffix: "') + suffix_values + pynutil.delete('"') + optional_suffix = pynini.closure(pynini.accep(" ") + suffix, 0, 1) + optional_zone = pynini.closure(pynini.accep(" ") + zone, 0, 1) + + second_singular = ( + pynutil.delete('seconds: "') + + pynini.closure(NEMO_DIGIT, 1) + @ number_verbalization + @ pynini.cdrewrite(pynini.cross("اثنا عشر", "اثنتا عشرة"), "", "", NEMO_SIGMA) + @ pynini.cdrewrite(pynini.cross("احد عشر", "احدى عشرة"), "", "", NEMO_SIGMA) + + pynutil.insert(" ثانية") + + pynutil.delete('"') + ) + second_plural = ( + pynutil.delete('seconds: "') + + (n3_10 @ pynini.closure(NEMO_DIGIT, 1) @ mas_3_10 + pynutil.insert(" ثواني")) + + pynutil.delete('"') + ) + second_13_19 = ( + pynutil.delete('seconds: "') + + n13_19 @ pynini.closure(NEMO_DIGIT, 1) @ graph_13_19 + + pynutil.insert(" ثانية") + + pynutil.delete('"') + ) + second_verbalized = ( + (second_plural | second_13_19 | pynutil.add_weight(second_singular, 0.001)) @ feminize_unit @ nominative_tens + ) + + # agreement fixes for 1/2 minute and second + agreement = pynini.cdrewrite( + pynini.cross("واحد دقيقة", "دقيقة واحدة") + | pynini.cross("واحد ثانية", "ثانية واحدة") + | pynini.cross("اثنين دقيقة", "دقيقتان") + | pynini.cross("اثنين ثانية", "ثانيتان"), + "", + "", + NEMO_SIGMA, + ) + + graph_hms = ( + hour_verbalized + + pynini.accep(" ") + + pynutil.insert("و") + + minute_verbalized + + pynini.accep(" ") + + pynutil.insert("و") + + second_verbalized + ) + graph_hms @= agreement + + graph_hm = hour_verbalized + pynini.accep(" ") + pynutil.insert("و") + minute_verbalized + graph_hm @= agreement + + # whole hour only (minutes were dropped by the tagger) + graph_h = hour_verbalized + + self.graph = (graph_hms | graph_h | graph_hm) + optional_suffix + optional_zone + delete_tokens = self.delete_tokens(self.graph + delete_preserve_order) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/ar/verbalizers/verbalize.py b/nemo_text_processing/text_normalization/ar/verbalizers/verbalize.py index d693abf28..11fede496 100644 --- a/nemo_text_processing/text_normalization/ar/verbalizers/verbalize.py +++ b/nemo_text_processing/text_normalization/ar/verbalizers/verbalize.py @@ -19,6 +19,7 @@ from nemo_text_processing.text_normalization.ar.verbalizers.fraction import FractionFst from nemo_text_processing.text_normalization.ar.verbalizers.measure import MeasureFst from nemo_text_processing.text_normalization.ar.verbalizers.money import MoneyFst +from nemo_text_processing.text_normalization.ar.verbalizers.time import TimeFst class VerbalizeFst(GraphFst): @@ -45,6 +46,8 @@ def __init__(self, deterministic: bool = True): money_graph = money.fst measure = MeasureFst(decimal=decimal, cardinal=cardinal, fraction=fraction, deterministic=deterministic) measure_graph = measure.fst + time = TimeFst(cardinal_tagger=cardinal_tagger, deterministic=deterministic) + time_graph = time.fst - graph = cardinal_graph | decimal_graph | fraction_graph | money_graph | measure_graph + graph = cardinal_graph | decimal_graph | fraction_graph | money_graph | measure_graph | time_graph self.fst = graph diff --git a/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_time.txt new file mode 100644 index 000000000..d851b7e64 --- /dev/null +++ b/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_time.txt @@ -0,0 +1,19 @@ +الثالثة~3 +الثالثة وخمس عشرة دقيقة~03:15 +الثالثة وخمس وأربعون دقيقة~03:45 +السابعة واثنتي عشرة دقيقة~07:12 +الواحدة ودقيقة واحدة~01:01 +الثانية ودقيقتان~02:02 +الحادية عشرة وإحدى عشرة دقيقة~11:11 +الثانية عشرة~12 +الثامنة عشرة وثلاثون دقيقة~18:30 +الثالثة والعشرين وتسع وخمسون دقيقة~23:59 +الخامسة وعشر دقائق وثانيتان~05:10:02 +العاشرة وثلاثون دقيقة وخمس ثواني~10:30:05 +الثامنة وخمس وعشرون دقيقة~08:25 +الرابعة وإحدى وعشرون دقيقة~04:21 +العاشرة واثنتان وعشرون دقيقة~10:22 +التاسعة صباحًا~9 صباحًا +التاسعة وثلاثون دقيقة مساءً~09:30 مساءً +العاشرة وصفر دقيقة وخمس ثواني~10:00:05 +الثامنة وصفر دقيقة وثلاثون ثانية~08:00:30 diff --git a/tests/nemo_text_processing/ar/data_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/ar/data_text_normalization/test_cases_time.txt new file mode 100644 index 000000000..924faeb4b --- /dev/null +++ b/tests/nemo_text_processing/ar/data_text_normalization/test_cases_time.txt @@ -0,0 +1,30 @@ +3:00~الثالثة +3:05~الثالثة وخمس دقائق +3:12~الثالثة واثنتي عشرة دقيقة +3:15~الثالثة وخمس عشرة دقيقة +3:20~الثالثة وعشرون دقيقة +3:30~الثالثة وثلاثون دقيقة +3:45~الثالثة وخمس وأربعون دقيقة +3:50~الثالثة وخمسون دقيقة +7:12~السابعة واثنتي عشرة دقيقة +1:01~الواحدة ودقيقة واحدة +2:02~الثانية ودقيقتان +11:11~الحادية عشرة وإحدى عشرة دقيقة +12:00~الثانية عشرة +18:30~الثامنة عشرة وثلاثون دقيقة +23:59~الثالثة والعشرين وتسع وخمسون دقيقة +00:15~الثانية عشرة وخمس عشرة دقيقة +5:10:02~الخامسة وعشر دقائق وثانيتان +10:30:05~العاشرة وثلاثون دقيقة وخمس ثواني +8:25~الثامنة وخمس وعشرون دقيقة +4:21~الرابعة وإحدى وعشرون دقيقة +10:22~العاشرة واثنتان وعشرون دقيقة +7:33~السابعة وثلاث وثلاثون دقيقة +2:48~الثانية وثمان وأربعون دقيقة +6:57~السادسة وسبع وخمسون دقيقة +1:24:38~الواحدة وأربع وعشرون دقيقة وثمان وثلاثون ثانية +6:40~السادسة وأربعون دقيقة +9:00 صباحاً~التاسعة صباحًا +9:30 مساءً~التاسعة وثلاثون دقيقة مساءً +10:00:05~العاشرة وصفر دقيقة وخمس ثواني +8:00:30~الثامنة وصفر دقيقة وثلاثون ثانية diff --git a/tests/nemo_text_processing/ar/test_time.py b/tests/nemo_text_processing/ar/test_time.py new file mode 100644 index 000000000..e147480ad --- /dev/null +++ b/tests/nemo_text_processing/ar/test_time.py @@ -0,0 +1,40 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer +from parameterized import parameterized + +from tests.nemo_text_processing.utils import CACHE_DIR, parse_test_case_file + + +class TestTime: + normalizer = Normalizer(input_case='cased', lang='ar', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ar/data_text_normalization/test_cases_time.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected, f"input: {test_input}" + + inverse_normalizer = InverseNormalizer(lang='ar', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ar/data_inverse_text_normalization/test_cases_time.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected, f"input: {test_input}" From 8192b0dc578e8f8118901e90d4ee6730e5e9e45e Mon Sep 17 00:00:00 2001 From: Enas Albasiri Date: Mon, 6 Jul 2026 17:02:29 +0000 Subject: [PATCH 11/12] feat(ar): add date normalization and inverse normalization (TN + ITN). tests 399 passed. --- .../ar/taggers/date.py | 113 ++++++++++++++ .../ar/taggers/tokenize_and_classify.py | 12 ++ .../ar/data/months/abbr_to_name.tsv | 12 ++ .../ar/data/months/hijri_months.tsv | 24 +++ .../ar/data/months/hijri_suffixes.tsv | 4 + .../ar/data/months/hijri_suffixes_inverse.tsv | 5 + .../ar/data/months/months_name.tsv | 33 ++++ .../ar/data/months/numbers.tsv | 24 +++ .../ar/data/ordinal/ordinals_date.tsv | 32 ++++ .../text_normalization/ar/taggers/date.py | 145 ++++++++++++++++++ .../ar/taggers/tokenize_and_classify.py | 4 + .../text_normalization/ar/verbalizers/date.py | 74 +++++++++ .../ar/verbalizers/verbalize.py | 13 +- .../test_cases_date.txt | 14 ++ .../test_cases_date.txt | 30 ++++ tests/nemo_text_processing/ar/test_date.py | 40 +++++ 16 files changed, 578 insertions(+), 1 deletion(-) create mode 100644 nemo_text_processing/inverse_text_normalization/ar/taggers/date.py create mode 100644 nemo_text_processing/text_normalization/ar/data/months/abbr_to_name.tsv create mode 100644 nemo_text_processing/text_normalization/ar/data/months/hijri_months.tsv create mode 100644 nemo_text_processing/text_normalization/ar/data/months/hijri_suffixes.tsv create mode 100644 nemo_text_processing/text_normalization/ar/data/months/hijri_suffixes_inverse.tsv create mode 100644 nemo_text_processing/text_normalization/ar/data/months/months_name.tsv create mode 100644 nemo_text_processing/text_normalization/ar/data/months/numbers.tsv create mode 100644 nemo_text_processing/text_normalization/ar/data/ordinal/ordinals_date.tsv create mode 100644 nemo_text_processing/text_normalization/ar/taggers/date.py create mode 100644 nemo_text_processing/text_normalization/ar/verbalizers/date.py create mode 100644 tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_date.txt create mode 100644 tests/nemo_text_processing/ar/data_text_normalization/test_cases_date.txt create mode 100644 tests/nemo_text_processing/ar/test_date.py diff --git a/nemo_text_processing/inverse_text_normalization/ar/taggers/date.py b/nemo_text_processing/inverse_text_normalization/ar/taggers/date.py new file mode 100644 index 000000000..b59550b9b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ar/taggers/date.py @@ -0,0 +1,113 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.ar.graph_utils import ( + NEMO_DIGIT, + NEMO_NOT_QUOTE, + NEMO_SIGMA, + GraphFst, + convert_space, +) +from nemo_text_processing.text_normalization.ar.utils import get_abs_path, load_labels +from pynini.lib import pynutil + + +class DateFst(GraphFst): + """ + Finite state transducer for classifying spoken Arabic date into a digit form, + by inverting the text-normalization date tagger and verbalizer, e.g. + الأول من شهر نوفمبر لعام ألفين وعشرة -> tokens { name: "01/11/2010" } + الأول من شهر نوفمبر -> tokens { name: "1 نوفمبر" } + الأول من شهر ربيع الثاني لعام ... هجري -> tokens { name: "01/04/1445 هـ" } + + Args: + itn_cardinal_tagger: ITN cardinal tagger + tn_date_tagger: TN date tagger + tn_date_verbalizer: TN date verbalizer + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__( + self, + itn_cardinal_tagger: GraphFst, + tn_date_tagger: GraphFst, + tn_date_verbalizer: GraphFst, + deterministic: bool = True, + ): + super().__init__(name="date", kind="classify", deterministic=deterministic) + + add_leading_zero_to_double_digit = (NEMO_DIGIT + NEMO_DIGIT) | (pynutil.insert("0") + NEMO_DIGIT) + optional_delete_space = pynini.closure(NEMO_SIGMA | pynutil.delete(" ", weight=0.0001)) + ordinals = pynini.string_file(get_abs_path("data/ordinal/ordinals_date.tsv")).invert().optimize() + hijri_suffixes = pynini.string_file(get_abs_path("data/months/hijri_suffixes_inverse.tsv")).optimize() + months_names = [x[0] for x in load_labels(get_abs_path("data/months/months_name.tsv"))] + months_names = pynini.union(*months_names) + month_to_number = tn_date_tagger.number_to_month.invert().optimize() + month_to_number_hijri = tn_date_tagger.month_hijri.invert().optimize() + tagger = tn_date_verbalizer.graph.invert().optimize() + + delete_day_marker = ( + pynutil.delete("day: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) @ ordinals + + month_as_number = pynutil.delete("month: \"") + itn_cardinal_tagger.graph + pynutil.delete("\"") + month_as_string = pynutil.delete("month: \"") + months_names + pynutil.delete("\"") + month_name_to_number = ( + pynutil.delete("month: \"") + (month_to_number | month_to_number_hijri) + pynutil.delete("\"") + ) + + convert_year = (tn_date_tagger.year @ optional_delete_space).invert().optimize() + convert_year |= ( + (tn_date_tagger.year @ optional_delete_space).invert().optimize() + pynini.accep(" ") + hijri_suffixes + ) + delete_year_marker = ( + pynutil.delete("year: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) @ convert_year + + # day month as string (year) + verbalizer = pynutil.add_weight( + ( + pynini.closure(delete_day_marker + pynini.accep(" "), 0, 1) + + month_as_string + + pynini.closure(pynini.accep(" ") + delete_year_marker, 0, 1) + ), + weight=0.0001, + ) + + # day month as number (year); trailing "/" only when a year is present + verbalizer |= ( + delete_day_marker @ add_leading_zero_to_double_digit + + pynutil.insert("/") + + pynutil.delete(" ") + + month_as_number @ add_leading_zero_to_double_digit + + pynini.closure(pynutil.insert("/") + pynutil.delete(" ") + delete_year_marker, 0, 1) + ) + # 02/03/2022 هـ + verbalizer |= ( + delete_day_marker @ add_leading_zero_to_double_digit + + pynutil.insert("/") + + pynutil.delete(" ") + + month_name_to_number @ add_leading_zero_to_double_digit + + pynini.closure(pynutil.insert("/") + pynutil.delete(" ") + delete_year_marker, 0, 1) + ) + + # year + verbalizer |= delete_year_marker + + final_graph = tagger @ verbalizer + + graph = pynutil.insert("name: \"") + convert_space(final_graph) + pynutil.insert("\"") + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ar/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/ar/taggers/tokenize_and_classify.py index 285bfcef7..7a325877a 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/inverse_text_normalization/ar/taggers/tokenize_and_classify.py @@ -17,6 +17,7 @@ import pynini from nemo_text_processing.inverse_text_normalization.ar.taggers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.ar.taggers.date import DateFst from nemo_text_processing.inverse_text_normalization.ar.taggers.decimal import DecimalFst from nemo_text_processing.inverse_text_normalization.ar.taggers.fraction import FractionFst from nemo_text_processing.inverse_text_normalization.ar.taggers.measure import MeasureFst @@ -30,7 +31,9 @@ delete_space, generator_main, ) +from nemo_text_processing.text_normalization.ar.taggers.date import DateFst as TNDateTagger from nemo_text_processing.text_normalization.ar.taggers.tokenize_and_classify import ClassifyFst as TNClassifyFst +from nemo_text_processing.text_normalization.ar.verbalizers.date import DateFst as TNDateVerbalizer from nemo_text_processing.text_normalization.ar.verbalizers.time import TimeFst as TNTimeVerbalizer from nemo_text_processing.text_normalization.en.graph_utils import INPUT_LOWER_CASED from pynini.lib import pynutil @@ -89,6 +92,14 @@ def __init__( tn_time_verbalizer = TNTimeVerbalizer(cardinal_tagger=tn_classify.cardinal, deterministic=True) time = TimeFst(tn_time_verbalizer=tn_time_verbalizer) time_graph = time.fst + tn_date_tagger = TNDateTagger(cardinal=tn_classify.cardinal, deterministic=True) + tn_date_verbalizer = TNDateVerbalizer(deterministic=True) + date = DateFst( + itn_cardinal_tagger=cardinal, + tn_date_tagger=tn_date_tagger, + tn_date_verbalizer=tn_date_verbalizer, + ) + date_graph = date.fst word_graph = WordFst().fst punct_graph = PunctuationFst().fst @@ -99,6 +110,7 @@ def __init__( | pynutil.add_weight(money_graph, 1.1) | pynutil.add_weight(measure_graph, 1.1) | pynutil.add_weight(time_graph, 1.1) + | pynutil.add_weight(date_graph, 1.1) | pynutil.add_weight(word_graph, 100) ) diff --git a/nemo_text_processing/text_normalization/ar/data/months/abbr_to_name.tsv b/nemo_text_processing/text_normalization/ar/data/months/abbr_to_name.tsv new file mode 100644 index 000000000..e4523311a --- /dev/null +++ b/nemo_text_processing/text_normalization/ar/data/months/abbr_to_name.tsv @@ -0,0 +1,12 @@ +جانوري يناير +فابيوري فبراير +مارش مارس +أبريل ابريل +ماي مايو +جون يونيو +جولاي يوليو +أوقست أغسطس +سبتمبر سبتمبر +أوكتوبر أكتوبر +نوفمبر نوفمبر +ديسمبر ديسمبر \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ar/data/months/hijri_months.tsv b/nemo_text_processing/text_normalization/ar/data/months/hijri_months.tsv new file mode 100644 index 000000000..04de46147 --- /dev/null +++ b/nemo_text_processing/text_normalization/ar/data/months/hijri_months.tsv @@ -0,0 +1,24 @@ +1 محرم +2 صفر +3 ربيع الأول +4 ربيع الثاني +5 جماد الأول +6 جماد الثاني +7 رجب +8 شعبان +9 رمضان +10 شوال +11 ذي القعدة +12 ذي الحجة +01 محرم +02 صفر +03 ربيع الأول +04 ربيع الثاني +05 جماد الأول +06 جماد الثاني +07 رجب +08 شعبان +09 رمضان +10 شوال +11 ذي القعدة +12 ذي الحجة \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ar/data/months/hijri_suffixes.tsv b/nemo_text_processing/text_normalization/ar/data/months/hijri_suffixes.tsv new file mode 100644 index 000000000..52a21fdc7 --- /dev/null +++ b/nemo_text_processing/text_normalization/ar/data/months/hijri_suffixes.tsv @@ -0,0 +1,4 @@ +هـ هجري +هجري هجري +هجريا هجري +ه هجري \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ar/data/months/hijri_suffixes_inverse.tsv b/nemo_text_processing/text_normalization/ar/data/months/hijri_suffixes_inverse.tsv new file mode 100644 index 000000000..101b5500f --- /dev/null +++ b/nemo_text_processing/text_normalization/ar/data/months/hijri_suffixes_inverse.tsv @@ -0,0 +1,5 @@ +هـ هـ +هجري هـ +هجريا هـ +ه هـ +هجري هـ \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ar/data/months/months_name.tsv b/nemo_text_processing/text_normalization/ar/data/months/months_name.tsv new file mode 100644 index 000000000..dde8a72f3 --- /dev/null +++ b/nemo_text_processing/text_normalization/ar/data/months/months_name.tsv @@ -0,0 +1,33 @@ +محرم +صفر +ربيع الأول +ربيع الثاني +جماد الأول +جماد الثاني +رجب +شعبان +رمضان +شوال +ذي القعدة +ذي الحجة +جانوري +يناير +فابيوري +فبراير +مارش +مارس +أبريل +ابريل +ماي +مايو +جون +يونيو +جولاي +يوليو +أوقست +أغسطس +سبتمبر +أوكتوبر +أكتوبر +نوفمبر +ديسمبر \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ar/data/months/numbers.tsv b/nemo_text_processing/text_normalization/ar/data/months/numbers.tsv new file mode 100644 index 000000000..098fbbc8e --- /dev/null +++ b/nemo_text_processing/text_normalization/ar/data/months/numbers.tsv @@ -0,0 +1,24 @@ +1 يناير +2 فبراير +3 مارس +4 أبريل +5 مايو +6 يونيو +7 يوليو +8 أغسطس +9 سبتمبر +10 أكتوبر +11 نوفمبر +12 ديسمبر +01 يناير +02 فبراير +03 مارس +04 أبريل +05 مايو +06 يونيو +07 يوليو +08 أغسطس +09 سبتمبر +10 أكتوبر +11 نوفمبر +12 ديسمبر \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ar/data/ordinal/ordinals_date.tsv b/nemo_text_processing/text_normalization/ar/data/ordinal/ordinals_date.tsv new file mode 100644 index 000000000..4be4b5fd8 --- /dev/null +++ b/nemo_text_processing/text_normalization/ar/data/ordinal/ordinals_date.tsv @@ -0,0 +1,32 @@ +1 الأول +2 الثاني +3 الثالث +4 الرابع +5 الخامس +6 السادس +7 السابع +8 الثامن +9 التاسع +10 العاشر +11 الحادي عشر +12 الثاني عشر +13 الثالث عشر +14 الرابع عشر +15 الخامس عشر +16 السادس عشر +17 السابع عشر +18 الثامن عشر +19 التاسع عشر +20 العشرين +21 الحادي والعشرين +22 الثاني والعشرين +23 الثالث والعشرين +24 الرابع والعشرين +25 الخامس والعشرين +26 السادس والعشرين +27 السابع والعشرين +28 الثامن والعشرين +29 التاسع والعشرين +30 الثلاثين +31 الحادي والثلاثين +32 الثاني والثلاثين diff --git a/nemo_text_processing/text_normalization/ar/taggers/date.py b/nemo_text_processing/text_normalization/ar/taggers/date.py new file mode 100644 index 000000000..6b852eb86 --- /dev/null +++ b/nemo_text_processing/text_normalization/ar/taggers/date.py @@ -0,0 +1,145 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.ar.graph_utils import ( + NEMO_CHAR, + NEMO_DIGIT, + GraphFst, + insert_space, +) +from nemo_text_processing.text_normalization.ar.utils import get_abs_path, load_labels +from pynini.lib import pynutil + +delete_leading_zero = (pynutil.delete("0") | (NEMO_DIGIT - "0")) + NEMO_DIGIT + + +class DateFst(GraphFst): + """ + Finite state transducer for classifying date, e.g. + "01/11/2010" -> date { day: "الأول" month: "نوفمبر" year: "ألفين وعشرة" preserve_order: true } + "1 نوفمبر" -> date { day: "الأول" month: "نوفمبر" } + "نوفمبر 2010" -> date { month: "نوفمبر" year: "ألفين وعشرة" preserve_order: true } + "2010" -> date { year: "ألفين وعشرة" } + + Supports the day-month-year, year-month-day, month-year, and bare-year orders, + Gregorian and Hijri month names, and both numeric (with -, /, \\, . separators) + and spelled-out month forms. The year is emitted in the idiomatic genitive form + already produced by the cardinal grammar. Note that in the full pipeline a + standalone number is classified as a cardinal, not a bare-year date. + + Args: + cardinal: cardinal GraphFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool): + super().__init__(name="date", kind="classify", deterministic=deterministic) + months_name = pynini.string_file(get_abs_path("data/months/months_name.tsv")).optimize() + + month_abbr_graph = load_labels(get_abs_path("data/months/abbr_to_name.tsv")) + number_to_month = pynini.string_file(get_abs_path("data/months/numbers.tsv")).optimize() + number_to_month_hijri = pynini.string_file(get_abs_path("data/months/hijri_months.tsv")).optimize() + self.month_hijri = number_to_month_hijri + self.number_to_month = number_to_month + month_graph = pynini.union(*[x[1] for x in month_abbr_graph]).optimize() + month_abbr_graph = pynini.string_map(month_abbr_graph) + month_abbr_graph = ( + pynutil.add_weight(month_abbr_graph, weight=0.0001) | (pynini.closure(NEMO_CHAR) @ month_abbr_graph) + ) + pynini.closure(pynutil.delete(" ", weight=-0.0001), 0, 1) + + self.month_abbr = month_abbr_graph + month_graph |= pynini.closure(NEMO_CHAR) @ month_graph + # normalized name, or transliteration variant -> normalized name + month_graph |= month_abbr_graph + + numbers = cardinal.graph + ordinals = pynini.string_file(get_abs_path("data/ordinal/ordinals_date.tsv")).optimize() + optional_leading_zero = delete_leading_zero | NEMO_DIGIT + # 01, 31, 1 + digit_day = optional_leading_zero @ pynini.union(*[str(x) for x in range(1, 32)]) @ ordinals + day = (pynutil.insert("day: \"") + digit_day + pynutil.insert("\"")).optimize() + + digit_month = optional_leading_zero @ pynini.union(*[str(x) for x in range(1, 13)]) + number_to_month = digit_month @ number_to_month + number_to_month_hijri = digit_month @ number_to_month_hijri + digit_month @= numbers + + month_name = (pynutil.insert("month: \"") + month_graph + pynutil.insert("\"")).optimize() + month_name_normalized = (pynutil.insert("month: \"") + months_name + pynutil.insert("\"")).optimize() + month_number = ( + pynutil.insert("month: \"") + + (pynutil.add_weight(digit_month, weight=0.0001) | number_to_month) + + pynutil.insert("\"") + ).optimize() + month_number_hijri = ( + pynutil.insert("month: \"") + + (pynutil.add_weight(digit_month, weight=0.0001) | number_to_month_hijri) + + pynutil.insert("\"") + ).optimize() + + # prefer cardinal over year + year = pynutil.add_weight(numbers, weight=0.001) + self.year = year + + year_only = pynutil.insert("year: \"") + year + pynutil.insert("\"") + + graph_dmy = ( + day + + pynini.closure(pynutil.delete(" "), 0, 1) + + insert_space + + month_name + + pynini.closure(pynini.accep(" ") + year_only, 0, 1) + ) + hijri_suffixes = pynini.string_file(get_abs_path("data/months/hijri_suffixes.tsv")).optimize() + + separators = ["-", "/", "\\"] + for sep in separators: + year_optional = pynini.closure(pynini.cross(sep, " ") + year_only, 0, 1) + new_graph = day + pynini.cross(sep, " ") + month_number + year_optional + self.year_hijri = year + pynini.accep(" ") + hijri_suffixes + graph_year_hijri = pynutil.insert("year: \"") + self.year_hijri + pynutil.insert("\"") + graph_dmy_hijri = day + pynini.cross(sep, " ") + month_number_hijri + pynini.cross(sep, " ") + graph_year_hijri + graph_dmy |= new_graph + graph_dmy |= graph_dmy_hijri + + # full day.month.year with a dot separator; the year is required so that + # two-component decimals (e.g. "1.5") are not misread as a date + year_only_4digit = pynutil.insert("year: \"") + ((NEMO_DIGIT ** 4) @ year) + pynutil.insert("\"") + graph_dmy |= day + pynini.cross(".", " ") + month_number + pynini.cross(".", " ") + year_only + + # month + year, spelled ("نوفمبر 2010") and numeric ("12-2019"); + # "/" is intentionally excluded here because "11/2010" collides with the + # fraction reading (11 على 2010), which wins the ambiguity + graph_dmy |= month_name + pynini.accep(" ") + year_only + graph_dmy |= month_number + pynini.cross("-", " ") + year_only_4digit + + dash = "-" + day_optional = pynini.closure(pynini.cross(dash, " ") + day, 0, 1) + graph_ymd = year_only + pynini.cross(dash, " ") + month_number + day_optional + # 1 نوفمبر -> الأول من نوفمبر + rule_1 = day + pynini.accep(" ") + month_name_normalized + pynini.closure(pynini.accep(" ") + year_only, 0, 1) + + # penalize the bare-year reading so a standalone number stays cardinal; + # day/month/year dates below have no cardinal competitor and are unaffected + final_graph = ( + (graph_dmy + pynutil.insert(" preserve_order: true")) + | pynutil.add_weight(year_only, weight=1.0) + | graph_ymd + | rule_1 + ) + + self.final_graph = final_graph.optimize() + self.fst = self.add_tokens(self.final_graph).optimize() diff --git a/nemo_text_processing/text_normalization/ar/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/ar/taggers/tokenize_and_classify.py index d5ca7e6c8..cf54cd3a0 100644 --- a/nemo_text_processing/text_normalization/ar/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/text_normalization/ar/taggers/tokenize_and_classify.py @@ -25,6 +25,7 @@ generator_main, ) from nemo_text_processing.text_normalization.ar.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.ar.taggers.date import DateFst from nemo_text_processing.text_normalization.ar.taggers.decimal import DecimalFst from nemo_text_processing.text_normalization.ar.taggers.fraction import FractionFst from nemo_text_processing.text_normalization.ar.taggers.measure import MeasureFst @@ -88,6 +89,8 @@ def __init__( measure_graph = self.measure.fst self.time = TimeFst(deterministic=deterministic) time_graph = self.time.fst + self.date = DateFst(cardinal=self.cardinal, deterministic=deterministic) + date_graph = self.date.fst word_graph = WordFst(deterministic=deterministic).fst punct_graph = PunctuationFst(deterministic=deterministic).fst @@ -98,6 +101,7 @@ def __init__( | pynutil.add_weight(money_graph, 1.0) | pynutil.add_weight(measure_graph, 1.0) | pynutil.add_weight(time_graph, 1.0) + | pynutil.add_weight(date_graph, 1.1) ) classify |= pynutil.add_weight(word_graph, 100) diff --git a/nemo_text_processing/text_normalization/ar/verbalizers/date.py b/nemo_text_processing/text_normalization/ar/verbalizers/date.py new file mode 100644 index 000000000..40a584425 --- /dev/null +++ b/nemo_text_processing/text_normalization/ar/verbalizers/date.py @@ -0,0 +1,74 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.ar.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + GraphFst, + delete_preserve_order, +) +from nemo_text_processing.text_normalization.ar.utils import get_abs_path, load_labels +from pynini.lib import pynutil + + +class DateFst(GraphFst): + """ + Finite state transducer for verbalizing date, e.g. + date { day: "الأول" month: "نوفمبر" year: "ألفين وعشرة" preserve_order: true } + -> "الأول من شهر نوفمبر لعام ألفين وعشرة" + date { month: "نوفمبر" year: "ألفين وعشرة" } -> "نوفمبر لعام ألفين وعشرة" + date { year: "ألفين وعشرين" } -> "ألفين وعشرين" + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="date", kind="verbalize", deterministic=deterministic) + + day = pynutil.delete('day: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + + months_names = pynini.union(*[x[1] for x in load_labels(get_abs_path("data/months/abbr_to_name.tsv"))]) + hijri_months_names = pynini.union(*[x[1] for x in load_labels(get_abs_path("data/months/hijri_months.tsv"))]) + month = pynutil.delete('month: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + final_month = month @ months_names + final_month |= month @ hijri_months_names + final_month |= month @ pynini.difference(NEMO_SIGMA, months_names | hijri_months_names) + + # The cardinal grammar already emits the idiomatic genitive year form + # (e.g. 2020 -> "ألفين وعشرين"), so the year field passes through unchanged. + year = pynutil.delete('year: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + + # day month year + graph_dmy = ( + day + + pynini.accep(" ") + + pynutil.insert("من شهر ") + + final_month + + pynini.accep(" ") + + pynutil.insert("لعام ") + + year + ) + graph_dmy |= final_month + pynini.accep(" ") + pynutil.insert("لعام ") + year + + # day month + graph_dm = day + pynini.accep(" ") + pynutil.insert("من شهر ") + final_month + + self.graph = graph_dmy | pynutil.add_weight(year, weight=0.0001) | graph_dm + final_graph = self.graph + delete_preserve_order + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/ar/verbalizers/verbalize.py b/nemo_text_processing/text_normalization/ar/verbalizers/verbalize.py index 11fede496..c49e95a4a 100644 --- a/nemo_text_processing/text_normalization/ar/verbalizers/verbalize.py +++ b/nemo_text_processing/text_normalization/ar/verbalizers/verbalize.py @@ -15,6 +15,7 @@ from nemo_text_processing.text_normalization.ar.graph_utils import GraphFst from nemo_text_processing.text_normalization.ar.taggers.cardinal import CardinalFst as CardinalTagger from nemo_text_processing.text_normalization.ar.verbalizers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.ar.verbalizers.date import DateFst from nemo_text_processing.text_normalization.ar.verbalizers.decimal import DecimalFst from nemo_text_processing.text_normalization.ar.verbalizers.fraction import FractionFst from nemo_text_processing.text_normalization.ar.verbalizers.measure import MeasureFst @@ -48,6 +49,16 @@ def __init__(self, deterministic: bool = True): measure_graph = measure.fst time = TimeFst(cardinal_tagger=cardinal_tagger, deterministic=deterministic) time_graph = time.fst + date = DateFst(deterministic=deterministic) + date_graph = date.fst - graph = cardinal_graph | decimal_graph | fraction_graph | money_graph | measure_graph | time_graph + graph = ( + cardinal_graph + | decimal_graph + | fraction_graph + | money_graph + | measure_graph + | time_graph + | date_graph + ) self.fst = graph diff --git a/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_date.txt b/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_date.txt new file mode 100644 index 000000000..7db042e9b --- /dev/null +++ b/tests/nemo_text_processing/ar/data_inverse_text_normalization/test_cases_date.txt @@ -0,0 +1,14 @@ +الأول من شهر نوفمبر لعام ألفين وعشرة~01/11/2010 +الأول من شهر نوفمبر~01/11 +الخامس من شهر مايو~05/05 +الثاني عشر من شهر ديسمبر لعام ألفين واثنين وعشرين~12/12/2022 +الخامس من شهر مايو لعام ألف وتسع مئة وتسعة وتسعين~05/05/1999 +الحادي والثلاثين من شهر ديسمبر لعام ألفين~31/12/2000 +الحادي والعشرين من شهر مارس لعام ألفين وثلاثة وعشرين~21/03/2023 +السابع من شهر يوليو لعام ألفين وسبعة~07/07/2007 +الأول من شهر ربيع الثاني لعام ألف وأربع مئة وخمسة وأربعين هجري~01/04/1445 هـ +الخامس عشر من شهر رمضان لعام ألف وأربع مئة وخمسة وأربعين هجري~15/09/1445 هـ +العاشر من شهر ذي الحجة لعام ألف وأربع مئة وأربعة وأربعين هجري~10/12/1444 هـ +نوفمبر لعام ألفين وعشرة~نوفمبر 2010 +يناير لعام ألفين وثلاثة وعشرين~يناير 2023 +ديسمبر لعام ألفين وتسعة عشر~ديسمبر 2019 diff --git a/tests/nemo_text_processing/ar/data_text_normalization/test_cases_date.txt b/tests/nemo_text_processing/ar/data_text_normalization/test_cases_date.txt new file mode 100644 index 000000000..7b8e51c50 --- /dev/null +++ b/tests/nemo_text_processing/ar/data_text_normalization/test_cases_date.txt @@ -0,0 +1,30 @@ +1 نوفمبر~الأول من شهر نوفمبر +5 مايو~الخامس من شهر مايو +10 يناير~العاشر من شهر يناير +22 أغسطس~الثاني والعشرين من شهر أغسطس +1 نوفمبر 2010~الأول من شهر نوفمبر لعام ألفين وعشرة +01/11/2010~الأول من شهر نوفمبر لعام ألفين وعشرة +01/04/2010~الأول من شهر أبريل لعام ألفين وعشرة +12/12/2022~الثاني عشر من شهر ديسمبر لعام ألفين واثنين وعشرين +5 مايو 1999~الخامس من شهر مايو لعام ألف وتسع مئة وتسعة وتسعين +31/12/2000~الحادي والثلاثين من شهر ديسمبر لعام ألفين +21/03/2023~الحادي والعشرين من شهر مارس لعام ألفين وثلاثة وعشرين +07/07/2007~السابع من شهر يوليو لعام ألفين وسبعة +15/06/1990~الخامس عشر من شهر يونيو لعام ألف وتسع مئة وتسعين +28/02/2016~الثامن والعشرين من شهر فبراير لعام ألفين وستة عشر +3-9-2019~الثالث من شهر سبتمبر لعام ألفين وتسعة عشر +2010-04-01~الأول من شهر أبريل لعام ألفين وعشرة +2023-01-15~الخامس عشر من شهر يناير لعام ألفين وثلاثة وعشرين +01/04/1445 هـ~الأول من شهر ربيع الثاني لعام ألف وأربع مئة وخمسة وأربعين هجري +15/09/1445 هجري~الخامس عشر من شهر رمضان لعام ألف وأربع مئة وخمسة وأربعين هجري +10/12/1444 هـ~العاشر من شهر ذي الحجة لعام ألف وأربع مئة وأربعة وأربعين هجري +نوفمبر 2010~نوفمبر لعام ألفين وعشرة +يناير 2023~يناير لعام ألفين وثلاثة وعشرين +12-2019~ديسمبر لعام ألفين وتسعة عشر +05-2023~مايو لعام ألفين وثلاثة وعشرين +01.11.2010~الأول من شهر نوفمبر لعام ألفين وعشرة +5.5.1999~الخامس من شهر مايو لعام ألف وتسع مئة وتسعة وتسعين +31.12.2000~الحادي والثلاثين من شهر ديسمبر لعام ألفين +2010~ألفين وعشرة +2020~ألفين وعشرين +1999~ألف وتسع مئة وتسعة وتسعين diff --git a/tests/nemo_text_processing/ar/test_date.py b/tests/nemo_text_processing/ar/test_date.py new file mode 100644 index 000000000..52ff60e30 --- /dev/null +++ b/tests/nemo_text_processing/ar/test_date.py @@ -0,0 +1,40 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer +from parameterized import parameterized + +from tests.nemo_text_processing.utils import CACHE_DIR, parse_test_case_file + + +class TestDate: + normalizer = Normalizer(input_case='cased', lang='ar', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ar/data_text_normalization/test_cases_date.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected, f"input: {test_input}" + + inverse_normalizer = InverseNormalizer(lang='ar', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ar/data_inverse_text_normalization/test_cases_date.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected, f"input: {test_input}" From 9563b578ab3c1b57f1e8d63d15d1a2c944eb3a7d Mon Sep 17 00:00:00 2001 From: Enas Albasiri Date: Tue, 7 Jul 2026 19:18:41 +0000 Subject: [PATCH 12/12] chore(ar): add __init__.py to time/date data folders (time, ordinal, months) Signed-off-by: Enas Albasiri --- .../text_normalization/ar/data/months/__init__.py | 13 +++++++++++++ .../text_normalization/ar/data/ordinal/__init__.py | 13 +++++++++++++ .../text_normalization/ar/data/time/__init__.py | 13 +++++++++++++ 3 files changed, 39 insertions(+) create mode 100644 nemo_text_processing/text_normalization/ar/data/months/__init__.py create mode 100644 nemo_text_processing/text_normalization/ar/data/ordinal/__init__.py create mode 100644 nemo_text_processing/text_normalization/ar/data/time/__init__.py diff --git a/nemo_text_processing/text_normalization/ar/data/months/__init__.py b/nemo_text_processing/text_normalization/ar/data/months/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/ar/data/months/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ar/data/ordinal/__init__.py b/nemo_text_processing/text_normalization/ar/data/ordinal/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/ar/data/ordinal/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ar/data/time/__init__.py b/nemo_text_processing/text_normalization/ar/data/time/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/ar/data/time/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License.