From ffa22558e09e4a385bac71076922c7de1831da75 Mon Sep 17 00:00:00 2001 From: Mai Anh Date: Fri, 3 Jul 2026 01:37:07 +0700 Subject: [PATCH 1/2] Japanese TN fix and improvement Signed-off-by: Mai Anh --- .../ja/data/numbers/zero.tsv | 1 - .../ja/data/time_hours.tsv | 2 +- .../ja/taggers/fraction.py | 4 +- .../ja/data/electronic/__init__.py | 13 ++ .../ja/data/electronic/card_cues.tsv | 2 + .../ja/data/electronic/file_extensions.tsv | 6 + .../ja/data/electronic/special_digit_runs.tsv | 1 + .../ja/data/electronic/symbol.tsv | 5 + .../ja/data/latin/__init__.py | 13 ++ .../ja/data/latin/letters.tsv | 52 ++++++ .../ja/data/measure/__init__.py | 13 ++ .../ja/data/measure/per_unit.tsv | 33 ++++ .../ja/data/measure/rate_numerator.tsv | 4 + .../ja/data/measure/unit.tsv | 79 +++++++++ .../ja/data/money/__init__.py | 1 + .../ja/data/money/currency_major.tsv | 27 +++ .../ja/data/money/currency_minor.tsv | 2 + .../ja/data/money/currency_minor_by_major.tsv | 15 ++ .../ja/data/money/currency_prefix.tsv | 19 +++ .../ja/data/money/quantity.tsv | 3 + .../ja/data/numbers/zero.tsv | 2 +- .../ja/data/numbers/zero_decimal.tsv | 1 + .../ja/data/roman/__init__.py | 13 ++ .../ja/data/roman/key_cardinal.tsv | 4 + .../ja/data/roman/key_ordinal.tsv | 2 + .../ja/data/roman/preserve.tsv | 1 + .../ja/data/roman/roman_numerals.tsv | 13 ++ .../ja/data/serial/__init__.py | 13 ++ .../ja/data/serial/words.tsv | 3 + .../text_normalization/ja/data/whitelist.tsv | 8 +- .../ja/data/whitelist_title.tsv | 10 ++ .../text_normalization/ja/taggers/address.py | 66 +++++++ .../text_normalization/ja/taggers/decimal.py | 17 +- .../ja/taggers/electronic.py | 132 ++++++++++++++ .../text_normalization/ja/taggers/fraction.py | 9 +- .../text_normalization/ja/taggers/measure.py | 160 +++++++++++++++++ .../text_normalization/ja/taggers/money.py | 161 ++++++++++++++++++ .../ja/taggers/punctuation.py | 12 +- .../text_normalization/ja/taggers/roman.py | 67 ++++++++ .../text_normalization/ja/taggers/serial.py | 88 ++++++++++ .../ja/taggers/telephone.py | 121 +++++++++++++ .../text_normalization/ja/taggers/time.py | 3 +- .../ja/taggers/tokenize_and_classify.py | 37 +++- .../ja/taggers/whitelist.py | 20 ++- .../text_normalization/ja/taggers/word.py | 60 +++---- .../ja/verbalizers/fraction.py | 12 +- .../ja/verbalizers/measure.py | 50 ++++++ .../ja/verbalizers/money.py | 64 +++++++ .../ja/verbalizers/post_processing.py | 131 ++++++++------ .../ja/verbalizers/postprocessor.py | 63 +++---- .../ja/verbalizers/telephone.py | 63 +++++++ .../ja/verbalizers/verbalize.py | 12 +- .../ja/verbalizers/verbalize_final.py | 2 +- .../ja/verbalizers/whitelist.py | 14 +- .../test_cases_decimal.txt | 64 +++---- .../test_cases_fraction.txt | 66 +++---- .../test_cases_time.txt | 80 ++++----- .../test_cases_address.txt | 9 + .../test_cases_cardinal.txt | 2 +- .../test_cases_date.txt | 1 - .../test_cases_decimal.txt | 2 +- .../test_cases_electronic.txt | 36 ++++ .../test_cases_fraction.txt | 11 +- .../test_cases_measure.txt | 34 ++++ .../test_cases_money.txt | 38 +++++ .../test_cases_roman.txt | 12 ++ .../test_cases_serial.txt | 15 ++ .../test_cases_telephone.txt | 12 ++ .../test_cases_telephone_extended.txt | 18 ++ .../test_cases_time.txt | 2 +- .../test_cases_whitelist.txt | 14 ++ tests/nemo_text_processing/ja/test_address.py | 30 ++++ tests/nemo_text_processing/ja/test_date.py | 2 +- .../ja/test_electronic.py | 30 ++++ tests/nemo_text_processing/ja/test_measure.py | 33 ++++ tests/nemo_text_processing/ja/test_money.py | 32 ++++ tests/nemo_text_processing/ja/test_range.py | 34 ++++ tests/nemo_text_processing/ja/test_roman.py | 30 ++++ tests/nemo_text_processing/ja/test_serial.py | 30 ++++ .../ja/test_sparrowhawk_normalization.sh | 36 ++++ .../nemo_text_processing/ja/test_telephone.py | 44 +++++ .../ja/test_telephone_extended.py | 31 ++++ .../nemo_text_processing/ja/test_whitelist.py | 31 ++++ 83 files changed, 2204 insertions(+), 304 deletions(-) create mode 100644 nemo_text_processing/text_normalization/ja/data/electronic/__init__.py create mode 100644 nemo_text_processing/text_normalization/ja/data/electronic/card_cues.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/electronic/file_extensions.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/electronic/special_digit_runs.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/electronic/symbol.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/latin/__init__.py create mode 100644 nemo_text_processing/text_normalization/ja/data/latin/letters.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/measure/__init__.py create mode 100644 nemo_text_processing/text_normalization/ja/data/measure/per_unit.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/measure/rate_numerator.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/measure/unit.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/money/__init__.py create mode 100644 nemo_text_processing/text_normalization/ja/data/money/currency_major.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/money/currency_minor.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/money/currency_minor_by_major.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/money/currency_prefix.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/money/quantity.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/numbers/zero_decimal.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/roman/__init__.py create mode 100644 nemo_text_processing/text_normalization/ja/data/roman/key_cardinal.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/roman/key_ordinal.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/roman/preserve.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/roman/roman_numerals.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/serial/__init__.py create mode 100644 nemo_text_processing/text_normalization/ja/data/serial/words.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/whitelist_title.tsv create mode 100644 nemo_text_processing/text_normalization/ja/taggers/address.py create mode 100644 nemo_text_processing/text_normalization/ja/taggers/electronic.py create mode 100644 nemo_text_processing/text_normalization/ja/taggers/measure.py create mode 100644 nemo_text_processing/text_normalization/ja/taggers/money.py create mode 100644 nemo_text_processing/text_normalization/ja/taggers/roman.py create mode 100644 nemo_text_processing/text_normalization/ja/taggers/serial.py create mode 100644 nemo_text_processing/text_normalization/ja/taggers/telephone.py create mode 100644 nemo_text_processing/text_normalization/ja/verbalizers/measure.py create mode 100644 nemo_text_processing/text_normalization/ja/verbalizers/money.py create mode 100644 nemo_text_processing/text_normalization/ja/verbalizers/telephone.py create mode 100644 tests/nemo_text_processing/ja/data_text_normalization/test_cases_address.txt create mode 100644 tests/nemo_text_processing/ja/data_text_normalization/test_cases_electronic.txt create mode 100644 tests/nemo_text_processing/ja/data_text_normalization/test_cases_measure.txt create mode 100644 tests/nemo_text_processing/ja/data_text_normalization/test_cases_money.txt create mode 100644 tests/nemo_text_processing/ja/data_text_normalization/test_cases_roman.txt create mode 100644 tests/nemo_text_processing/ja/data_text_normalization/test_cases_serial.txt create mode 100644 tests/nemo_text_processing/ja/data_text_normalization/test_cases_telephone.txt create mode 100644 tests/nemo_text_processing/ja/data_text_normalization/test_cases_telephone_extended.txt create mode 100644 tests/nemo_text_processing/ja/data_text_normalization/test_cases_whitelist.txt create mode 100644 tests/nemo_text_processing/ja/test_address.py create mode 100644 tests/nemo_text_processing/ja/test_electronic.py create mode 100644 tests/nemo_text_processing/ja/test_measure.py create mode 100644 tests/nemo_text_processing/ja/test_money.py create mode 100644 tests/nemo_text_processing/ja/test_range.py create mode 100644 tests/nemo_text_processing/ja/test_roman.py create mode 100644 tests/nemo_text_processing/ja/test_serial.py create mode 100644 tests/nemo_text_processing/ja/test_telephone.py create mode 100644 tests/nemo_text_processing/ja/test_telephone_extended.py create mode 100644 tests/nemo_text_processing/ja/test_whitelist.py diff --git a/nemo_text_processing/inverse_text_normalization/ja/data/numbers/zero.tsv b/nemo_text_processing/inverse_text_normalization/ja/data/numbers/zero.tsv index 147ef1da5..1923adb60 100644 --- a/nemo_text_processing/inverse_text_normalization/ja/data/numbers/zero.tsv +++ b/nemo_text_processing/inverse_text_normalization/ja/data/numbers/zero.tsv @@ -1,5 +1,4 @@ ゼロ 0 -零 0 れい 0 ○ 0 〇 0 diff --git a/nemo_text_processing/inverse_text_normalization/ja/data/time_hours.tsv b/nemo_text_processing/inverse_text_normalization/ja/data/time_hours.tsv index 8a5eced19..1957b95ad 100644 --- a/nemo_text_processing/inverse_text_normalization/ja/data/time_hours.tsv +++ b/nemo_text_processing/inverse_text_normalization/ja/data/time_hours.tsv @@ -22,4 +22,4 @@ 二十二 22 二十三 23 二十四 24 -零 0 \ No newline at end of file +ゼロ 0 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/ja/taggers/fraction.py b/nemo_text_processing/inverse_text_normalization/ja/taggers/fraction.py index 0ced0c679..10882ae99 100644 --- a/nemo_text_processing/inverse_text_normalization/ja/taggers/fraction.py +++ b/nemo_text_processing/inverse_text_normalization/ja/taggers/fraction.py @@ -26,7 +26,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst): e.g., 四分の三 -> fraction { denominator: "4" numerator: "3" } 一と四分の三 -> fraction { integer: "1" denominator: "4" numerator: "3" } - 一荷四分の三 -> fraction { integer: "1" denominator: "4" numerator: "3" } + 一と四分の三 -> fraction { integer: "1" denominator: "4" numerator: "3" } ルート三分の一 -> fraction { denominator: "√3" numerator: "1" } 一点六五分の五十 -> fraction { denominator: "1.65" numerator: "50" } 二ルート六分の三 -> -> fraction { denominator: "2√6 " numerator: "3" } @@ -40,7 +40,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst): pynutil.delete("分の") | pynutil.delete(" 分 の　") | pynutil.delete("分 の　") | pynutil.delete("分 の") ) - integer_word = pynutil.delete("と") | pynutil.delete("荷") + integer_word = pynutil.delete("と") root_word = pynini.accep("√") | pynini.cross("ルート", "√") graph_sign = ( diff --git a/nemo_text_processing/text_normalization/ja/data/electronic/__init__.py b/nemo_text_processing/text_normalization/ja/data/electronic/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/electronic/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ja/data/electronic/card_cues.tsv b/nemo_text_processing/text_normalization/ja/data/electronic/card_cues.tsv new file mode 100644 index 000000000..769dee97d --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/electronic/card_cues.tsv @@ -0,0 +1,2 @@ +カード末尾 +カード番号 diff --git a/nemo_text_processing/text_normalization/ja/data/electronic/file_extensions.tsv b/nemo_text_processing/text_normalization/ja/data/electronic/file_extensions.tsv new file mode 100644 index 000000000..96c957484 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/electronic/file_extensions.tsv @@ -0,0 +1,6 @@ +.jpg ドット jpg +.JPG ドット JPG +.png ドット png +.PNG ドット PNG +.pdf ドット pdf +.PDF ドット PDF diff --git a/nemo_text_processing/text_normalization/ja/data/electronic/special_digit_runs.tsv b/nemo_text_processing/text_normalization/ja/data/electronic/special_digit_runs.tsv new file mode 100644 index 000000000..8609135f3 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/electronic/special_digit_runs.tsv @@ -0,0 +1 @@ +01 ゼロイチ diff --git a/nemo_text_processing/text_normalization/ja/data/electronic/symbol.tsv b/nemo_text_processing/text_normalization/ja/data/electronic/symbol.tsv new file mode 100644 index 000000000..568588980 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/electronic/symbol.tsv @@ -0,0 +1,5 @@ +@ アット +. ドット +- ハイフン +/ スラッシュ +: コロン diff --git a/nemo_text_processing/text_normalization/ja/data/latin/__init__.py b/nemo_text_processing/text_normalization/ja/data/latin/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/latin/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ja/data/latin/letters.tsv b/nemo_text_processing/text_normalization/ja/data/latin/letters.tsv new file mode 100644 index 000000000..a5ca20a02 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/latin/letters.tsv @@ -0,0 +1,52 @@ +A エー +B ビー +C シー +D ディー +E イー +F エフ +G ジー +H エイチ +I アイ +J ジェー +K ケー +L エル +M エム +N エヌ +O オー +P ピー +Q キュー +R アール +S エス +T ティー +U ユー +V ブイ +W ダブリュー +X エックス +Y ワイ +Z ゼット +a エー +b ビー +c シー +d ディー +e イー +f エフ +g ジー +h エイチ +i アイ +j ジェー +k ケー +l エル +m エム +n エヌ +o オー +p ピー +q キュー +r アール +s エス +t ティー +u ユー +v ブイ +w ダブリュー +x エックス +y ワイ +z ゼット diff --git a/nemo_text_processing/text_normalization/ja/data/measure/__init__.py b/nemo_text_processing/text_normalization/ja/data/measure/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/measure/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ja/data/measure/per_unit.tsv b/nemo_text_processing/text_normalization/ja/data/measure/per_unit.tsv new file mode 100644 index 000000000..ed878a1bf --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/measure/per_unit.tsv @@ -0,0 +1,33 @@ +h 時 +ｈ 時 +hour 時 +hours 時 +s 秒 +ｓ 秒 +sec 秒 +second 秒 +seconds 秒 +min 分 +ｍｉｎ 分 +minute 分 +minutes 分 +kg キロ +ｋｇ キロ +g グラム +ｇ グラム +m メートル +ｍ メートル +km キロ +ｋｍ キロ +cm センチメートル +ｃｍ センチメートル +mm ミリメートル +ｍｍ ミリメートル +L リットル +Ｌ リットル +l リットル +ｌ リットル +ml ミリリットル +mL ミリリットル +ｍｌ ミリリットル +ｍＬ ミリリットル diff --git a/nemo_text_processing/text_normalization/ja/data/measure/rate_numerator.tsv b/nemo_text_processing/text_normalization/ja/data/measure/rate_numerator.tsv new file mode 100644 index 000000000..126b4a3e1 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/measure/rate_numerator.tsv @@ -0,0 +1,4 @@ +km キロ +ｋｍ キロ +m メートル +ｍ メートル diff --git a/nemo_text_processing/text_normalization/ja/data/measure/unit.tsv b/nemo_text_processing/text_normalization/ja/data/measure/unit.tsv new file mode 100644 index 000000000..0573068f8 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/measure/unit.tsv @@ -0,0 +1,79 @@ +kg キロ +ｋｇ キロ +キロ キロ +キログラム キログラム +g グラム +ｇ グラム +グラム グラム +mg ミリグラム +ｍｇ ミリグラム +ミリグラム ミリグラム +μg マイクログラム +µg マイクログラム +マイクログラム マイクログラム +km キロ +ｋｍ キロ +キロメートル キロ +m メートル +ｍ メートル +メートル メートル +cm センチ +ｃｍ センチ +センチ センチ +センチメートル センチメートル +mm ミリメートル +ｍｍ ミリメートル +ミリメートル ミリメートル +m2 平方メートル +m² 平方メートル +ｍ2 平方メートル +ｍ² 平方メートル +平方メートル 平方メートル +cm2 平方センチメートル +cm² 平方センチメートル +平方センチメートル 平方センチメートル +L リットル +l リットル +Ｌ リットル +ｌ リットル +リットル リットル +mL ミリリットル +ml ミリリットル +ｍＬ ミリリットル +ｍｌ ミリリットル +ミリリットル ミリリットル +% パーセント +％ パーセント +パーセント パーセント +° 度 +º 度 +度 度 +°C 度 +℃ 度 +ºC 度 +°F 度エフ +℉ 度エフ +W ワット +w ワット +ワット ワット +kW キロワット +kw キロワット +キロワット キロワット +V ボルト +v ボルト +ボルト ボルト +Hz ヘルツ +hz ヘルツ +ヘルツ ヘルツ +kHz キロヘルツ +khz キロヘルツ +MHz メガヘルツ +mhz メガヘルツ +GHz ギガヘルツ +ghz ギガヘルツ +GB ギガバイト +gb ギガバイト +ギガバイト ギガバイト +MB メガバイト +mb メガバイト +メガバイト メガバイト diff --git a/nemo_text_processing/text_normalization/ja/data/money/__init__.py b/nemo_text_processing/text_normalization/ja/data/money/__init__.py new file mode 100644 index 000000000..3e4afde2e --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/money/__init__.py @@ -0,0 +1 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. diff --git a/nemo_text_processing/text_normalization/ja/data/money/currency_major.tsv b/nemo_text_processing/text_normalization/ja/data/money/currency_major.tsv new file mode 100644 index 000000000..c5f2efa84 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/money/currency_major.tsv @@ -0,0 +1,27 @@ +¥ 円 +￥ 円 +JPY 円 +jpy 円 +円 円 +$ ドル +US$ ドル +USD ドル +usd ドル +ドル ドル +米ドル ドル +€ ユーロ +EUR ユーロ +eur ユーロ +ユーロ ユーロ +£ ポンド +GBP ポンド +gbp ポンド +ポンド ポンド +₩ ウォン +KRW ウォン +krw ウォン +ウォン ウォン +CNY 元 +cny 元 +元 元 +人民元 人民元 diff --git a/nemo_text_processing/text_normalization/ja/data/money/currency_minor.tsv b/nemo_text_processing/text_normalization/ja/data/money/currency_minor.tsv new file mode 100644 index 000000000..3709c9572 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/money/currency_minor.tsv @@ -0,0 +1,2 @@ +銭 銭 +セント セント diff --git a/nemo_text_processing/text_normalization/ja/data/money/currency_minor_by_major.tsv b/nemo_text_processing/text_normalization/ja/data/money/currency_minor_by_major.tsv new file mode 100644 index 000000000..bdb9c4d07 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/money/currency_minor_by_major.tsv @@ -0,0 +1,15 @@ +¥ 銭 +￥ 銭 +JPY 銭 +jpy 銭 +円 銭 +$ セント +US$ セント +USD セント +usd セント +ドル セント +米ドル セント +€ セント +EUR セント +eur セント +ユーロ セント diff --git a/nemo_text_processing/text_normalization/ja/data/money/currency_prefix.tsv b/nemo_text_processing/text_normalization/ja/data/money/currency_prefix.tsv new file mode 100644 index 000000000..c959bf0ca --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/money/currency_prefix.tsv @@ -0,0 +1,19 @@ +¥ 円 +￥ 円 +JPY 円 +jpy 円 +$ ドル +US$ ドル +USD ドル +usd ドル +€ ユーロ +EUR ユーロ +eur ユーロ +£ ポンド +GBP ポンド +gbp ポンド +₩ ウォン +KRW ウォン +krw ウォン +CNY 元 +cny 元 diff --git a/nemo_text_processing/text_normalization/ja/data/money/quantity.tsv b/nemo_text_processing/text_normalization/ja/data/money/quantity.tsv new file mode 100644 index 000000000..68677c4c1 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/money/quantity.tsv @@ -0,0 +1,3 @@ +万 万 +億 億 +兆 兆 diff --git a/nemo_text_processing/text_normalization/ja/data/numbers/zero.tsv b/nemo_text_processing/text_normalization/ja/data/numbers/zero.tsv index 7fab21718..ac94cc719 100644 --- a/nemo_text_processing/text_normalization/ja/data/numbers/zero.tsv +++ b/nemo_text_processing/text_normalization/ja/data/numbers/zero.tsv @@ -1 +1 @@ -0 零 \ No newline at end of file +0 ゼロ diff --git a/nemo_text_processing/text_normalization/ja/data/numbers/zero_decimal.tsv b/nemo_text_processing/text_normalization/ja/data/numbers/zero_decimal.tsv new file mode 100644 index 000000000..d6b9cece5 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/numbers/zero_decimal.tsv @@ -0,0 +1 @@ +0 零 diff --git a/nemo_text_processing/text_normalization/ja/data/roman/__init__.py b/nemo_text_processing/text_normalization/ja/data/roman/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/roman/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ja/data/roman/key_cardinal.tsv b/nemo_text_processing/text_normalization/ja/data/roman/key_cardinal.tsv new file mode 100644 index 000000000..661279d14 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/roman/key_cardinal.tsv @@ -0,0 +1,4 @@ +Chapter +chapter +Part +Super Bowl diff --git a/nemo_text_processing/text_normalization/ja/data/roman/key_ordinal.tsv b/nemo_text_processing/text_normalization/ja/data/roman/key_ordinal.tsv new file mode 100644 index 000000000..12cecc1cc --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/roman/key_ordinal.tsv @@ -0,0 +1,2 @@ +Henry +Louis diff --git a/nemo_text_processing/text_normalization/ja/data/roman/preserve.tsv b/nemo_text_processing/text_normalization/ja/data/roman/preserve.tsv new file mode 100644 index 000000000..5d5d106b0 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/roman/preserve.tsv @@ -0,0 +1 @@ +World War II World War II diff --git a/nemo_text_processing/text_normalization/ja/data/roman/roman_numerals.tsv b/nemo_text_processing/text_normalization/ja/data/roman/roman_numerals.tsv new file mode 100644 index 000000000..f443fd3a9 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/roman/roman_numerals.tsv @@ -0,0 +1,13 @@ +I 1 +V 5 +X 10 +L 50 +C 100 +D 500 +M 1000 +IV 4 +IX 9 +XL 40 +XC 90 +CD 400 +CM 900 diff --git a/nemo_text_processing/text_normalization/ja/data/serial/__init__.py b/nemo_text_processing/text_normalization/ja/data/serial/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/serial/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ja/data/serial/words.tsv b/nemo_text_processing/text_normalization/ja/data/serial/words.tsv new file mode 100644 index 000000000..87725d259 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/serial/words.tsv @@ -0,0 +1,3 @@ +covid コビッド +COVID コビッド +Covid コビッド diff --git a/nemo_text_processing/text_normalization/ja/data/whitelist.tsv b/nemo_text_processing/text_normalization/ja/data/whitelist.tsv index d0d7bef70..c45764921 100644 --- a/nemo_text_processing/text_normalization/ja/data/whitelist.tsv +++ b/nemo_text_processing/text_normalization/ja/data/whitelist.tsv @@ -4,10 +4,10 @@ Mr. ミスター mr. ミスター Ms. ミス ms. ミス -Mrs. ミシーズ -mrs. ミシーず -st. ストリード -St. ストリード +Mrs. ミセス +mrs. ミセス +st. ストリート +St. ストリート mt. マウント Mt. マウント Prof. プロフェッサー diff --git a/nemo_text_processing/text_normalization/ja/data/whitelist_title.tsv b/nemo_text_processing/text_normalization/ja/data/whitelist_title.tsv new file mode 100644 index 000000000..649cabe0d --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/whitelist_title.tsv @@ -0,0 +1,10 @@ +Dr. ドクター +dr. ドクター +Mr. ミスター +mr. ミスター +Mrs. ミセス +mrs. ミセス +Ms. ミス +ms. ミス +Prof. プロフェッサー +prof. プロフェッサー diff --git a/nemo_text_processing/text_normalization/ja/taggers/address.py b/nemo_text_processing/text_normalization/ja/taggers/address.py new file mode 100644 index 000000000..2e0c28b2b --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/taggers/address.py @@ -0,0 +1,66 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_DIGIT, NEMO_NOT_SPACE, GraphFst +from nemo_text_processing.text_normalization.ja.utils import get_abs_path + + +class AddressFst(GraphFst): + """ + Finite state transducer for classifying Japanese address-like expressions. + + Examples: + 東京都千代田区丸の内1-1-1 -> name: "東京都千代田区丸の内一の一の一" + 503号室 -> name: "五〇三号室" + 〒100-0001 -> name: "郵便番号一ゼロゼロのゼロゼロゼロ一" + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="address", kind="classify", deterministic=deterministic) + + digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + zero_maru = pynini.cross("0", "〇") + + address_number = cardinal.just_cardinals + digit_for_room = digit | zero_maru + digit_for_postal = digit | zero + + hyphen_to_no = pynini.closure(pynutil.delete(" "), 0, 1) + ( + pynutil.delete("-") | pynutil.delete("－") | pynutil.delete("ー") + ) + pynini.closure(pynutil.delete(" "), 0, 1) + pynutil.insert("の") + + address_chain = address_number + hyphen_to_no + address_number + hyphen_to_no + address_number + + address_prefix_char = pynini.difference( + NEMO_NOT_SPACE, + NEMO_DIGIT | pynini.union("-", "－", "ー"), + ) + address_with_prefix = pynini.closure(address_prefix_char, 1) + address_chain + + postal_code = ( + pynutil.delete("〒") + + pynutil.insert("郵便番号") + + digit_for_postal**3 + + hyphen_to_no + + digit_for_postal**4 + ) + + room = (NEMO_DIGIT**3 @ (digit_for_room**3)) + pynini.accep("号室") + + graph = address_with_prefix | postal_code | room + self.fst = (pynutil.insert('name: "') + graph + pynutil.insert('"')).optimize() diff --git a/nemo_text_processing/text_normalization/ja/taggers/decimal.py b/nemo_text_processing/text_normalization/ja/taggers/decimal.py index 8fdea4c87..149345d4f 100644 --- a/nemo_text_processing/text_normalization/ja/taggers/decimal.py +++ b/nemo_text_processing/text_normalization/ja/taggers/decimal.py @@ -16,7 +16,7 @@ import pynini from pynini.lib import pynutil -from nemo_text_processing.text_normalization.ja.graph_utils import GraphFst +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_DIGIT, GraphFst from nemo_text_processing.text_normalization.ja.utils import get_abs_path @@ -24,6 +24,7 @@ class DecimalFst(GraphFst): """ Finite state transducer for classifying decimal, e.g. 0.5 -> decimal { integer_part: "零" fractional_part: "五" } + 0.05 -> decimal { integer_part: "零" fractional_part: "零五" } -0.5万 -> decimal { negative: "マイナス" integer_part: "零" fractional_part: "五" quantity: "万"} Args: @@ -35,12 +36,20 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): cardinal_before_decimal = cardinal.just_cardinals cardinal_after_decimal = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) - zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + zero_decimal = pynini.string_file(get_abs_path("data/numbers/zero_decimal.tsv")) - graph_integer = pynutil.insert('integer_part: \"') + cardinal_before_decimal + pynutil.insert("\"") + graph_integer = ( + pynutil.insert('integer_part: \"') + + ( + pynini.cross("0", "零") + | (pynini.difference(NEMO_DIGIT, "0") @ cardinal_before_decimal) + | (pynini.closure(NEMO_DIGIT, 2) @ cardinal_before_decimal) + ) + + pynutil.insert("\"") + ) graph_fraction = ( pynutil.insert("fractional_part: \"") - + pynini.closure((cardinal_after_decimal | zero), 1) + + pynini.closure((cardinal_after_decimal | zero_decimal), 1) + pynutil.insert("\"") ) graph_decimal_no_sign = graph_integer + pynutil.delete('.') + pynutil.insert(" ") + graph_fraction diff --git a/nemo_text_processing/text_normalization/ja/taggers/electronic.py b/nemo_text_processing/text_normalization/ja/taggers/electronic.py new file mode 100644 index 000000000..e767a83db --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/taggers/electronic.py @@ -0,0 +1,132 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_NOT_SPACE, + NEMO_SIGMA, + GraphFst, + delete_space, + insert_space, +) +from nemo_text_processing.text_normalization.ja.utils import get_abs_path + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for classifying Japanese electronic expressions. + + Examples: + abc@abc.com -> name: "abc アット abc ドット com" + https://www.nvidia.com -> name: "https コロン スラッシュ スラッシュ www ドット nvidia ドット com" + 1234-5678-9012-3456 -> name: "一二三四 五六七八 九〇一二 三四五六" + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="electronic", kind="classify", deterministic=deterministic) + + digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + digit_zero_maru = digit | pynini.cross("0", "〇") + digit_zero_user = digit | pynini.cross("0", "ゼロ") + + special_digit_run = pynutil.add_weight( + pynini.string_file(get_abs_path("data/electronic/special_digit_runs.tsv")), + -0.1, + ) + symbol = pynini.string_file(get_abs_path("data/electronic/symbol.tsv")) + at = symbol @ pynini.cross("アット", " アット ") + dot = symbol @ pynini.cross("ドット", " ドット ") + hyphen = symbol @ pynini.cross("ハイフン", " ハイフン ") + slash = symbol @ pynini.cross("スラッシュ", " スラッシュ ") + colon = symbol @ pynini.cross("コロン", " コロン ") + + raw_label = pynini.closure(NEMO_ALPHA | NEMO_DIGIT, 1) + alpha_label = pynini.closure(NEMO_ALPHA, 1) + digit_label = pynini.closure(NEMO_DIGIT, 1) + raw_label_with_hyphen = raw_label + pynini.closure(pynutil.delete("-") + pynutil.insert(" ハイフン ") + raw_label) + + insert_alpha_digit_space = pynini.cdrewrite(pynutil.insert(" "), NEMO_ALPHA, NEMO_DIGIT, NEMO_SIGMA) + insert_digit_alpha_space = pynini.cdrewrite(pynutil.insert(" "), NEMO_DIGIT, NEMO_ALPHA, NEMO_SIGMA) + alnum_spacing = insert_alpha_digit_space @ insert_digit_alpha_space + username_reader = pynini.closure(NEMO_ALPHA | special_digit_run | digit_zero_user | pynini.accep(" "), 1) + + raw_alnum = pynini.closure(NEMO_ALPHA | NEMO_DIGIT, 1) + raw_mixed_alnum = ( + pynini.closure(NEMO_ALPHA | NEMO_DIGIT) + + NEMO_ALPHA + + pynini.closure(NEMO_ALPHA | NEMO_DIGIT) + + NEMO_DIGIT + + pynini.closure(NEMO_ALPHA | NEMO_DIGIT) + ) | ( + pynini.closure(NEMO_ALPHA | NEMO_DIGIT) + + NEMO_DIGIT + + pynini.closure(NEMO_ALPHA | NEMO_DIGIT) + + NEMO_ALPHA + + pynini.closure(NEMO_ALPHA | NEMO_DIGIT) + ) + username_alnum = (raw_mixed_alnum @ alnum_spacing @ username_reader).optimize() + username_segment = pynutil.add_weight(username_alnum, -0.1) | alpha_label | digit_label + username = username_segment + pynini.closure((dot | hyphen) + username_segment) + + domain = raw_label + pynini.closure(dot + raw_label) + dot + alpha_label + email = username + at + domain + + protocol = (pynini.accep("https") | pynini.accep("http")) + colon + slash + slash + path = slash + raw_label_with_hyphen + pynini.closure(slash + raw_label_with_hyphen) + url = protocol + domain + pynini.closure(path, 0, 1) + + four_digits = NEMO_DIGIT**4 @ (digit_zero_maru**4) + card_separator = (pynutil.delete("-") | pynutil.delete(" ")) + insert_space + credit_card = four_digits + card_separator + four_digits + card_separator + four_digits + card_separator + four_digits + + card_cue = pynini.string_file(get_abs_path("data/electronic/card_cues.tsv")) + card_with_cue = card_cue + credit_card + card_tail_with_cue = card_cue + four_digits + card_tail = pynini.accep("カード下") + (NEMO_DIGIT @ cardinal.just_cardinals) + pynini.accep("桁") + ( + NEMO_DIGIT**4 @ (digit_zero_maru**4) + ) + + extension = pynini.string_file(get_abs_path("data/electronic/file_extensions.tsv")) + filename_stem = pynini.closure( + pynini.difference(NEMO_NOT_SPACE, pynini.union(".", "/", "@")), + 1, + ) + filename = filename_stem + insert_space + extension + email_with_context = ( + pynini.accep("email") + + delete_space + + insert_space + + email + + delete_space + + insert_space + + pynini.accep("です") + ) + + graph = ( + pynutil.add_weight(credit_card, -0.1) + | pynutil.add_weight(card_with_cue, -0.1) + | pynutil.add_weight(card_tail_with_cue, -0.1) + | card_tail + | email_with_context + | email + | url + | domain + | filename + ) + + self.fst = (pynutil.insert('name: "') + graph.optimize() + pynutil.insert('"')).optimize() diff --git a/nemo_text_processing/text_normalization/ja/taggers/fraction.py b/nemo_text_processing/text_normalization/ja/taggers/fraction.py index 94fb4af68..ca2110296 100644 --- a/nemo_text_processing/text_normalization/ja/taggers/fraction.py +++ b/nemo_text_processing/text_normalization/ja/taggers/fraction.py @@ -25,7 +25,7 @@ class FractionFst(GraphFst): Finite state transducer for classifying fractions, e.g. 1/2 -> tokens { fraction { denominator: "二" numerator: "一"} } 1と3/4 -> fraction { integer: "一" denominator: "四" numerator: "三" } - 一荷四分の三 -> fraction { integer: "1" denominator: "4" numerator: "3" } + 一と四分の三 -> fraction { integer: "1" denominator: "4" numerator: "3" } ルート三分の一 -> fraction { denominator: "√3" numerator: "1" } 一点六五分の五十 -> fraction { denominator: "1.65" numerator: "50" } マイナス1/2 -> tokens { fraction { denominator: "二" numerator: "一"} } @@ -60,7 +60,7 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): pynutil.insert('integer_part: \"') + ( (cardinal | (root + cardinal) | decimal_number | (root + decimal_number)) - + (pynini.accep("と") | pynini.accep("荷")) + + pynini.accep("と") ) + pynutil.insert("\"") + pynutil.insert(NEMO_SPACE) @@ -115,9 +115,8 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): + pynutil.insert("\"") ) - graph_fraction_slash_sigh = pynini.closure(optional_sign + pynutil.insert(NEMO_SPACE), 0, 1) + ( - graph_fraction_slash | graph_fraction_word - ) + self.graph = (graph_fraction_slash | graph_fraction_word).optimize() + graph_fraction_slash_sigh = pynini.closure(optional_sign + pynutil.insert(NEMO_SPACE), 0, 1) + self.graph graph = graph_fraction_slash_sigh # | diff --git a/nemo_text_processing/text_normalization/ja/taggers/measure.py b/nemo_text_processing/text_normalization/ja/taggers/measure.py new file mode 100644 index 000000000..fa88e8c4c --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/taggers/measure.py @@ -0,0 +1,160 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import GraphFst, delete_space +from nemo_text_processing.text_normalization.ja.utils import get_abs_path, load_labels + + +class MeasureFst(GraphFst): + """ + Finite state transducer for classifying Japanese measure expressions. + + Examples: + 5kg -> measure { cardinal { integer: "五" } units: "キロ" preserve_order: true } + 0kg -> measure { cardinal { integer: "ゼロ" } units: "キロ" preserve_order: true } + 0.05m -> measure { decimal { integer_part: "零" fractional_part: "零五" } units: "メートル" preserve_order: true } + 60km/h -> measure { cardinal { integer: "時速六十" } units: "キロ" preserve_order: true } + 50m/s -> measure { cardinal { integer: "秒速五十" } units: "メートル" preserve_order: true } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + fraction: FractionFst + deterministic: if True provides a single transduction option + """ + + def __init__( + self, + cardinal: GraphFst, + decimal: GraphFst, + fraction: GraphFst, + deterministic: bool = True, + ): + super().__init__(name="measure", kind="classify", deterministic=deterministic) + + unit_path = get_abs_path("data/measure/unit.tsv") + per_unit_path = get_abs_path("data/measure/per_unit.tsv") + rate_numerator_path = get_abs_path("data/measure/rate_numerator.tsv") + unit = pynini.string_file(unit_path) + per_unit = pynini.string_file(per_unit_path) + rate_numerator = pynini.string_file(rate_numerator_path) + + slash = pynutil.delete("/") | pynutil.delete("／") + kilometer_rate_unit = rate_numerator @ pynini.cross("キロ", "") + meter_rate_unit = rate_numerator @ pynini.cross("メートル", "") + hour_per_unit = per_unit @ pynini.cross("時", "") + second_per_unit = per_unit @ pynini.cross("秒", "") + + speed_kmh_inputs = {written for written, spoken in load_labels(rate_numerator_path) if spoken == "キロ"} + speed_ms_inputs = {written for written, spoken in load_labels(rate_numerator_path) if spoken == "メートル"} + general_per_unit = pynini.Fst() + for unit_written, unit_spoken in load_labels(unit_path): + for per_written, per_spoken in load_labels(per_unit_path): + if (unit_written in speed_kmh_inputs and per_spoken == "時") or ( + unit_written in speed_ms_inputs and per_spoken == "秒" + ): + continue + general_per_unit |= ( + pynini.cross(unit_written, unit_spoken) + + delete_space + + slash + + delete_space + + pynutil.insert("毎") + + pynini.cross(per_written, per_spoken) + ) + unit_graph = unit | general_per_unit + + speed_kmh_unit = ( + delete_space + + kilometer_rate_unit + + delete_space + + slash + + delete_space + + hour_per_unit + + pynutil.insert(' units: "キロ"') + ) + speed_ms_unit = ( + delete_space + + meter_rate_unit + + delete_space + + slash + + delete_space + + second_per_unit + + pynutil.insert(' units: "メートル"') + ) + + unit_component = delete_space + pynutil.insert(' units: "') + unit_graph + pynutil.insert('"') + + optional_sign = ( + pynutil.insert('negative: "') + + (pynini.cross("-", "マイナス") | pynini.accep("マイナス")) + + pynutil.insert('" ') + + delete_space + ) + + cardinal_graph = ( + pynutil.insert("cardinal { ") + + pynini.closure(optional_sign, 0, 1) + + pynutil.insert('integer: "') + + cardinal.just_cardinals + + pynutil.insert('" }') + ) + decimal_graph = ( + pynutil.insert("decimal { ") + + pynini.closure(optional_sign, 0, 1) + + decimal.just_decimal + + pynutil.insert(" }") + ) + fraction_graph = ( + pynutil.insert("fraction { ") + + pynini.closure(optional_sign, 0, 1) + + fraction.graph + + pynutil.insert(" }") + ) + + speed_kmh_number = ( + pynutil.insert("cardinal { ") + + pynini.closure(optional_sign, 0, 1) + + pynutil.insert('integer: "時速') + + cardinal.just_cardinals + + pynutil.insert('" }') + ) + speed_ms_number = ( + pynutil.insert("cardinal { ") + + pynini.closure(optional_sign, 0, 1) + + pynutil.insert('integer: "秒速') + + cardinal.just_cardinals + + pynutil.insert('" }') + ) + + number = cardinal_graph | decimal_graph | fraction_graph + + speed_kmh_graph = ( + speed_kmh_number + + speed_kmh_unit + + pynutil.insert(" preserve_order: true") + ) + speed_ms_graph = ( + speed_ms_number + + speed_ms_unit + + pynutil.insert(" preserve_order: true") + ) + general_graph = number + unit_component + pynutil.insert(" preserve_order: true") + + graph = speed_kmh_graph | speed_ms_graph | general_graph + + self.fst = self.add_tokens(graph.optimize()).optimize() diff --git a/nemo_text_processing/text_normalization/ja/taggers/money.py b/nemo_text_processing/text_normalization/ja/taggers/money.py new file mode 100644 index 000000000..0024b48e5 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/taggers/money.py @@ -0,0 +1,161 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_DIGIT, GraphFst, delete_space +from nemo_text_processing.text_normalization.ja.utils import get_abs_path, load_labels + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying Japanese money expressions. + + Examples: + 100円 -> money { integer_part: "百" currency_maj: "円" preserve_order: true } + ¥3万 -> money { integer_part: "三" quantity: "万" currency_maj: "円" preserve_order: true } + 1.5万円 -> money { integer_part: "一点五" quantity: "万" currency_maj: "円" preserve_order: true } + 5ドル25セント -> money { integer_part: "五" currency_maj: "ドル" fractional_part: "二十五" currency_min: "セント" preserve_order: true } + $12.50 -> money { integer_part: "十二" currency_maj: "ドル" fractional_part: "五十" currency_min: "セント" preserve_order: true } + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="classify", deterministic=deterministic) + + graph_cardinal = cardinal.just_cardinals + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + + integer_input = (NEMO_DIGIT + pynini.closure(NEMO_DIGIT | pynutil.delete(","))) @ graph_cardinal + fractional_digits = pynini.closure(graph_digit | graph_zero, 1) + decimal_input = integer_input + pynutil.delete(".") + pynutil.insert("点") + fractional_digits + sign_input = (pynini.cross("-", "マイナス") | pynini.accep("マイナス")) + delete_space + + integer_component = pynutil.insert('integer_part: "') + integer_input + pynutil.insert('"') + decimal_component = pynutil.insert('integer_part: "') + decimal_input + pynutil.insert('"') + signed_integer_component = ( + pynutil.insert('integer_part: "') + pynini.closure(sign_input, 0, 1) + integer_input + pynutil.insert('"') + ) + signed_decimal_component = ( + pynutil.insert('integer_part: "') + pynini.closure(sign_input, 0, 1) + decimal_input + pynutil.insert('"') + ) + + number_component = signed_decimal_component | signed_integer_component + + quantity = pynini.string_file(get_abs_path("data/money/quantity.tsv")) + quantity_component = delete_space + pynutil.insert(' quantity: "') + quantity + pynutil.insert('"') + + currency_major_labels = load_labels(get_abs_path("data/money/currency_major.tsv")) + currency_major = pynini.string_file(get_abs_path("data/money/currency_major.tsv")) + currency_major_component = ( + delete_space + pynutil.insert(' currency_maj: "') + currency_major + pynutil.insert('"') + ) + + currency_prefix_labels = load_labels(get_abs_path("data/money/currency_prefix.tsv")) + currency_minor_by_major = dict(load_labels(get_abs_path("data/money/currency_minor_by_major.tsv"))) + currency_minor = pynini.string_file(get_abs_path("data/money/currency_minor.tsv")) + non_zero_digit = pynini.difference(NEMO_DIGIT, "0") + minor_decimal_input = ( + (NEMO_DIGIT**2 @ graph_cardinal) | (pynutil.delete("0") + (non_zero_digit @ graph_digit)) + ) + + suffix_graph = ( + number_component + + pynini.closure(quantity_component, 0, 1) + + currency_major_component + ) + for written, spoken in currency_major_labels: + minor_spoken = currency_minor_by_major.get(written) + if not minor_spoken: + continue + + currency_major_suffix = ( + delete_space + + pynutil.delete(written) + + pynutil.insert(f' currency_maj: "{spoken}"') + ) + minor_suffix = ( + delete_space + + pynutil.insert(' fractional_part: "') + + integer_input + + pynutil.insert('"') + + delete_space + + (currency_minor @ pynini.cross(minor_spoken, "")) + + pynutil.insert(f' currency_min: "{minor_spoken}"') + ) + suffix_graph |= ( + signed_integer_component + + currency_major_suffix + + minor_suffix + ) + + prefix_graph = pynini.Fst() + for written, spoken in currency_prefix_labels: + currency_prefix = pynutil.delete(written) + delete_space + currency_field = pynutil.insert(f' currency_maj: "{spoken}"') + minor_spoken = currency_minor_by_major.get(written) + + prefix_graph |= ( + currency_prefix + + number_component + + pynini.closure(quantity_component, 0, 1) + + currency_field + ) + prefix_graph |= ( + pynutil.insert('integer_part: "') + + sign_input + + currency_prefix + + (decimal_input | integer_input) + + pynutil.insert('"') + + pynini.closure(quantity_component, 0, 1) + + currency_field + ) + if minor_spoken: + decimal_minor_component = ( + integer_component + + pynutil.delete(".") + + currency_field + + pynutil.insert(' fractional_part: "') + + minor_decimal_input + + pynutil.insert(f'" currency_min: "{minor_spoken}"') + ) + signed_decimal_minor_component = ( + pynutil.insert('integer_part: "') + + sign_input + + currency_prefix + + integer_input + + pynutil.delete(".") + + pynutil.insert('"') + + currency_field + + pynutil.insert(' fractional_part: "') + + minor_decimal_input + + pynutil.insert(f'" currency_min: "{minor_spoken}"') + ) + prefix_graph |= pynutil.add_weight( + currency_prefix + decimal_minor_component, + -0.1, + ) + prefix_graph |= pynutil.add_weight( + signed_decimal_minor_component, + -0.1, + ) + + graph = (suffix_graph | prefix_graph) + pynutil.insert(" preserve_order: true") + + self.fst = self.add_tokens(graph.optimize()).optimize() diff --git a/nemo_text_processing/text_normalization/ja/taggers/punctuation.py b/nemo_text_processing/text_normalization/ja/taggers/punctuation.py index c5df8388c..a485fd153 100644 --- a/nemo_text_processing/text_normalization/ja/taggers/punctuation.py +++ b/nemo_text_processing/text_normalization/ja/taggers/punctuation.py @@ -37,7 +37,7 @@ class PunctuationFst(GraphFst): def __init__(self, deterministic: bool = True): super().__init__(name="punctuation", kind="classify", deterministic=deterministic) - s = "!#$%&'()*+,-./:;<=>?@^_`{|}。，；：《》“”·~【】！？、‘’.<>-——_、。.「」『』‘`／・；’”“”‷･〔〕々〃ゝゞヽ〲〱〳〴〵ヾ〆，~" + s = "!#$%&'()*+,-./:;<=>?@^_`{|}~" punct_symbols_to_exclude = ["[", "]"] punct_unicode = [ @@ -62,9 +62,7 @@ def __init__(self, deterministic: bool = True): + pynini.accep(">") ) punct = plurals._priority_union(emphasis, punct, NEMO_SIGMA) - range_component = pynini.cross("〜", "から") | pynini.accep( - "から" - ) # forcing this conversion for special tilde - - self.graph = punct | pynutil.add_weight(range_component, -1.0) - self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() + range_component = pynini.cross("〜", "から") + + self.graph = plurals._priority_union(range_component, punct, NEMO_SIGMA) + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/nemo_text_processing/text_normalization/ja/taggers/roman.py b/nemo_text_processing/text_normalization/ja/taggers/roman.py new file mode 100644 index 000000000..bc9b4d490 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/taggers/roman.py @@ -0,0 +1,67 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import GraphFst, insert_space +from nemo_text_processing.text_normalization.ja.utils import get_abs_path, load_labels + + +class RomanFst(GraphFst): + """ + Finite state transducer for classifying Roman numerals in supported contexts. + + Examples: + 第III章 -> name: "第三章" + Chapter IV -> name: "Chapter 四" + Henry VIII -> name: "Henry 八世" + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="roman", kind="classify", deterministic=deterministic) + + roman_values = {roman: int(value) for roman, value in load_labels(get_abs_path("data/roman/roman_numerals.tsv"))} + valid_roman_pairs = [] + for number in range(1, 4000): + roman = self._int_to_roman(number, roman_values) + valid_roman_pairs.append((roman, str(number))) + valid_roman_pairs.append((roman.lower(), str(number))) + + roman_to_number = pynini.string_map(valid_roman_pairs).optimize() + roman_to_cardinal = roman_to_number @ cardinal.just_cardinals + + japanese_suffix = pynini.union("章", "条", "巻", "回") + japanese_context = pynini.accep("第") + roman_to_cardinal + japanese_suffix + + key_cardinal = pynini.union(*[pynini.accep(x[0]) for x in load_labels(get_abs_path("data/roman/key_cardinal.tsv"))]) + key_ordinal = pynini.union(*[pynini.accep(x[0]) for x in load_labels(get_abs_path("data/roman/key_ordinal.tsv"))]) + + cardinal_context = key_cardinal + pynutil.delete(" ") + insert_space + roman_to_cardinal + ordinal_context = key_ordinal + pynutil.delete(" ") + insert_space + roman_to_cardinal + pynutil.insert("世") + preserve = pynini.string_file(get_abs_path("data/roman/preserve.tsv")) + + graph = japanese_context | cardinal_context | ordinal_context | preserve + self.fst = (pynutil.insert('name: "') + graph.optimize() + pynutil.insert('"')).optimize() + + @staticmethod + def _int_to_roman(number: int, roman_values: dict) -> str: + value_to_roman = sorted(((value, roman) for roman, value in roman_values.items()), reverse=True) + result = [] + remaining = number + for value, roman in value_to_roman: + while remaining >= value: + result.append(roman) + remaining -= value + return "".join(result) diff --git a/nemo_text_processing/text_normalization/ja/taggers/serial.py b/nemo_text_processing/text_normalization/ja/taggers/serial.py new file mode 100644 index 000000000..aeab3779c --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/taggers/serial.py @@ -0,0 +1,88 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_DIGIT, NEMO_SIGMA, GraphFst, insert_space +from nemo_text_processing.text_normalization.ja.utils import get_abs_path + + +class SerialFst(GraphFst): + """ + Finite state transducer for classifying compact serial/model identifiers. + + Examples: + B2A23C -> name: "ビー 二 エー 二三 シー" + MIG-25/235212-asdg -> name: "エムアイジー ハイフン 二五 スラッシュ 二三五二一二 ハイフン エーエスディージー" + Room 301 -> name: "Room 三〇一" + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="serial", kind="classify", deterministic=deterministic) + + letters = pynini.string_file(get_abs_path("data/latin/letters.tsv")) + letter_input = pynini.project(letters, "input") + digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) | pynini.cross("0", "〇") + digit_group = pynini.closure(digit, 1) + letter_group = pynini.closure(letters, 1) + + insert_letter_digit_space = pynini.cdrewrite(pynutil.insert(" "), letter_input, NEMO_DIGIT, NEMO_SIGMA) + insert_digit_letter_space = pynini.cdrewrite(pynutil.insert(" "), NEMO_DIGIT, letter_input, NEMO_SIGMA) + alnum_spacing = insert_letter_digit_space @ insert_digit_letter_space + raw_alnum = pynini.closure(letter_input | NEMO_DIGIT, 1) + raw_letter_leading_alnum = ( + letter_input + + pynini.closure(letter_input | NEMO_DIGIT) + + NEMO_DIGIT + + pynini.closure(letter_input | NEMO_DIGIT) + ) + currency_prefix_input = pynini.project( + pynini.string_file(get_abs_path("data/money/currency_prefix.tsv")), "input" + ) + currency_code_number = currency_prefix_input + pynini.closure(NEMO_DIGIT, 1) + raw_letter_leading_alnum = pynini.difference(raw_letter_leading_alnum, currency_code_number) + raw_short_digit_leading_alnum = NEMO_DIGIT + (pynini.accep("x") | pynini.accep("X")) + raw_mixed_alnum = raw_letter_leading_alnum | raw_short_digit_leading_alnum + alnum_reader = pynini.closure(letters | digit | pynini.accep(" "), 1) + alnum = (raw_mixed_alnum @ alnum_spacing @ alnum_reader).optimize() + + delimiter = pynini.cross("-", " ハイフン ") | pynini.cross("/", " スラッシュ ") + unit_input = pynini.project(pynini.string_file(get_abs_path("data/measure/unit.tsv")), "input") + numeric_measure_segment = pynini.closure(NEMO_DIGIT, 1) + unit_input + raw_alnum_segment = pynini.difference(raw_alnum, numeric_measure_segment) + raw_alnum_with_letter = pynini.closure(letter_input | NEMO_DIGIT) + letter_input + pynini.closure( + letter_input | NEMO_DIGIT + ) + raw_alnum_with_letter = pynini.difference(raw_alnum_with_letter, numeric_measure_segment) + segment = (raw_alnum_segment @ alnum_spacing @ alnum_reader).optimize() + segment_with_letter = (raw_alnum_with_letter @ alnum_spacing @ alnum_reader).optimize() + delimited = ( + segment_with_letter + pynini.closure(delimiter + segment, 1) + | segment + delimiter + segment_with_letter + pynini.closure(delimiter + segment) + ) + + special_word = pynini.string_file(get_abs_path("data/serial/words.tsv")) + covid_style = ( + special_word + + pynutil.delete("-") + + insert_space + + (NEMO_DIGIT**2 @ cardinal.just_cardinals) + ) + + model_number = pynini.accep("型番") + insert_space + delimited + room_number = pynini.accep("Room") + pynutil.delete(" ") + insert_space + digit_group + + graph = pynutil.add_weight(covid_style, -0.1) | model_number | room_number | delimited | alnum + self.fst = (pynutil.insert('name: "') + graph.optimize() + pynutil.insert('"')).optimize() diff --git a/nemo_text_processing/text_normalization/ja/taggers/telephone.py b/nemo_text_processing/text_normalization/ja/taggers/telephone.py new file mode 100644 index 000000000..b071625a1 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/taggers/telephone.py @@ -0,0 +1,121 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import ( + NEMO_WHITE_SPACE, + GraphFst, + delete_space, + insert_space, +) +from nemo_text_processing.text_normalization.ja.utils import get_abs_path + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for classifying Japanese telephone numbers. + + Examples: + 090-1234-5678 -> telephone { number_part: "ゼロ九ゼロ 一二三四 五六七八" preserve_order: true } + 03-1234-5678 -> telephone { number_part: "ゼロ三 一二三四 五六七八" preserve_order: true } + +81 90-1234-5678 -> telephone { country_code: "八一" number_part: "九ゼロ 一二三四 五六七八" preserve_order: true } + + Args: + deterministic: if True will provide a single transduction option + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="telephone", kind="classify", deterministic=deterministic) + + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + digit = graph_digit | graph_zero + + sep_char = pynini.union("-", "－", "ー", ".", "．") + delete_sep = pynutil.delete(sep_char) + delete_required_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE, 1)) + block_sep = delete_space + (delete_sep | delete_required_space) + delete_space + insert_space + optional_after_paren_sep = delete_space + pynini.closure(delete_sep + delete_space, 0, 1) + required_country_sep = delete_space + (delete_sep | delete_required_space) + delete_space + + open_paren = pynutil.delete("(") | pynutil.delete("（") + close_paren = pynutil.delete(")") | pynutil.delete("）") + + digits = {count: digit**count for count in range(1, 11)} + paren = { + count: open_paren + digits[count] + close_paren + optional_after_paren_sep + insert_space + for count in range(1, 4) + } + + local_grouped_number = ( + digits[2] + block_sep + digits[4] + block_sep + digits[4] + | digits[3] + block_sep + digits[3] + block_sep + digits[4] + | digits[3] + block_sep + digits[4] + block_sep + digits[4] + | digits[4] + block_sep + digits[2] + block_sep + digits[4] + | digits[4] + block_sep + digits[3] + block_sep + digits[3] + | digits[4] + block_sep + digits[3] + block_sep + digits[4] + | digits[4] + block_sep + digits[4] + block_sep + digits[3] + ) + + local_parenthesized_number = ( + paren[2] + digits[4] + block_sep + digits[4] + | paren[3] + digits[3] + block_sep + digits[4] + | paren[3] + digits[4] + block_sep + digits[4] + ) + + compact_local_number = graph_zero + (digits[9] | digits[10]) + local_number = local_grouped_number | local_parenthesized_number | compact_local_number + + international_number = ( + digits[1] + block_sep + digits[4] + block_sep + digits[4] + | digits[2] + block_sep + digits[3] + block_sep + digits[4] + | digits[2] + block_sep + digits[4] + block_sep + digits[4] + | digits[3] + block_sep + digits[2] + block_sep + digits[4] + | digits[3] + block_sep + digits[3] + block_sep + digits[4] + | paren[1] + digits[4] + block_sep + digits[4] + | paren[2] + digits[3] + block_sep + digits[4] + | paren[2] + digits[4] + block_sep + digits[4] + | paren[3] + digits[2] + block_sep + digits[4] + | paren[3] + digits[3] + block_sep + digits[4] + ) + + country_code = digits[1] | digits[2] + country_code_component = ( + (pynutil.delete("+") | pynutil.delete("＋")) + + pynutil.insert('country_code: "') + + country_code + + pynutil.insert('"') + + required_country_sep + + pynutil.insert(" ") + ) + + extension = ( + delete_space + + (pynutil.delete("内線番号") | pynutil.delete("内線")) + + delete_space + + pynutil.insert(' extension: "') + + pynini.closure(digit, 1, 4) + + pynutil.insert('"') + ) + + number_part = pynutil.insert('number_part: "') + local_number + pynutil.insert('"') + international_number_part = pynutil.insert('number_part: "') + international_number + pynutil.insert('"') + + graph = number_part | (country_code_component + international_number_part) + graph = graph + pynini.closure(extension, 0, 1) + graph = graph + pynutil.insert(" preserve_order: true") + + self.fst = self.add_tokens(graph.optimize()).optimize() diff --git a/nemo_text_processing/text_normalization/ja/taggers/time.py b/nemo_text_processing/text_normalization/ja/taggers/time.py index 7c74bc53e..80b002ab8 100644 --- a/nemo_text_processing/text_normalization/ja/taggers/time.py +++ b/nemo_text_processing/text_normalization/ja/taggers/time.py @@ -41,9 +41,10 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): division = pynini.string_file(get_abs_path("data/time/division.tsv")) division_component = pynutil.insert("suffix: \"") + division + pynutil.insert("\"") + hour_number = pynutil.add_weight(pynini.cross("0", "零"), -0.1) | graph_cardinal hour_component = ( pynutil.insert("hours: \"") - + (graph_cardinal | (graph_cardinal + pynini.cross(".", "点") + graph_cardinal)) + + (hour_number | (graph_cardinal + pynini.cross(".", "点") + graph_cardinal)) + (pynini.accep("時") | pynini.accep("時間") | pynini.accep("時頃")) + pynutil.insert("\"") ) diff --git a/nemo_text_processing/text_normalization/ja/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/ja/taggers/tokenize_and_classify.py index f992e9b70..df55effd3 100644 --- a/nemo_text_processing/text_normalization/ja/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/text_normalization/ja/taggers/tokenize_and_classify.py @@ -18,13 +18,20 @@ import pynini from pynini.lib import pynutil -from nemo_text_processing.text_normalization.ja.graph_utils import GraphFst, generator_main +from nemo_text_processing.text_normalization.ja.graph_utils import GraphFst, delete_extra_space, delete_space, generator_main +from nemo_text_processing.text_normalization.ja.taggers.address import AddressFst from nemo_text_processing.text_normalization.ja.taggers.cardinal import CardinalFst from nemo_text_processing.text_normalization.ja.taggers.date import DateFst from nemo_text_processing.text_normalization.ja.taggers.decimal import DecimalFst +from nemo_text_processing.text_normalization.ja.taggers.electronic import ElectronicFst from nemo_text_processing.text_normalization.ja.taggers.fraction import FractionFst +from nemo_text_processing.text_normalization.ja.taggers.measure import MeasureFst +from nemo_text_processing.text_normalization.ja.taggers.money import MoneyFst from nemo_text_processing.text_normalization.ja.taggers.ordinal import OrdinalFst from nemo_text_processing.text_normalization.ja.taggers.punctuation import PunctuationFst +from nemo_text_processing.text_normalization.ja.taggers.roman import RomanFst +from nemo_text_processing.text_normalization.ja.taggers.serial import SerialFst +from nemo_text_processing.text_normalization.ja.taggers.telephone import TelephoneFst from nemo_text_processing.text_normalization.ja.taggers.time import TimeFst from nemo_text_processing.text_normalization.ja.taggers.whitelist import WhiteListFst from nemo_text_processing.text_normalization.ja.taggers.word import WordFst @@ -59,7 +66,7 @@ def __init__( if cache_dir is not None and cache_dir != "None": os.makedirs(cache_dir, exist_ok=True) whitelist_file = os.path.basename(whitelist) if whitelist else "" - far_file = os.path.join(cache_dir, f"zh_tn_{deterministic}_deterministic_{whitelist_file}_tokenize.far") + far_file = os.path.join(cache_dir, f"ja_tn_{deterministic}_deterministic_{whitelist_file}_tokenize.far") if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] else: @@ -68,25 +75,41 @@ def __init__( decimal = DecimalFst(cardinal=cardinal, deterministic=deterministic) time = TimeFst(cardinal=cardinal, deterministic=deterministic) fraction = FractionFst(cardinal=cardinal, deterministic=deterministic) + measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) + money = MoneyFst(cardinal=cardinal, deterministic=deterministic) + telephone = TelephoneFst(deterministic=deterministic) ordinal = OrdinalFst(cardinal=cardinal, deterministic=deterministic) + address = AddressFst(cardinal=cardinal, deterministic=deterministic) + electronic = ElectronicFst(cardinal=cardinal, deterministic=deterministic) + roman = RomanFst(cardinal=cardinal, deterministic=deterministic) + serial = SerialFst(cardinal=cardinal, deterministic=deterministic) whitelist = WhiteListFst(deterministic=deterministic) word = WordFst(deterministic=deterministic) punctuation = PunctuationFst(deterministic=deterministic) classify = pynini.union( + pynutil.add_weight(electronic.fst, 1.1), + pynutil.add_weight(address.fst, 1.1), + pynutil.add_weight(roman.fst, 1.1), + pynutil.add_weight(serial.fst, 1.1), pynutil.add_weight(date.fst, 1.1), - pynutil.add_weight(fraction.fst, 1.0), + pynutil.add_weight(fraction.fst, 1.1), + pynutil.add_weight(money.fst, 1.1), + pynutil.add_weight(measure.fst, 1.1), + pynutil.add_weight(telephone.fst, 1.1), pynutil.add_weight(time.fst, 1.1), pynutil.add_weight(whitelist.fst, 1.1), pynutil.add_weight(cardinal.fst, 1.1), - pynutil.add_weight(decimal.fst, 3.05), + pynutil.add_weight(decimal.fst, 1.1), pynutil.add_weight(ordinal.fst, 1.1), - pynutil.add_weight(punctuation.fst, 1.0), + pynutil.add_weight(punctuation.fst, 1.1), pynutil.add_weight(word.fst, 100), ) - token = pynutil.insert("tokens { ") + classify + pynutil.insert(" } ") - tagger = pynini.closure(token, 1) + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + tagger = ( + delete_space + token + pynini.closure((delete_extra_space | pynini.accep("")) + token) + delete_space + ) self.fst = tagger diff --git a/nemo_text_processing/text_normalization/ja/taggers/whitelist.py b/nemo_text_processing/text_normalization/ja/taggers/whitelist.py index 2f9391ddf..944fbdecf 100644 --- a/nemo_text_processing/text_normalization/ja/taggers/whitelist.py +++ b/nemo_text_processing/text_normalization/ja/taggers/whitelist.py @@ -14,11 +14,11 @@ # limitations under the License. -import pynini -from pynini.lib import pynutil - -from nemo_text_processing.text_normalization.ja.graph_utils import GraphFst -from nemo_text_processing.text_normalization.ja.utils import get_abs_path +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import GraphFst, delete_space +from nemo_text_processing.text_normalization.ja.utils import get_abs_path class WhiteListFst(GraphFst): @@ -34,7 +34,9 @@ class WhiteListFst(GraphFst): def __init__(self, deterministic: bool = True): super().__init__(name="whitelist", kind="classify", deterministic=deterministic) - whitelist = pynini.string_file(get_abs_path("data/whitelist.tsv")) - graph = (pynutil.insert('name: "')) + (whitelist) + pynutil.insert('"') - - self.fst = graph.optimize() + whitelist = pynini.string_file(get_abs_path("data/whitelist.tsv")) + title = pynini.string_file(get_abs_path("data/whitelist_title.tsv")) + title_with_space = title + delete_space + pynutil.insert(" ") + graph = (pynutil.insert('name: "')) + (title_with_space | whitelist) + pynutil.insert('"') + + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/ja/taggers/word.py b/nemo_text_processing/text_normalization/ja/taggers/word.py index b1403221b..7caaa0e37 100644 --- a/nemo_text_processing/text_normalization/ja/taggers/word.py +++ b/nemo_text_processing/text_normalization/ja/taggers/word.py @@ -1,30 +1,30 @@ -# Copyright (c) 2024 NVIDIA CORPORATION. All rights reserved. -# Copyright 2015 and onwards Google, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from pynini.lib import pynutil - -from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_SPACE, GraphFst - - -class WordFst(GraphFst): - """ - Finite state transducer for classifying plain tokens, that do not belong to any special class. This can be considered as the default class. - e.g. 文字 -> tokens { name: "文字" } - """ - - def __init__(self, deterministic: bool = True): - super().__init__(name="word", kind="classify", deterministic=deterministic) - word = pynutil.insert("name: \"") + NEMO_NOT_SPACE + pynutil.insert("\"") - self.fst = word.optimize() +# Copyright (c) 2024 NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_SPACE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying plain tokens, that do not belong to any special class. This can be considered as the default class. + e.g. 文字 -> tokens { name: "文字" } + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="classify", deterministic=deterministic) + word = pynutil.insert("name: \"") + NEMO_NOT_SPACE + pynutil.insert("\"") + self.fst = word.optimize() diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/fraction.py b/nemo_text_processing/text_normalization/ja/verbalizers/fraction.py index 4743d3bcd..4723b62a8 100644 --- a/nemo_text_processing/text_normalization/ja/verbalizers/fraction.py +++ b/nemo_text_processing/text_normalization/ja/verbalizers/fraction.py @@ -24,7 +24,7 @@ class FractionFst(GraphFst): Finite state transducer for verbalizing fractionss, e.g. tokens { fraction { denominator: "二" numerator: "一"} } -> 1/2 tokens { fraction { integer: "一" denominator: "四" numerator: "三" } } -> 1と3/4 - tokens { fraction { integer: "1" denominator: "4" numerator: "3" } } -> 一荷四分の三 + tokens { fraction { integer: "1" denominator: "4" numerator: "3" } } -> 一と四分の三 tokens { fraction { denominator: "√3" numerator: "1" } } -> ルート三分の一 tokens { fraction { denominator: "1.65" numerator: "50" } } -> 一点六五分の五十 tokens { fraction { denominator: "二" numerator: "一"} } -> マイナス1/2 @@ -82,9 +82,9 @@ def __init__(self, deterministic: bool = True): + pynutil.delete("\"") + pynini.closure(pynini.cross("√", "ルート"), 0, 1) + pynini.closure( - NEMO_NOT_QUOTE - pynini.union("荷", "と", "√") - ) # had to remove these 3 items fron nemo_not _quote so the root is properly converted in a deterministic way. - + pynutil.insert("荷") + NEMO_NOT_QUOTE - pynini.union("と", "√") + ) + + pynutil.insert("と") + pynutil.delete("\"") ) @@ -93,8 +93,8 @@ def __init__(self, deterministic: bool = True): + delete_space + pynutil.delete("\"") + pynini.closure(pynini.cross("√", "ルート"), 0, 1) - + pynini.closure(NEMO_NOT_QUOTE - pynini.union("荷", "と", "√")) - + (pynini.accep("と") | pynini.accep("荷")) + + pynini.closure(NEMO_NOT_QUOTE - pynini.union("と", "√")) + + pynini.accep("と") + pynutil.delete("\"") ) diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/measure.py b/nemo_text_processing/text_normalization/ja/verbalizers/measure.py new file mode 100644 index 000000000..db07360cd --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/verbalizers/measure.py @@ -0,0 +1,50 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_preserve_order, + delete_space, +) + + +class MeasureFst(GraphFst): + """ + Finite state transducer for verbalizing Japanese measure tokens. + + Examples: + measure { cardinal { integer: "五" } units: "キロ" preserve_order: true } -> 五キロ + measure { cardinal { integer: "時速六十" } units: "キロ" preserve_order: true } -> 時速六十キロ + measure { cardinal { integer: "秒速五十" } units: "メートル" preserve_order: true } -> 秒速五十メートル + """ + + def __init__( + self, + cardinal: GraphFst, + decimal: GraphFst, + fraction: GraphFst, + deterministic: bool = True, + ): + super().__init__(name="measure", kind="verbalize", deterministic=deterministic) + + unit = delete_space + pynutil.delete('units: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + number = cardinal.fst | decimal.fst | fraction.fst + + graph = number + unit + delete_preserve_order + + self.fst = self.delete_tokens(graph.optimize()).optimize() diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/money.py b/nemo_text_processing/text_normalization/ja/verbalizers/money.py new file mode 100644 index 000000000..dd299bd78 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/verbalizers/money.py @@ -0,0 +1,64 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_preserve_order, + delete_space, +) + + +class MoneyFst(GraphFst): + """ + Finite state transducer for verbalizing Japanese money. + + Example: + money { integer_part: "百" currency_maj: "円" preserve_order: true } -> 百円 + """ + + def __init__(self, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="verbalize", deterministic=deterministic) + + field_value = pynini.closure(NEMO_NOT_QUOTE, 1) + + integer_part = pynutil.delete('integer_part: "') + field_value + pynutil.delete('"') + + quantity = pynini.closure( + delete_space + pynutil.delete('quantity: "') + field_value + pynutil.delete('"'), + 0, + 1, + ) + + currency_major = delete_space + pynutil.delete('currency_maj: "') + field_value + pynutil.delete('"') + + fractional_part = pynini.closure( + delete_space + + pynutil.delete('fractional_part: "') + + field_value + + pynutil.delete('"') + + delete_space + + pynutil.delete('currency_min: "') + + field_value + + pynutil.delete('"'), + 0, + 1, + ) + + graph = integer_part + quantity + currency_major + fractional_part + delete_preserve_order + + self.fst = self.delete_tokens(graph.optimize()).optimize() diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/post_processing.py b/nemo_text_processing/text_normalization/ja/verbalizers/post_processing.py index 8b196dcaf..f6d38a585 100644 --- a/nemo_text_processing/text_normalization/ja/verbalizers/post_processing.py +++ b/nemo_text_processing/text_normalization/ja/verbalizers/post_processing.py @@ -16,13 +16,17 @@ import os import pynini +from pynini.lib import pynutil -from nemo_text_processing.text_normalization.en.graph_utils import ( +from nemo_text_processing.text_normalization.ja.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_NON_BREAKING_SPACE, NEMO_NOT_SPACE, NEMO_SIGMA, - delete_space, generator_main, ) +from nemo_text_processing.text_normalization.ja.utils import get_abs_path, load_labels from nemo_text_processing.utils.logging import logger @@ -41,73 +45,92 @@ def __init__(self, cache_dir: str = None, overwrite_cache: bool = False): far_file = None if cache_dir is not None and cache_dir != "None": os.makedirs(cache_dir, exist_ok=True) - far_file = os.path.join(cache_dir, "zh_tn_post_processing.far") + far_file = os.path.join(cache_dir, "ja_tn_post_processing.far") if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["post_process_graph"] logger.info(f'Post processing graph was restored from {far_file}.') else: - self.set_punct_dict() self.fst = self.get_punct_postprocess_graph() if far_file: generator_main(far_file, {"post_process_graph": self.fst}) - def set_punct_dict(self): - self.punct_marks = { - "'": [ - "'", - '´', - 'ʹ', - 'ʻ', - 'ʼ', - 'ʽ', - 'ʾ', - 'ˈ', - 'ˊ', - 'ˋ', - '˴', - 'ʹ', - '΄', - '՚', - '՝', - 'י', - '׳', - 'ߴ', - 'ߵ', - 'ᑊ', - 'ᛌ', - '᾽', - '᾿', - '`', - '´', - '῾', - '‘', - '’', - '‛', - '′', - '‵', - 'ꞌ', - '＇', - '｀', - '𖽑', - '𖽒', - ], - } - def get_punct_postprocess_graph(self): """ - Returns graph to post process punctuation marks. + Returns graph to post process Japanese TN output. - {``} quotes are converted to {"}. Note, if there are spaces around single quote {'}, they will be kept. - By default, a space is added after a punctuation mark, and spaces are removed before punctuation marks. + Japanese verbalizers need ordinary inter-token spaces removed, but some + classes intentionally use spaces internally. Protect those spaces as NBSP + before deleting remaining technical spaces; Sparrowhawk tests convert NBSP + back to regular spaces after normalization. """ - remove_space_around_single_quote = pynini.cdrewrite( - delete_space, NEMO_NOT_SPACE, NEMO_NOT_SPACE, pynini.closure(NEMO_SIGMA) + protect_space = pynini.cross(" ", NEMO_NON_BREAKING_SPACE) + ascii_char = NEMO_ALPHA | NEMO_DIGIT + phone_digit = pynini.union("ゼロ", "零", "〇", "一", "二", "三", "四", "五", "六", "七", "八", "九") + space_sensitive_tokens = ( + pynini.project(pynini.string_file(get_abs_path("data/electronic/symbol.tsv")), "output") + | pynini.project(pynini.string_file(get_abs_path("data/latin/letters.tsv")), "output") + | pynini.project(pynini.string_file(get_abs_path("data/serial/words.tsv")), "output") + ).optimize() + title_tokens = pynini.union( + *{spoken for _, spoken in load_labels(get_abs_path("data/whitelist_title.tsv"))} + ).optimize() + japanese_number = pynini.union("零", "〇", "一", "二", "三", "四", "五", "六", "七", "八", "九", "十") + collapse_double_space = pynini.cdrewrite(pynini.cross(" ", " "), "", "", pynini.closure(NEMO_SIGMA)) + + protect_whitelist_internal_space = pynini.closure(NEMO_SIGMA) + for spoken in {spoken for _, spoken in load_labels(get_abs_path("data/whitelist.tsv")) if " " in spoken}: + parts = spoken.split() + for left, right in zip(parts, parts[1:]): + protect_whitelist_internal_space @= pynini.cdrewrite( + protect_space, left, right, pynini.closure(NEMO_SIGMA) + ) + delete_ascii_inner_space = pynini.cdrewrite( + pynutil.delete(" "), ascii_char, ascii_char, pynini.closure(NEMO_SIGMA) + ) + protect_ascii_word_space = pynini.cdrewrite( + protect_space, ascii_char**2, ascii_char**2, pynini.closure(NEMO_SIGMA) + ) + protect_ascii_before_desu = pynini.cdrewrite( + protect_space, ascii_char**2, "です", pynini.closure(NEMO_SIGMA) + ) + protect_ascii_before_japanese_number = pynini.cdrewrite( + protect_space, ascii_char**2, japanese_number, pynini.closure(NEMO_SIGMA) + ) + protect_title_before_ascii = pynini.cdrewrite( + protect_space, title_tokens, ascii_char**2, pynini.closure(NEMO_SIGMA) + ) + protect_after_space_sensitive_token = pynini.cdrewrite( + protect_space, space_sensitive_tokens, "", pynini.closure(NEMO_SIGMA) + ) + protect_before_space_sensitive_token = pynini.cdrewrite( + protect_space, "", space_sensitive_tokens, pynini.closure(NEMO_SIGMA) + ) + protect_phone_group_separator = pynini.cdrewrite( + protect_space, "、", phone_digit**2 | phone_digit**3 | phone_digit**4, pynini.closure(NEMO_SIGMA) + ) + protect_credit_card_group_separator = pynini.cdrewrite( + protect_space, phone_digit**4, phone_digit**4, pynini.closure(NEMO_SIGMA) + ) + delete_technical_space = pynini.cdrewrite( + pynutil.delete(" "), NEMO_NOT_SPACE, NEMO_NOT_SPACE, pynini.closure(NEMO_SIGMA) ) - # this works if spaces in between (good) - # delete space between 2 NEMO_NOT_SPACE（left and right to the space) that are with in a content of NEMO_SIGMA - graph = remove_space_around_single_quote.optimize() + graph = ( + collapse_double_space + @ collapse_double_space + @ protect_whitelist_internal_space + @ protect_ascii_word_space + @ delete_ascii_inner_space + @ protect_ascii_before_desu + @ protect_ascii_before_japanese_number + @ protect_title_before_ascii + @ protect_after_space_sensitive_token + @ protect_before_space_sensitive_token + @ protect_phone_group_separator + @ protect_credit_card_group_separator + @ delete_technical_space + ).optimize() return graph diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/postprocessor.py b/nemo_text_processing/text_normalization/ja/verbalizers/postprocessor.py index 3ff05fa57..5fb49bb6f 100644 --- a/nemo_text_processing/text_normalization/ja/verbalizers/postprocessor.py +++ b/nemo_text_processing/text_normalization/ja/verbalizers/postprocessor.py @@ -14,26 +14,25 @@ import pynini -from pynini.lib import pynutil, utf8 +from pynini.lib import pynutil from nemo_text_processing.text_normalization.ja.graph_utils import ( - NEMO_ALPHA, - NEMO_DIGIT, - NEMO_PUNCT, NEMO_SIGMA, - NEMO_WHITE_SPACE, GraphFst, + TO_LOWER, + TO_UPPER, ) -from nemo_text_processing.text_normalization.ja.utils import get_abs_path +from nemo_text_processing.text_normalization.ja.taggers.punctuation import PunctuationFst class PostProcessor(GraphFst): - ''' - Postprocessing of TN, now contains: - 1. punctuation removal - 2. letter case conversion - 3. oov tagger - ''' + """ + Optional postprocessing for Japanese TN. + + The default graph is an identity rewrite. Optional punctuation removal and + ASCII case conversion are kept generic; OOV tagging needs a Japanese-specific + character inventory and is intentionally not implemented here. + """ def __init__( self, @@ -44,38 +43,20 @@ def __init__( ): super().__init__(name="PostProcessor", kind="processor") - graph = pynini.cdrewrite('', '', '', NEMO_SIGMA) + if to_upper and to_lower: + raise ValueError("to_upper and to_lower cannot both be enabled.") + if tag_oov: + raise ValueError("tag_oov is not supported for Japanese TN without a Japanese charset inventory.") + + graph = pynini.cdrewrite("", "", "", NEMO_SIGMA) if remove_puncts: - remove_puncts_graph = pynutil.delete( - pynini.union(NEMO_PUNCT, pynini.string_file(get_abs_path('data/char/punctuations_zh.tsv'))) - ) + remove_puncts_graph = pynutil.delete(pynini.union(*PunctuationFst().punct_marks)) graph @= pynini.cdrewrite(remove_puncts_graph, "", "", NEMO_SIGMA).optimize() - if to_upper or to_lower: - if to_upper: - conv_cases_graph = pynini.inverse(pynini.string_file(get_abs_path('data/char/upper_to_lower.tsv'))) - else: - conv_cases_graph = pynini.string_file(get_abs_path('data/char/upper_to_lower.tsv')) - + if to_upper: + graph @= pynini.cdrewrite(TO_UPPER, "", "", NEMO_SIGMA).optimize() + elif to_lower: + conv_cases_graph = TO_LOWER graph @= pynini.cdrewrite(conv_cases_graph, "", "", NEMO_SIGMA).optimize() - if tag_oov: - zh_charset_std = pynini.string_file(get_abs_path("data/char/charset_national_standard_2013_8105.tsv")) - zh_charset_ext = pynini.string_file(get_abs_path("data/char/charset_extension.tsv")) - - zh_charset = ( - zh_charset_std | zh_charset_ext | pynini.string_file(get_abs_path("data/char/punctuations_zh.tsv")) - ) - en_charset = NEMO_DIGIT | NEMO_ALPHA | NEMO_PUNCT | NEMO_WHITE_SPACE - charset = zh_charset | en_charset - - with open(get_abs_path("data/char/oov_tags.tsv"), "r") as f: - tags = f.readline().strip().split('\t') - assert len(tags) == 2 - ltag, rtag = tags - - oov_charset = pynini.difference(utf8.VALID_UTF8_CHAR, charset) - tag_oov_graph = pynutil.insert(ltag) + oov_charset + pynutil.insert(rtag) - graph @= pynini.cdrewrite(tag_oov_graph, "", "", NEMO_SIGMA).optimize() - self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/telephone.py b/nemo_text_processing/text_normalization/ja/verbalizers/telephone.py new file mode 100644 index 000000000..5bc4b4d1d --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/verbalizers/telephone.py @@ -0,0 +1,63 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_preserve_order, + delete_space, +) + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for verbalizing Japanese telephone numbers. + + Example: + telephone { number_part: "ゼロ九ゼロ 一二三四 五六七八" preserve_order: true } + -> ゼロ九ゼロ、 一二三四、 五六七八 + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="telephone", kind="verbalize", deterministic=deterministic) + + country_code = ( + pynutil.delete('country_code: "') + + pynutil.insert("プラス") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + ) + + group_separator = pynutil.insert("、 ") + number_group = pynini.closure((NEMO_NOT_QUOTE - " ") | pynini.cross(" ", "、 "), 1) + number_part = pynutil.delete('number_part: "') + number_group + pynutil.delete('"') + + extension = ( + pynutil.delete('extension: "') + + pynutil.insert("内線 ") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + ) + optional_extension = pynini.closure(delete_space + group_separator + extension, 0, 1) + + graph = ( + ((country_code + delete_space + group_separator + number_part) | number_part) + + optional_extension + + delete_preserve_order + ) + + self.fst = self.delete_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/verbalize.py b/nemo_text_processing/text_normalization/ja/verbalizers/verbalize.py index 6a16f96d9..b4233e74d 100644 --- a/nemo_text_processing/text_normalization/ja/verbalizers/verbalize.py +++ b/nemo_text_processing/text_normalization/ja/verbalizers/verbalize.py @@ -20,7 +20,10 @@ from nemo_text_processing.text_normalization.ja.verbalizers.date import DateFst from nemo_text_processing.text_normalization.ja.verbalizers.decimal import DecimalFst from nemo_text_processing.text_normalization.ja.verbalizers.fraction import FractionFst +from nemo_text_processing.text_normalization.ja.verbalizers.measure import MeasureFst +from nemo_text_processing.text_normalization.ja.verbalizers.money import MoneyFst from nemo_text_processing.text_normalization.ja.verbalizers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.ja.verbalizers.telephone import TelephoneFst from nemo_text_processing.text_normalization.ja.verbalizers.time import TimeFst from nemo_text_processing.text_normalization.ja.verbalizers.whitelist import WhiteListFst from nemo_text_processing.text_normalization.ja.verbalizers.word import WordFst @@ -45,9 +48,9 @@ def __init__(self, deterministic: bool = True): decimal = DecimalFst(deterministic=deterministic) word = WordFst(deterministic=deterministic) fraction = FractionFst(deterministic=deterministic) - - # money = MoneyFst(decimal=decimal, deterministic=deterministic) - # measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) + money = MoneyFst(decimal=decimal, deterministic=deterministic) + measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) + telephone = TelephoneFst(deterministic=deterministic) time = TimeFst(deterministic=deterministic) whitelist = WhiteListFst(deterministic=deterministic) @@ -57,6 +60,9 @@ def __init__(self, deterministic: bool = True): ordinal.fst, decimal.fst, fraction.fst, + money.fst, + measure.fst, + telephone.fst, word.fst, time.fst, whitelist.fst, diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/ja/verbalizers/verbalize_final.py index 750598649..ec2325844 100644 --- a/nemo_text_processing/text_normalization/ja/verbalizers/verbalize_final.py +++ b/nemo_text_processing/text_normalization/ja/verbalizers/verbalize_final.py @@ -33,7 +33,7 @@ def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_ far_file = None if cache_dir is not None and cache_dir != "None": os.makedirs(cache_dir, exist_ok=True) - far_file = os.path.join(cache_dir, f"jp_tn_{deterministic}_deterministic_verbalizer.far") + far_file = os.path.join(cache_dir, f"ja_tn_{deterministic}_deterministic_verbalizer.far") if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["verbalize"] else: diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/whitelist.py b/nemo_text_processing/text_normalization/ja/verbalizers/whitelist.py index 11b0b3ae0..75ab59117 100644 --- a/nemo_text_processing/text_normalization/ja/verbalizers/whitelist.py +++ b/nemo_text_processing/text_normalization/ja/verbalizers/whitelist.py @@ -16,7 +16,7 @@ import pynini from pynini.lib import pynutil -from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, GraphFst, delete_space class WhiteListFst(GraphFst): @@ -28,11 +28,11 @@ class WhiteListFst(GraphFst): def __init__(self, deterministic: bool = True): super().__init__(name="whitelist", kind="verbalize", deterministic=deterministic) graph = ( - pynutil.delete("name:") - + delete_space - + pynutil.delete("\"") - + pynini.closure(NEMO_CHAR - " ", 1) - + pynutil.delete("\"") - ) + pynutil.delete("name:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) self.fst = graph.optimize() diff --git a/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_decimal.txt b/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_decimal.txt index d5080acd0..4f91cf83a 100644 --- a/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_decimal.txt +++ b/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_decimal.txt @@ -1,32 +1,32 @@ -マイナス一点零六~-1.06 -マイナス七点零零六~-7.006 -マイナス三十九点五七四~-39.574 -マイナス三点八六~-3.86 -マイナス九十二点一五七四~-92.1574 -マイナス九点零三八~-9.038 -マイナス二点八七四一~-2.8741 -マイナス二百三十一点四六零九~-231.4609 -マイナス五十二点一八~-52.18 -マイナス五点三~-5.3 -マイナス五百七十九点三零零二~-579.3002 -マイナス八十六点四~-86.4 -マイナス八点四零九~-8.409 -マイナス八百二十一点七九五四~-821.7954 -マイナス八百五十二点七~-852.7 -マイナス六十一点零七~-61.07 -マイナス六点八一四~-6.814 -マイナス六百五十七点三零二四~-657.3024 -マイナス四十二点六零五~-42.605 -マイナス四百八十九点零五二一~-489.0521 -答えはマイナス一点零六~答えは-1.06 -計算の結果はマイナス七点零零六~計算の結果は-7.006 -マイナス二点八七四はかなり悪いスコア~-2.874はかなり悪いスコア -五点三は平均点~5.3は平均点 -テストの点数は八十六点四~テストの点数は86.4 -マイナス三十九点五七四は低すぎる~-39.574は低すぎる -答えはマイナス一点零六~答えは-1.06 -計算の結果はマイナス八十六点四~計算の結果は-86.4 -マイナス五十二点一八はかなり悪いスコア~-52.18はかなり悪いスコア -六点八一四は平均点~6.814は平均点 -テストの点数は九十二点一五七四~テストの点数は92.1574 -マイナス七点零零六は低すぎる~-7.006は低すぎる +マイナス一点ゼロ六~-1.06 +マイナス七点ゼロゼロ六~-7.006 +マイナス三十九点五七四~-39.574 +マイナス三点八六~-3.86 +マイナス九十二点一五七四~-92.1574 +マイナス九点ゼロ三八~-9.038 +マイナス二点八七四一~-2.8741 +マイナス二百三十一点四六ゼロ九~-231.4609 +マイナス五十二点一八~-52.18 +マイナス五点三~-5.3 +マイナス五百七十九点三ゼロゼロ二~-579.3002 +マイナス八十六点四~-86.4 +マイナス八点四ゼロ九~-8.409 +マイナス八百二十一点七九五四~-821.7954 +マイナス八百五十二点七~-852.7 +マイナス六十一点ゼロ七~-61.07 +マイナス六点八一四~-6.814 +マイナス六百五十七点三ゼロ二四~-657.3024 +マイナス四十二点六ゼロ五~-42.605 +マイナス四百八十九点ゼロ五二一~-489.0521 +答えはマイナス一点ゼロ六~答えは-1.06 +計算の結果はマイナス七点ゼロゼロ六~計算の結果は-7.006 +マイナス二点八七四はかなり悪いスコア~-2.874はかなり悪いスコア +五点三は平均点~5.3は平均点 +テストの点数は八十六点四~テストの点数は86.4 +マイナス三十九点五七四は低すぎる~-39.574は低すぎる +答えはマイナス一点ゼロ六~答えは-1.06 +計算の結果はマイナス八十六点四~計算の結果は-86.4 +マイナス五十二点一八はかなり悪いスコア~-52.18はかなり悪いスコア +六点八一四は平均点~6.814は平均点 +テストの点数は九十二点一五七四~テストの点数は92.1574 +マイナス七点ゼロゼロ六は低すぎる~-7.006は低すぎる diff --git a/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_fraction.txt b/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_fraction.txt index 32f80e812..d2abd62f6 100644 --- a/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_fraction.txt +++ b/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_fraction.txt @@ -1,34 +1,34 @@ -マイナス一と四分の三~-1 3/4 -一と四分の三~1 3/4 -マイナス一分の九~-9/1 -マイナス一分の六十~-60/1 -マイナス一分の百二十三~-123/1 -マイナス一荷四分の三~-1 3/4 -マイナス七百二十分の一~-1/720 -マイナス三十二分の三十一~-31/32 -マイナス三百九十七分の四~-4/397 -マイナス三百五十分の一~-1/350 -マイナス九十八分の四百七十一~-471/98 -マイナス二と五分の三~-2 3/5 -マイナス二十分の九~-9/20 -マイナス二十分の二十一~-21/20 -マイナス二十四分の一~-1/24 -マイナス二百二十分の一~-1/220 -マイナス二百五十二分の百四十七~-147/252 -マイナス二百五十六分の一~-1/256 -マイナス二荷五分の三~-2 3/5 -マイナス五分の七~-7/5 -マイナス五分の八~-8/5 -マイナス五分の十四~-14/5 -マイナス五分の百三十二~-132/5 -マイナス八分の五~-5/8 -答えはマイナス八分の五~答えは-5/8 -三分の一の人がその場を離れた~1/3の人がその場を離れた -約二分の一を削る~約1/2を削る -十分の三を削って吟醸をつくる~3/10を削って吟醸をつくる -一人三分の一ぐらい取る~1人1/3ぐらい取る -答えは九分の一~答えは1/9 -三分の二の人がその場を離れた~2/3の人がその場を離れた -約十分の一を削る~約1/10を削る -三分の一を削って吟醸をつくる~1/3を削って吟醸をつくる +マイナス一と四分の三~-1 3/4 +一と四分の三~1 3/4 +マイナス一分の九~-9/1 +マイナス一分の六十~-60/1 +マイナス一分の百二十三~-123/1 +マイナス一と四分の三~-1 3/4 +マイナス七百二十分の一~-1/720 +マイナス三十二分の三十一~-31/32 +マイナス三百九十七分の四~-4/397 +マイナス三百五十分の一~-1/350 +マイナス九十八分の四百七十一~-471/98 +マイナス二と五分の三~-2 3/5 +マイナス二十分の九~-9/20 +マイナス二十分の二十一~-21/20 +マイナス二十四分の一~-1/24 +マイナス二百二十分の一~-1/220 +マイナス二百五十二分の百四十七~-147/252 +マイナス二百五十六分の一~-1/256 +マイナス二と五分の三~-2 3/5 +マイナス五分の七~-7/5 +マイナス五分の八~-8/5 +マイナス五分の十四~-14/5 +マイナス五分の百三十二~-132/5 +マイナス八分の五~-5/8 +答えはマイナス八分の五~答えは-5/8 +三分の一の人がその場を離れた~1/3の人がその場を離れた +約二分の一を削る~約1/2を削る +十分の三を削って吟醸をつくる~3/10を削って吟醸をつくる +一人三分の一ぐらい取る~1人1/3ぐらい取る +答えは九分の一~答えは1/9 +三分の二の人がその場を離れた~2/3の人がその場を離れた +約十分の一を削る~約1/10を削る +三分の一を削って吟醸をつくる~1/3を削って吟醸をつくる 一人二分の一とぐらい取る~1人1/2とぐらい取る \ No newline at end of file diff --git a/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_time.txt index 6a5082124..f7b9f664a 100644 --- a/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_time.txt +++ b/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_time.txt @@ -1,40 +1,40 @@ -七時一分~7時1分 -七時四分~7時4分 -九時五十八分~9時58分 -九時十分前~9時10分前 -九時四十分~9時40分 -五時二十六分~5時26分 -六時五十五分~6時55分 -三時~3時 -三時~3時 -正午一分前~正午1分前 -正午十分過ぎ~正午10分過ぎ -九時三十分~9時30分 -七時五十分頃~7時50分頃 -一時~1時 -一時十分~1時10分 -三時~3時 -十七時~17時 -二十時~20時 -二十一時~21時 -二時~2時 -十二時三十分~12時30分 -零時~0時 -零時一分前~0時1分前 -二時~2時 -十二時~12時 -二十時~20時 -二十三時~23時 -二十四時~24時 -零時~0時 -四時~4時 -毎日五時に起きる~毎日5時に起きる -九時四十分の予約になります~9時40分の予約になります -現在の時間は十二時三十分~現在の時間は12時30分 -ちょうど零時になった~ちょうど0時になった -四時で店を閉める~4時で店を閉める -毎日六時に起きる~毎日6時に起きる -十時三十分の予約になります~10時30分の予約になります -現在の時間は十時三分~現在の時間は10時3分 -ちょうど一時になった~ちょうど1時になった -七時で店を閉める~7時で店を閉める +七時一分~7時1分 +七時四分~7時4分 +九時五十八分~9時58分 +九時十分前~9時10分前 +九時四十分~9時40分 +五時二十六分~5時26分 +六時五十五分~6時55分 +三時~3時 +三時~3時 +正午一分前~正午1分前 +正午十分過ぎ~正午10分過ぎ +九時三十分~9時30分 +七時五十分頃~7時50分頃 +一時~1時 +一時十分~1時10分 +三時~3時 +十七時~17時 +二十時~20時 +二十一時~21時 +二時~2時 +十二時三十分~12時30分 +ゼロ時~0時 +ゼロ時一分前~0時1分前 +二時~2時 +十二時~12時 +二十時~20時 +二十三時~23時 +二十四時~24時 +ゼロ時~0時 +四時~4時 +毎日五時に起きる~毎日5時に起きる +九時四十分の予約になります~9時40分の予約になります +現在の時間は十二時三十分~現在の時間は12時30分 +ちょうどゼロ時になった~ちょうど0時になった +四時で店を閉める~4時で店を閉める +毎日六時に起きる~毎日6時に起きる +十時三十分の予約になります~10時30分の予約になります +現在の時間は十時三分~現在の時間は10時3分 +ちょうど一時になった~ちょうど1時になった +七時で店を閉める~7時で店を閉める diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_address.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_address.txt new file mode 100644 index 000000000..4e10ad910 --- /dev/null +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_address.txt @@ -0,0 +1,9 @@ +東京都千代田区丸の内1-1-1~東京都千代田区丸の内一の一の一 +大阪府大阪市北区梅田3-1-1~大阪府大阪市北区梅田三の一の一 +神奈川県横浜市西区みなとみらい2-2-1~神奈川県横浜市西区みなとみらい二の二の一 +1丁目2番3号~一丁目二番三号 +3丁目5番1号~三丁目五番一号 +503号室~五〇三号室 +25階~二十五階 +〒100-0001~郵便番号一ゼロゼロのゼロゼロゼロ一 +住所は東京都港区六本木6-10-1です。~住所は東京都港区六本木六の十の一です。 diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_cardinal.txt index 1d8a2801a..44a159017 100644 --- a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_cardinal.txt +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_cardinal.txt @@ -36,7 +36,7 @@ お年玉50000あげる~お年玉五万あげる 500000000000円分の株式を買った~五千億円分の株式を買った 今年の収益は100000000000になる~今年の収益は一千億になる -隣の会社の年収益は990000000000だそうだ~隣の会社の年収益は九千九百億だそうだ +隣の会社の収益は990000000000だそうだ~隣の会社の収益は九千九百億だそうだ 政府は100000000000の赤字で困っている~政府は一千億の赤字で困っている 兵士500人を派遣する~兵士五百人を派遣する お寺に10000寄付した~お寺に一万寄付した diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_date.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_date.txt index 6625ad4d0..b6eb392bd 100644 --- a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_date.txt +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_date.txt @@ -7,7 +7,6 @@ R.4~令和四年 1日から来年2月末まで~一日から来年二月末まで 1月〜12月~一月から十二月 1月の最終金曜日~一月の最終金曜日 -1月1日(月)〜3日(水)~一月一日月曜日から三日水曜日 1月22日~一月二十二日 70〜80年代~七十から八十年代 70年代~七十年代 diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_decimal.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_decimal.txt index 622b61d1a..a1e6f8c13 100644 --- a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_decimal.txt +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_decimal.txt @@ -12,4 +12,4 @@ 今年の冬の平均気温は-4.2度でした。~今年の冬の平均気温はマイナス四点二度でした。 昨日の株価は-3.6ポイント下落しました。~昨日の株価はマイナス三点六ポイント下落しました。 彼の体重は-2.4キログラム減りました。~彼の体重はマイナス二点四キログラム減りました。 -その製品の評価は-1.5ポイントでした。~その製品の評価はマイナス一点五ポイントでした。 \ No newline at end of file +その製品の評価は-1.5ポイントでした。~その製品の評価はマイナス一点五ポイントでした。 diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_electronic.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_electronic.txt new file mode 100644 index 000000000..f43fca7ae --- /dev/null +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_electronic.txt @@ -0,0 +1,36 @@ +a@hotmail.de~a アット hotmail ドット de +a@hotmail.fr~a アット hotmail ドット fr +a@hotmail.it~a アット hotmail ドット it +a@aol.it~a アット aol ドット it +a@msn.it~a アット msn ドット it +abc@nvidia.app~abc アット nvidia ドット app +nvidia.co.jp~nvidia ドット co ドット jp +a.bc@gmail.com~a ドット bc アット gmail ドット com +cdf@abc.edu~cdf アット abc ドット edu +abc@gmail.abc~abc アット gmail ドット abc +abc@abc.com~abc アット abc ドット com +asdf123@abc.com~asdf 一二三 アット abc ドット com +ab3.sdd.3@gmail.com~ab 三 ドット sdd ドット 3 アット gmail ドット com +ab3-sdd-3@gmail.com~ab 三 ハイフン sdd ハイフン 3 アット gmail ドット com +www.nvidia.com~www ドット nvidia ドット com +nvidia.com~nvidia ドット com +google.co.jp~google ドット co ドット jp +www.google.co.jp~www ドット google ドット co ドット jp +nvidia.ai~nvidia ドット ai +http://www.nvidia.com~http コロン スラッシュ スラッシュ www ドット nvidia ドット com +https://www.nvidia.com~https コロン スラッシュ スラッシュ www ドット nvidia ドット com +https://developer.nvidia.com/drive-cuda/early-access~https コロン スラッシュ スラッシュ developer ドット nvidia ドット com スラッシュ drive ハイフン cuda スラッシュ early ハイフン access +email abc@abc.com です~email abc アット abc ドット com です +abc@abc.com.~abc アット abc ドット com. +1234-5678-9012-3456~一二三四 五六七八 九〇一二 三四五六 +2345-2222-3333-4444~二三四五 二二二二 三三三三 四四四四 +9090-1234-5555-9876~九〇九〇 一二三四 五五五五 九八七六 +カード末尾3456~カード末尾三四五六 +カード下4桁7890~カード下四桁七八九〇 +カード番号1234-5678-9012-3456~カード番号一二三四 五六七八 九〇一二 三四五六 +写真.jpg~写真 ドット jpg +写真.JPG~写真 ドット JPG +写真.png~写真 ドット png +写真.PNG~写真 ドット PNG +資料.pdf~資料 ドット pdf +資料.PDF~資料 ドット PDF diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_fraction.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_fraction.txt index e2095fbfa..0d1ca7aa3 100644 --- a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_fraction.txt +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_fraction.txt @@ -1,18 +1,9 @@ 1/2~二分の一 -1/2~マイナス二分の一 -1 1/2~一荷二分の一 +1 1/2~一と二分の一 1と1/2~一と二分の一 -1荷1/2~一荷二分の一 --1荷1/2~マイナス一荷二分の一 -マイナス1荷1/2~マイナス一荷二分の一 -マイナス√1荷1/2~マイナスルート一荷二分の一 --√1荷1/2~マイナスルート一荷二分の一 --1荷√1/2~マイナス一荷二分のルート一 3分の1~三分の一 -3分の1~マイナス三分の一 -√3分の1~マイナスルート三分の一 -1荷√1/2~一荷二分のルート一 -1荷√1/3~一荷三分のルート一 1と1/3~一と三分の一 -1荷√1/4~一荷四分のルート一 1と√1/4~一と四分のルート一 diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_measure.txt new file mode 100644 index 000000000..6eab137ce --- /dev/null +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_measure.txt @@ -0,0 +1,34 @@ +5kg~五キロ +0kg~ゼロキロ +1 kg~一キロ +2ｋｇ~二キロ +100g~百グラム +500mg~五百ミリグラム +-3kg~マイナス三キロ +1.5kg~一点五キロ +-2.5kg~マイナス二点五キロ +0.05m~零点零五メートル +2.5cm~二点五センチ +3.14m2~三点一四平方メートル +3.14m²~三点一四平方メートル +1/2L~二分の一リットル +-1/2L~マイナス二分の一リットル +3分の1m~三分の一メートル +60km/h~時速六十キロ +60 km / h~時速六十キロ +60kg/h~六十キロ毎時 +50m／s~秒速五十メートル +50ｍ／ｓ~秒速五十メートル +100%~百パーセント +100％~百パーセント +30°C~三十度 +5μg~五マイクログラム +250mL~二百五十ミリリットル +440Hz~四百四十ヘルツ +220V~二百二十ボルト +10GB~十ギガバイト +5kg.~五キロ. +重さ 5kgです。~重さ五キロです。 +この箱は5kgです。~この箱は五キロです。 +温度は30°C。~温度は三十度。 +容量は1.5L、重さは500gです。~容量は一点五リットル、重さは五百グラムです。 diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_money.txt new file mode 100644 index 000000000..b930dda27 --- /dev/null +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_money.txt @@ -0,0 +1,38 @@ +100円~百円 +1円~一円 +0円~ゼロ円 +10円~十円 +1,000円~千円 +12,345円~一万二千三百四十五円 +-500円~マイナス五百円 +マイナス500円~マイナス五百円 +¥100~百円 +￥2500~二千五百円 +JPY 3000~三千円 +jpy400~四百円 +¥3万~三万円 +1.5万円~一点五万円 +2億円~二億円 +3兆円~三兆円 +12円50銭~十二円五十銭 +¥12.50~十二円五十銭 +￥12.05~十二円五銭 +$5~五ドル +US$ 12~十二ドル +USD99~九十九ドル +usd 1.25~一ドル二十五セント +5ドル25セント~五ドル二十五セント +5米ドル25セント~五ドル二十五セント +$12.50~十二ドル五十セント +$0.99~ゼロドル九十九セント +-US$10.50~マイナス十ドル五十セント +€7~七ユーロ +EUR 8.40~八ユーロ四十セント +eur9~九ユーロ +£15~十五ポンド +GBP 20~二十ポンド +gbp30~三十ポンド +₩1000~千ウォン +KRW 5000~五千ウォン +CNY 10~十元 +100人民元~百人民元 diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_roman.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_roman.txt new file mode 100644 index 000000000..60f95de94 --- /dev/null +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_roman.txt @@ -0,0 +1,12 @@ +第III章~第三章 +第IV条~第四条 +第XII巻~第十二巻 +Chapter IV~Chapter 四 +Super Bowl LV~Super Bowl 五十五 +World War II~World War II +Henry VIII~Henry 八世 +第IX回~第九回 +Louis XIV~Louis 十四世 +Part III~Part 三 +第v条~第五条 +chapter ix~chapter 九 diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_serial.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_serial.txt new file mode 100644 index 000000000..be6b8da1f --- /dev/null +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_serial.txt @@ -0,0 +1,15 @@ +B2A23C~ビー 二 エー 二三 シー +C24~シー 二四 +W2s~ダブリュー 二 エス +2x~二 エックス +covid-19~コビッド 十九 +133-ABC~一三三 ハイフン エービーシー +MIG-25/235212-asdg~エムアイジー ハイフン 二五 スラッシュ 二三五二一二 ハイフン エーエスディージー +JL123~ジェーエル 一二三 +NH456~エヌエイチ 四五六 +CX-5~シーエックス ハイフン 五 +型番ABC-1234~型番 エービーシー ハイフン 一二三四 +〒100-0001~郵便番号一ゼロゼロのゼロゼロゼロ一 +ISBN978-4-123456-78-9~アイエスビーエヌ 九七八 ハイフン 四 ハイフン 一二三四五六 ハイフン 七八 ハイフン 九 +Room 301~Room 三〇一 +3号車~三号車 diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_telephone.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_telephone.txt new file mode 100644 index 000000000..23e757cfb --- /dev/null +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_telephone.txt @@ -0,0 +1,12 @@ +090-1234-5678~ゼロ九ゼロ、 一二三四、 五六七八 +080.1234.5678~ゼロ八ゼロ、 一二三四、 五六七八 +090－1234－5678~ゼロ九ゼロ、 一二三四、 五六七八 +03-1234-5678~ゼロ三、 一二三四、 五六七八 +(03) 1234-5678~ゼロ三、 一二三四、 五六七八 +045-123-4567~ゼロ四五、 一二三、 四五六七 +0120-123-456~ゼロ一二ゼロ、 一二三、 四五六 +050-1234-5678~ゼロ五ゼロ、 一二三四、 五六七八 ++81 90-1234-5678~プラス八一、 九ゼロ、 一二三四、 五六七八 ++81-90-1234-5678~プラス八一、 九ゼロ、 一二三四、 五六七八 +090-1234-5678.~ゼロ九ゼロ、 一二三四、 五六七八. +電話番号は090-1234-5678です。~電話番号はゼロ九ゼロ、 一二三四、 五六七八です。 diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_telephone_extended.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_telephone_extended.txt new file mode 100644 index 000000000..b16083126 --- /dev/null +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_telephone_extended.txt @@ -0,0 +1,18 @@ +09012345678~ゼロ九ゼロ一二三四五六七八 +090 1234 5678~ゼロ九ゼロ、 一二三四、 五六七八 +(090) 1234-5678~ゼロ九ゼロ、 一二三四、 五六七八 +0570-123-456~ゼロ五七ゼロ、 一二三、 四五六 ++81 (3) 1234-5678~プラス八一、 三、 一二三四、 五六七八 ++1 (415) 555-0123~プラス一、 四一五、 五五五、 ゼロ一二三 +0800-123-4567~ゼロ八ゼロゼロ、 一二三、 四五六七 +0120-1234-567~ゼロ一二ゼロ、 一二三四、 五六七 +082-123-4567~ゼロ八二、 一二三、 四五六七 +0466-12-3456~ゼロ四六六、 一二、 三四五六 ++81 82-123-4567~プラス八一、 八二、 一二三、 四五六七 ++81 (45) 123-4567~プラス八一、 四五、 一二三、 四五六七 ++44 20-7946-0958~プラス四四、 二ゼロ、 七九四六、 ゼロ九五八 ++84 (28) 3822-9999~プラス八四、 二八、 三八二二、 九九九九 +111-222-3333~一一一、 二二二、 三三三三 +03-1234-5678 内線123~ゼロ三、 一二三四、 五六七八、 内線 一二三 +090-1234-5678に電話してください。~ゼロ九ゼロ、 一二三四、 五六七八に電話してください。 +0120-123-456に電話~ゼロ一二ゼロ、 一二三、 四五六に電話 diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_time.txt index 42e7600a7..ca805d214 100644 --- a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_time.txt +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_time.txt @@ -199,4 +199,4 @@ 翌日27時~翌日二十七時 翌日25時~翌日二十五時 翌日26時~翌日二十六時 -0時~零時 \ No newline at end of file +0時~零時 diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_whitelist.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_whitelist.txt new file mode 100644 index 000000000..1f637ed7e --- /dev/null +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_whitelist.txt @@ -0,0 +1,14 @@ +Dr.田中~ドクター田中 +Mr.山田~ミスター山田 +Mrs.佐藤~ミセス佐藤 +Ms.鈴木~ミス鈴木 +Prof. Smith~プロフェッサー Smith +vs.~バーサス +No.3~ナンバー三 +etc.~エトセトラ +Ph.D.~p h d +St.~ストリート +Mt.~マウント +Ave.~アベニュー +jr.~ジュニア +Sr.~シニア diff --git a/tests/nemo_text_processing/ja/test_address.py b/tests/nemo_text_processing/ja/test_address.py new file mode 100644 index 000000000..ba75abe01 --- /dev/null +++ b/tests/nemo_text_processing/ja/test_address.py @@ -0,0 +1,30 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + +class TestAddress: + normalizer_ja = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('ja/data_text_normalization/test_cases_address.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_address(self, test_input, expected): + preds = self.normalizer_ja.normalize(test_input) + assert expected == preds diff --git a/tests/nemo_text_processing/ja/test_date.py b/tests/nemo_text_processing/ja/test_date.py index cd7127698..d6125b842 100644 --- a/tests/nemo_text_processing/ja/test_date.py +++ b/tests/nemo_text_processing/ja/test_date.py @@ -37,6 +37,6 @@ def test_denorm(self, test_input, expected): @parameterized.expand(parse_test_case_file('ja/data_text_normalization/test_cases_date.txt')) @pytest.mark.run_only_on('CPU') @pytest.mark.unit - def test_denorm(self, test_input, expected): + def test_norm(self, test_input, expected): pred = self.normalizer.normalize(test_input, verbose=False) assert pred == expected diff --git a/tests/nemo_text_processing/ja/test_electronic.py b/tests/nemo_text_processing/ja/test_electronic.py new file mode 100644 index 000000000..81d14a0ce --- /dev/null +++ b/tests/nemo_text_processing/ja/test_electronic.py @@ -0,0 +1,30 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + +class TestElectronic: + normalizer_ja = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('ja/data_text_normalization/test_cases_electronic.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_electronic(self, test_input, expected): + preds = self.normalizer_ja.normalize(test_input) + assert expected == preds diff --git a/tests/nemo_text_processing/ja/test_measure.py b/tests/nemo_text_processing/ja/test_measure.py new file mode 100644 index 000000000..6653f682f --- /dev/null +++ b/tests/nemo_text_processing/ja/test_measure.py @@ -0,0 +1,33 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestMeasure: + normalizer_ja = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('ja/data_text_normalization/test_cases_measure.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_measure(self, test_input, expected): + preds = self.normalizer_ja.normalize(test_input) + assert expected == preds \ No newline at end of file diff --git a/tests/nemo_text_processing/ja/test_money.py b/tests/nemo_text_processing/ja/test_money.py new file mode 100644 index 000000000..9625e8d26 --- /dev/null +++ b/tests/nemo_text_processing/ja/test_money.py @@ -0,0 +1,32 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestMoney: + normalizer_ja = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('ja/data_text_normalization/test_cases_money.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_money(self, test_input, expected): + preds = self.normalizer_ja.normalize(test_input) + assert expected == preds diff --git a/tests/nemo_text_processing/ja/test_range.py b/tests/nemo_text_processing/ja/test_range.py new file mode 100644 index 000000000..6760efb5c --- /dev/null +++ b/tests/nemo_text_processing/ja/test_range.py @@ -0,0 +1,34 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +pytestmark = pytest.mark.skip(reason="JA TN range is not wired while ownership and priority are being stabilized") + + +class TestRange: + normalizer_ja = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('ja/data_text_normalization/test_cases_range.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_range(self, test_input, expected): + preds = self.normalizer_ja.normalize(test_input) + assert expected == preds diff --git a/tests/nemo_text_processing/ja/test_roman.py b/tests/nemo_text_processing/ja/test_roman.py new file mode 100644 index 000000000..c67b5f048 --- /dev/null +++ b/tests/nemo_text_processing/ja/test_roman.py @@ -0,0 +1,30 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + +class TestRoman: + normalizer_ja = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('ja/data_text_normalization/test_cases_roman.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_roman(self, test_input, expected): + preds = self.normalizer_ja.normalize(test_input) + assert expected == preds diff --git a/tests/nemo_text_processing/ja/test_serial.py b/tests/nemo_text_processing/ja/test_serial.py new file mode 100644 index 000000000..72187a8cb --- /dev/null +++ b/tests/nemo_text_processing/ja/test_serial.py @@ -0,0 +1,30 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + +class TestSerial: + normalizer_ja = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('ja/data_text_normalization/test_cases_serial.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_serial(self, test_input, expected): + preds = self.normalizer_ja.normalize(test_input) + assert expected == preds diff --git a/tests/nemo_text_processing/ja/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/ja/test_sparrowhawk_normalization.sh index 42db11fd6..bce50220d 100644 --- a/tests/nemo_text_processing/ja/test_sparrowhawk_normalization.sh +++ b/tests/nemo_text_processing/ja/test_sparrowhawk_normalization.sh @@ -46,6 +46,42 @@ testTNDateText() { input=$PROJECT_DIR/ja/data_text_normalization/test_cases_date.txt runtest $input } +testTNMoneyText() { + input=$PROJECT_DIR/ja/data_text_normalization/test_cases_money.txt + runtest $input +} +testTNMeasureText() { + input=$PROJECT_DIR/ja/data_text_normalization/test_cases_measure.txt + runtest $input +} +testTNTelephoneText() { + input=$PROJECT_DIR/ja/data_text_normalization/test_cases_telephone.txt + runtest $input +} +testTNElectronicText() { + input=$PROJECT_DIR/ja/data_text_normalization/test_cases_electronic.txt + runtest $input +} +testTNSerialText() { + input=$PROJECT_DIR/ja/data_text_normalization/test_cases_serial.txt + runtest $input +} +# testTNRangeText() { +# input=$PROJECT_DIR/ja/data_text_normalization/test_cases_range.txt +# runtest $input +# } +testTNAddressText() { + input=$PROJECT_DIR/ja/data_text_normalization/test_cases_address.txt + runtest $input +} +testTNRomanText() { + input=$PROJECT_DIR/ja/data_text_normalization/test_cases_roman.txt + runtest $input +} +testTNWhitelistText() { + input=$PROJECT_DIR/ja/data_text_normalization/test_cases_whitelist.txt + runtest $input +} # Load shUnit2 diff --git a/tests/nemo_text_processing/ja/test_telephone.py b/tests/nemo_text_processing/ja/test_telephone.py new file mode 100644 index 000000000..d5e85dc81 --- /dev/null +++ b/tests/nemo_text_processing/ja/test_telephone.py @@ -0,0 +1,44 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestTelephone: + normalizer_ja = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('ja/data_text_normalization/test_cases_telephone.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_telephone(self, test_input, expected): + preds = self.normalizer_ja.normalize(test_input) + assert expected == preds + + @parameterized.expand( + [ + ("119", "百十九"), + ("110番", "百十番"), + ] + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_telephone_boundaries(self, test_input, expected): + preds = self.normalizer_ja.normalize(test_input) + assert expected == preds diff --git a/tests/nemo_text_processing/ja/test_telephone_extended.py b/tests/nemo_text_processing/ja/test_telephone_extended.py new file mode 100644 index 000000000..d01f99772 --- /dev/null +++ b/tests/nemo_text_processing/ja/test_telephone_extended.py @@ -0,0 +1,31 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestTelephoneExtended: + normalizer_ja = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('ja/data_text_normalization/test_cases_telephone_extended.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_telephone_extended(self, test_input, expected): + preds = self.normalizer_ja.normalize(test_input) + assert expected == preds diff --git a/tests/nemo_text_processing/ja/test_whitelist.py b/tests/nemo_text_processing/ja/test_whitelist.py new file mode 100644 index 000000000..77b4307b2 --- /dev/null +++ b/tests/nemo_text_processing/ja/test_whitelist.py @@ -0,0 +1,31 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestWhitelist: + normalizer_ja = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('ja/data_text_normalization/test_cases_whitelist.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_whitelist(self, test_input, expected): + preds = self.normalizer_ja.normalize(test_input) + assert expected == preds From 86b45aa5019107d33c0321f06117f69195c69cd0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 2 Jul 2026 18:41:21 +0000 Subject: [PATCH 2/2] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../text_normalization/ja/taggers/address.py | 9 ++++--- .../ja/taggers/electronic.py | 15 ++++++++--- .../text_normalization/ja/taggers/fraction.py | 5 +--- .../text_normalization/ja/taggers/measure.py | 17 +++---------- .../text_normalization/ja/taggers/money.py | 25 ++++--------------- .../text_normalization/ja/taggers/roman.py | 12 ++++++--- .../text_normalization/ja/taggers/serial.py | 18 +++++-------- .../ja/taggers/tokenize_and_classify.py | 7 +++++- .../ja/taggers/whitelist.py | 10 ++++---- .../ja/verbalizers/fraction.py | 4 +-- .../ja/verbalizers/post_processing.py | 4 +-- .../ja/verbalizers/postprocessor.py | 7 +----- .../ja/verbalizers/whitelist.py | 2 +- tests/nemo_text_processing/ja/test_address.py | 1 + .../ja/test_electronic.py | 1 + tests/nemo_text_processing/ja/test_measure.py | 2 +- tests/nemo_text_processing/ja/test_roman.py | 1 + tests/nemo_text_processing/ja/test_serial.py | 1 + 18 files changed, 61 insertions(+), 80 deletions(-) diff --git a/nemo_text_processing/text_normalization/ja/taggers/address.py b/nemo_text_processing/text_normalization/ja/taggers/address.py index 2e0c28b2b..6f108d38e 100644 --- a/nemo_text_processing/text_normalization/ja/taggers/address.py +++ b/nemo_text_processing/text_normalization/ja/taggers/address.py @@ -40,9 +40,12 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): digit_for_room = digit | zero_maru digit_for_postal = digit | zero - hyphen_to_no = pynini.closure(pynutil.delete(" "), 0, 1) + ( - pynutil.delete("-") | pynutil.delete("－") | pynutil.delete("ー") - ) + pynini.closure(pynutil.delete(" "), 0, 1) + pynutil.insert("の") + hyphen_to_no = ( + pynini.closure(pynutil.delete(" "), 0, 1) + + (pynutil.delete("-") | pynutil.delete("－") | pynutil.delete("ー")) + + pynini.closure(pynutil.delete(" "), 0, 1) + + pynutil.insert("の") + ) address_chain = address_number + hyphen_to_no + address_number + hyphen_to_no + address_number diff --git a/nemo_text_processing/text_normalization/ja/taggers/electronic.py b/nemo_text_processing/text_normalization/ja/taggers/electronic.py index e767a83db..52fdf232b 100644 --- a/nemo_text_processing/text_normalization/ja/taggers/electronic.py +++ b/nemo_text_processing/text_normalization/ja/taggers/electronic.py @@ -58,7 +58,9 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): raw_label = pynini.closure(NEMO_ALPHA | NEMO_DIGIT, 1) alpha_label = pynini.closure(NEMO_ALPHA, 1) digit_label = pynini.closure(NEMO_DIGIT, 1) - raw_label_with_hyphen = raw_label + pynini.closure(pynutil.delete("-") + pynutil.insert(" ハイフン ") + raw_label) + raw_label_with_hyphen = raw_label + pynini.closure( + pynutil.delete("-") + pynutil.insert(" ハイフン ") + raw_label + ) insert_alpha_digit_space = pynini.cdrewrite(pynutil.insert(" "), NEMO_ALPHA, NEMO_DIGIT, NEMO_SIGMA) insert_digit_alpha_space = pynini.cdrewrite(pynutil.insert(" "), NEMO_DIGIT, NEMO_ALPHA, NEMO_SIGMA) @@ -92,13 +94,18 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): four_digits = NEMO_DIGIT**4 @ (digit_zero_maru**4) card_separator = (pynutil.delete("-") | pynutil.delete(" ")) + insert_space - credit_card = four_digits + card_separator + four_digits + card_separator + four_digits + card_separator + four_digits + credit_card = ( + four_digits + card_separator + four_digits + card_separator + four_digits + card_separator + four_digits + ) card_cue = pynini.string_file(get_abs_path("data/electronic/card_cues.tsv")) card_with_cue = card_cue + credit_card card_tail_with_cue = card_cue + four_digits - card_tail = pynini.accep("カード下") + (NEMO_DIGIT @ cardinal.just_cardinals) + pynini.accep("桁") + ( - NEMO_DIGIT**4 @ (digit_zero_maru**4) + card_tail = ( + pynini.accep("カード下") + + (NEMO_DIGIT @ cardinal.just_cardinals) + + pynini.accep("桁") + + (NEMO_DIGIT**4 @ (digit_zero_maru**4)) ) extension = pynini.string_file(get_abs_path("data/electronic/file_extensions.tsv")) diff --git a/nemo_text_processing/text_normalization/ja/taggers/fraction.py b/nemo_text_processing/text_normalization/ja/taggers/fraction.py index ca2110296..addc0d2e7 100644 --- a/nemo_text_processing/text_normalization/ja/taggers/fraction.py +++ b/nemo_text_processing/text_normalization/ja/taggers/fraction.py @@ -58,10 +58,7 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): ) integer_component_with_char = ( pynutil.insert('integer_part: \"') - + ( - (cardinal | (root + cardinal) | decimal_number | (root + decimal_number)) - + pynini.accep("と") - ) + + ((cardinal | (root + cardinal) | decimal_number | (root + decimal_number)) + pynini.accep("と")) + pynutil.insert("\"") + pynutil.insert(NEMO_SPACE) ) diff --git a/nemo_text_processing/text_normalization/ja/taggers/measure.py b/nemo_text_processing/text_normalization/ja/taggers/measure.py index fa88e8c4c..306aae207 100644 --- a/nemo_text_processing/text_normalization/ja/taggers/measure.py +++ b/nemo_text_processing/text_normalization/ja/taggers/measure.py @@ -120,10 +120,7 @@ def __init__( + pynutil.insert(" }") ) fraction_graph = ( - pynutil.insert("fraction { ") - + pynini.closure(optional_sign, 0, 1) - + fraction.graph - + pynutil.insert(" }") + pynutil.insert("fraction { ") + pynini.closure(optional_sign, 0, 1) + fraction.graph + pynutil.insert(" }") ) speed_kmh_number = ( @@ -143,16 +140,8 @@ def __init__( number = cardinal_graph | decimal_graph | fraction_graph - speed_kmh_graph = ( - speed_kmh_number - + speed_kmh_unit - + pynutil.insert(" preserve_order: true") - ) - speed_ms_graph = ( - speed_ms_number - + speed_ms_unit - + pynutil.insert(" preserve_order: true") - ) + speed_kmh_graph = speed_kmh_number + speed_kmh_unit + pynutil.insert(" preserve_order: true") + speed_ms_graph = speed_ms_number + speed_ms_unit + pynutil.insert(" preserve_order: true") general_graph = number + unit_component + pynutil.insert(" preserve_order: true") graph = speed_kmh_graph | speed_ms_graph | general_graph diff --git a/nemo_text_processing/text_normalization/ja/taggers/money.py b/nemo_text_processing/text_normalization/ja/taggers/money.py index 0024b48e5..258e56179 100644 --- a/nemo_text_processing/text_normalization/ja/taggers/money.py +++ b/nemo_text_processing/text_normalization/ja/taggers/money.py @@ -71,24 +71,16 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): currency_minor_by_major = dict(load_labels(get_abs_path("data/money/currency_minor_by_major.tsv"))) currency_minor = pynini.string_file(get_abs_path("data/money/currency_minor.tsv")) non_zero_digit = pynini.difference(NEMO_DIGIT, "0") - minor_decimal_input = ( - (NEMO_DIGIT**2 @ graph_cardinal) | (pynutil.delete("0") + (non_zero_digit @ graph_digit)) - ) + minor_decimal_input = (NEMO_DIGIT**2 @ graph_cardinal) | (pynutil.delete("0") + (non_zero_digit @ graph_digit)) - suffix_graph = ( - number_component - + pynini.closure(quantity_component, 0, 1) - + currency_major_component - ) + suffix_graph = number_component + pynini.closure(quantity_component, 0, 1) + currency_major_component for written, spoken in currency_major_labels: minor_spoken = currency_minor_by_major.get(written) if not minor_spoken: continue currency_major_suffix = ( - delete_space - + pynutil.delete(written) - + pynutil.insert(f' currency_maj: "{spoken}"') + delete_space + pynutil.delete(written) + pynutil.insert(f' currency_maj: "{spoken}"') ) minor_suffix = ( delete_space @@ -99,11 +91,7 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): + (currency_minor @ pynini.cross(minor_spoken, "")) + pynutil.insert(f' currency_min: "{minor_spoken}"') ) - suffix_graph |= ( - signed_integer_component - + currency_major_suffix - + minor_suffix - ) + suffix_graph |= signed_integer_component + currency_major_suffix + minor_suffix prefix_graph = pynini.Fst() for written, spoken in currency_prefix_labels: @@ -112,10 +100,7 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): minor_spoken = currency_minor_by_major.get(written) prefix_graph |= ( - currency_prefix - + number_component - + pynini.closure(quantity_component, 0, 1) - + currency_field + currency_prefix + number_component + pynini.closure(quantity_component, 0, 1) + currency_field ) prefix_graph |= ( pynutil.insert('integer_part: "') diff --git a/nemo_text_processing/text_normalization/ja/taggers/roman.py b/nemo_text_processing/text_normalization/ja/taggers/roman.py index bc9b4d490..146e80f50 100644 --- a/nemo_text_processing/text_normalization/ja/taggers/roman.py +++ b/nemo_text_processing/text_normalization/ja/taggers/roman.py @@ -32,7 +32,9 @@ class RomanFst(GraphFst): def __init__(self, cardinal: GraphFst, deterministic: bool = True): super().__init__(name="roman", kind="classify", deterministic=deterministic) - roman_values = {roman: int(value) for roman, value in load_labels(get_abs_path("data/roman/roman_numerals.tsv"))} + roman_values = { + roman: int(value) for roman, value in load_labels(get_abs_path("data/roman/roman_numerals.tsv")) + } valid_roman_pairs = [] for number in range(1, 4000): roman = self._int_to_roman(number, roman_values) @@ -45,8 +47,12 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): japanese_suffix = pynini.union("章", "条", "巻", "回") japanese_context = pynini.accep("第") + roman_to_cardinal + japanese_suffix - key_cardinal = pynini.union(*[pynini.accep(x[0]) for x in load_labels(get_abs_path("data/roman/key_cardinal.tsv"))]) - key_ordinal = pynini.union(*[pynini.accep(x[0]) for x in load_labels(get_abs_path("data/roman/key_ordinal.tsv"))]) + key_cardinal = pynini.union( + *[pynini.accep(x[0]) for x in load_labels(get_abs_path("data/roman/key_cardinal.tsv"))] + ) + key_ordinal = pynini.union( + *[pynini.accep(x[0]) for x in load_labels(get_abs_path("data/roman/key_ordinal.tsv"))] + ) cardinal_context = key_cardinal + pynutil.delete(" ") + insert_space + roman_to_cardinal ordinal_context = key_ordinal + pynutil.delete(" ") + insert_space + roman_to_cardinal + pynutil.insert("世") diff --git a/nemo_text_processing/text_normalization/ja/taggers/serial.py b/nemo_text_processing/text_normalization/ja/taggers/serial.py index aeab3779c..b24a23cbc 100644 --- a/nemo_text_processing/text_normalization/ja/taggers/serial.py +++ b/nemo_text_processing/text_normalization/ja/taggers/serial.py @@ -62,24 +62,18 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): unit_input = pynini.project(pynini.string_file(get_abs_path("data/measure/unit.tsv")), "input") numeric_measure_segment = pynini.closure(NEMO_DIGIT, 1) + unit_input raw_alnum_segment = pynini.difference(raw_alnum, numeric_measure_segment) - raw_alnum_with_letter = pynini.closure(letter_input | NEMO_DIGIT) + letter_input + pynini.closure( - letter_input | NEMO_DIGIT + raw_alnum_with_letter = ( + pynini.closure(letter_input | NEMO_DIGIT) + letter_input + pynini.closure(letter_input | NEMO_DIGIT) ) raw_alnum_with_letter = pynini.difference(raw_alnum_with_letter, numeric_measure_segment) segment = (raw_alnum_segment @ alnum_spacing @ alnum_reader).optimize() segment_with_letter = (raw_alnum_with_letter @ alnum_spacing @ alnum_reader).optimize() - delimited = ( - segment_with_letter + pynini.closure(delimiter + segment, 1) - | segment + delimiter + segment_with_letter + pynini.closure(delimiter + segment) - ) + delimited = segment_with_letter + pynini.closure( + delimiter + segment, 1 + ) | segment + delimiter + segment_with_letter + pynini.closure(delimiter + segment) special_word = pynini.string_file(get_abs_path("data/serial/words.tsv")) - covid_style = ( - special_word - + pynutil.delete("-") - + insert_space - + (NEMO_DIGIT**2 @ cardinal.just_cardinals) - ) + covid_style = special_word + pynutil.delete("-") + insert_space + (NEMO_DIGIT**2 @ cardinal.just_cardinals) model_number = pynini.accep("型番") + insert_space + delimited room_number = pynini.accep("Room") + pynutil.delete(" ") + insert_space + digit_group diff --git a/nemo_text_processing/text_normalization/ja/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/ja/taggers/tokenize_and_classify.py index df55effd3..f1de00ee7 100644 --- a/nemo_text_processing/text_normalization/ja/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/text_normalization/ja/taggers/tokenize_and_classify.py @@ -18,7 +18,12 @@ import pynini from pynini.lib import pynutil -from nemo_text_processing.text_normalization.ja.graph_utils import GraphFst, delete_extra_space, delete_space, generator_main +from nemo_text_processing.text_normalization.ja.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) from nemo_text_processing.text_normalization.ja.taggers.address import AddressFst from nemo_text_processing.text_normalization.ja.taggers.cardinal import CardinalFst from nemo_text_processing.text_normalization.ja.taggers.date import DateFst diff --git a/nemo_text_processing/text_normalization/ja/taggers/whitelist.py b/nemo_text_processing/text_normalization/ja/taggers/whitelist.py index 944fbdecf..b272a8e22 100644 --- a/nemo_text_processing/text_normalization/ja/taggers/whitelist.py +++ b/nemo_text_processing/text_normalization/ja/taggers/whitelist.py @@ -14,11 +14,11 @@ # limitations under the License. -import pynini -from pynini.lib import pynutil - -from nemo_text_processing.text_normalization.ja.graph_utils import GraphFst, delete_space -from nemo_text_processing.text_normalization.ja.utils import get_abs_path +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import GraphFst, delete_space +from nemo_text_processing.text_normalization.ja.utils import get_abs_path class WhiteListFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/fraction.py b/nemo_text_processing/text_normalization/ja/verbalizers/fraction.py index 4723b62a8..241041cf0 100644 --- a/nemo_text_processing/text_normalization/ja/verbalizers/fraction.py +++ b/nemo_text_processing/text_normalization/ja/verbalizers/fraction.py @@ -81,9 +81,7 @@ def __init__(self, deterministic: bool = True): + delete_space + pynutil.delete("\"") + pynini.closure(pynini.cross("√", "ルート"), 0, 1) - + pynini.closure( - NEMO_NOT_QUOTE - pynini.union("と", "√") - ) + + pynini.closure(NEMO_NOT_QUOTE - pynini.union("と", "√")) + pynutil.insert("と") + pynutil.delete("\"") ) diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/post_processing.py b/nemo_text_processing/text_normalization/ja/verbalizers/post_processing.py index f6d38a585..a0870b5b7 100644 --- a/nemo_text_processing/text_normalization/ja/verbalizers/post_processing.py +++ b/nemo_text_processing/text_normalization/ja/verbalizers/post_processing.py @@ -92,9 +92,7 @@ def get_punct_postprocess_graph(self): protect_ascii_word_space = pynini.cdrewrite( protect_space, ascii_char**2, ascii_char**2, pynini.closure(NEMO_SIGMA) ) - protect_ascii_before_desu = pynini.cdrewrite( - protect_space, ascii_char**2, "です", pynini.closure(NEMO_SIGMA) - ) + protect_ascii_before_desu = pynini.cdrewrite(protect_space, ascii_char**2, "です", pynini.closure(NEMO_SIGMA)) protect_ascii_before_japanese_number = pynini.cdrewrite( protect_space, ascii_char**2, japanese_number, pynini.closure(NEMO_SIGMA) ) diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/postprocessor.py b/nemo_text_processing/text_normalization/ja/verbalizers/postprocessor.py index 5fb49bb6f..cd4b6291a 100644 --- a/nemo_text_processing/text_normalization/ja/verbalizers/postprocessor.py +++ b/nemo_text_processing/text_normalization/ja/verbalizers/postprocessor.py @@ -16,12 +16,7 @@ import pynini from pynini.lib import pynutil -from nemo_text_processing.text_normalization.ja.graph_utils import ( - NEMO_SIGMA, - GraphFst, - TO_LOWER, - TO_UPPER, -) +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_SIGMA, TO_LOWER, TO_UPPER, GraphFst from nemo_text_processing.text_normalization.ja.taggers.punctuation import PunctuationFst diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/whitelist.py b/nemo_text_processing/text_normalization/ja/verbalizers/whitelist.py index 75ab59117..44d694323 100644 --- a/nemo_text_processing/text_normalization/ja/verbalizers/whitelist.py +++ b/nemo_text_processing/text_normalization/ja/verbalizers/whitelist.py @@ -16,7 +16,7 @@ import pynini from pynini.lib import pynutil -from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, GraphFst, delete_space +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, GraphFst, delete_space class WhiteListFst(GraphFst): diff --git a/tests/nemo_text_processing/ja/test_address.py b/tests/nemo_text_processing/ja/test_address.py index ba75abe01..82d31999d 100644 --- a/tests/nemo_text_processing/ja/test_address.py +++ b/tests/nemo_text_processing/ja/test_address.py @@ -19,6 +19,7 @@ from ..utils import CACHE_DIR, parse_test_case_file + class TestAddress: normalizer_ja = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') diff --git a/tests/nemo_text_processing/ja/test_electronic.py b/tests/nemo_text_processing/ja/test_electronic.py index 81d14a0ce..ad22fb082 100644 --- a/tests/nemo_text_processing/ja/test_electronic.py +++ b/tests/nemo_text_processing/ja/test_electronic.py @@ -19,6 +19,7 @@ from ..utils import CACHE_DIR, parse_test_case_file + class TestElectronic: normalizer_ja = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') diff --git a/tests/nemo_text_processing/ja/test_measure.py b/tests/nemo_text_processing/ja/test_measure.py index 6653f682f..026542e8d 100644 --- a/tests/nemo_text_processing/ja/test_measure.py +++ b/tests/nemo_text_processing/ja/test_measure.py @@ -30,4 +30,4 @@ class TestMeasure: @pytest.mark.unit def test_norm_measure(self, test_input, expected): preds = self.normalizer_ja.normalize(test_input) - assert expected == preds \ No newline at end of file + assert expected == preds diff --git a/tests/nemo_text_processing/ja/test_roman.py b/tests/nemo_text_processing/ja/test_roman.py index c67b5f048..6d0465908 100644 --- a/tests/nemo_text_processing/ja/test_roman.py +++ b/tests/nemo_text_processing/ja/test_roman.py @@ -19,6 +19,7 @@ from ..utils import CACHE_DIR, parse_test_case_file + class TestRoman: normalizer_ja = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') diff --git a/tests/nemo_text_processing/ja/test_serial.py b/tests/nemo_text_processing/ja/test_serial.py index 72187a8cb..bcb2b2c00 100644 --- a/tests/nemo_text_processing/ja/test_serial.py +++ b/tests/nemo_text_processing/ja/test_serial.py @@ -19,6 +19,7 @@ from ..utils import CACHE_DIR, parse_test_case_file + class TestSerial: normalizer_ja = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased')