From f859ee48a547e56229241b8c0e1bb48e73135701 Mon Sep 17 00:00:00 2001 From: Shreyas Pawar Date: Wed, 9 Sep 2026 07:02:36 +0000 Subject: [PATCH 1/5] math datatype added Signed-off-by: Shreyas Pawar --- .../hi/data/measure/math_operation.tsv | 9 +++++ .../text_normalization/hi/taggers/measure.py | 34 ++++++++++++++++++ .../hi/verbalizers/measure.py | 31 ++++++++++++---- .../test_cases_math.txt | 22 ++++++++++++ tests/nemo_text_processing/hi/test_math.py | 35 +++++++++++++++++++ 5 files changed, 125 insertions(+), 6 deletions(-) create mode 100644 nemo_text_processing/text_normalization/hi/data/measure/math_operation.tsv create mode 100644 tests/nemo_text_processing/hi/data_text_normalization/test_cases_math.txt create mode 100644 tests/nemo_text_processing/hi/test_math.py diff --git a/nemo_text_processing/text_normalization/hi/data/measure/math_operation.tsv b/nemo_text_processing/text_normalization/hi/data/measure/math_operation.tsv new file mode 100644 index 000000000..8fdf49823 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/measure/math_operation.tsv @@ -0,0 +1,9 @@ ++ प्लस +- माइनस +/ बटा +÷ बटा +: बटा +× गुणा +* गुणा +· गुणा += बराबर \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/taggers/measure.py b/nemo_text_processing/text_normalization/hi/taggers/measure.py index 67043b727..bd38bab1e 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/measure.py +++ b/nemo_text_processing/text_normalization/hi/taggers/measure.py @@ -454,6 +454,39 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, ser + pynutil.insert("\"") ) + #Math Graph + math_operations = pynini.string_file(get_abs_path("data/measure/math_operation.tsv")) + delimiter = pynini.accep(" ") | pynutil.insert(" ") + math_alpha = capitalized_input_graph(letters_map) + equals = pynini.cross("=", "बराबर") + + digit_or_zero = digit | zero + fractional_bare = digit_or_zero + pynini.closure(insert_space + digit_or_zero) + + flat_decimal = ( + cardinal_graph + + pynutil.insert(" ") + + pynini.cross(".", "दशमलव") + + pynutil.insert(" ") + + fractional_bare + ) + + operand = cardinal_graph | flat_decimal | math_alpha + + math_expr = operand + pynini.closure(delimiter + math_operations + delimiter + operand) + + math_expr_with_op = operand + pynini.closure(delimiter + math_operations + delimiter + operand, 1) + + math = ( + math_expr_with_op + delimiter + equals + delimiter + math_expr + ) | ( + math_expr + delimiter + equals + delimiter + math_expr_with_op + ) + + graph_math = ( + pynutil.insert('units: "math" cardinal { integer: "') + math + pynutil.insert('" } preserve_order: true') + ) + address_graph = self.get_address_graph(cardinal, ordinal, serial, input_case) structured_address_graph = self.get_structured_address_graph(cardinal, ordinal, input_case) @@ -468,6 +501,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, ser | pynutil.add_weight(graph_savva, -0.1) | pynutil.add_weight(graph_sadhe, -0.1) | pynutil.add_weight(graph_paune, -0.5) + | pynutil.add_weight(graph_math, 0.1) | address_graph | structured_address_graph ) diff --git a/nemo_text_processing/text_normalization/hi/verbalizers/measure.py b/nemo_text_processing/text_normalization/hi/verbalizers/measure.py index cba08057d..fcb8a2d23 100644 --- a/nemo_text_processing/text_normalization/hi/verbalizers/measure.py +++ b/nemo_text_processing/text_normalization/hi/verbalizers/measure.py @@ -42,11 +42,11 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst): ) unit = ( - pynutil.delete("units: \"") - + pynini.difference(pynini.closure(NEMO_NOT_QUOTE, 1), pynini.accep("address")) - + pynutil.delete("\"") - + delete_space - ) + pynutil.delete("units: \"") + + pynini.difference(pynini.closure(NEMO_NOT_QUOTE, 1), pynini.union("address", "math")) + + pynutil.delete("\"") + + delete_space + ) graph_decimal = ( pynutil.delete("decimal {") @@ -71,6 +71,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst): graph = (graph_cardinal | graph_decimal) + delete_space + insert_space + unit preserve_order = pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + delete_space + address = ( pynutil.delete("units: \"address\" ") + delete_space @@ -79,7 +80,25 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst): + pynini.closure(preserve_order) ) - graph |= address + math_cardinal = ( + pynutil.delete("cardinal {") + + delete_space + + pynutil.delete("integer: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + + delete_space + + pynutil.delete("}") + ) + + math_graph = ( + pynutil.delete("units: \"math\" ") + + delete_space + + math_cardinal + + delete_space + + pynini.closure(preserve_order) + ) + + graph |= address | math_graph self.decimal = graph_decimal delete_tokens = self.delete_tokens(graph) diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_math.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_math.txt new file mode 100644 index 000000000..88302fb07 --- /dev/null +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_math.txt @@ -0,0 +1,22 @@ +1-2=5~एक माइनस दो बराबर पाँच +1- 2 = 5~एक माइनस दो बराबर पाँच +y=x +1~वाई बराबर एक्स प्लस एक +x +1 = y~एक्स प्लस एक बराबर वाई +१-२=५~एक माइनस दो बराबर पाँच +१- २ = ५~एक माइनस दो बराबर पाँच +y=x +१~वाई बराबर एक्स प्लस एक +x +१ = y~एक्स प्लस एक बराबर वाई +२ + २ = ४~दो प्लस दो बराबर चार +७ + ३ = १०~सात प्लस तीन बराबर दस +१५ + ५ = २०~पंद्रह प्लस पाँच बराबर बीस +५ - ३ = २~पाँच माइनस तीन बराबर दो +२० - ८ = १२~बीस माइनस आठ बराबर बारह +२ * ३ = ६~दो गुणा तीन बराबर छह +४ * ५ = २०~चार गुणा पाँच बराबर बीस +९ * ३ = २७~नौ गुणा तीन बराबर सत्ताईस +६ / २ = ३~छह बटा दो बराबर तीन +१०० / ४ = २५~एक सौ बटा चार बराबर पच्चीस +२ + ३ + ४ = ९~दो प्लस तीन प्लस चार बराबर नौ +१० - २ - ३ = ५~दस माइनस दो माइनस तीन बराबर पाँच +४ = २ + २~चार बराबर दो प्लस दो +100 + 50 = 150~एक सौ प्लस पचास बराबर एक सौ पचास \ No newline at end of file diff --git a/tests/nemo_text_processing/hi/test_math.py b/tests/nemo_text_processing/hi/test_math.py new file mode 100644 index 000000000..f2dc6cfc9 --- /dev/null +++ b/tests/nemo_text_processing/hi/test_math.py @@ -0,0 +1,35 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestMath: + normalizer = Normalizer( + input_case='cased', lang='hi', cache_dir=CACHE_DIR, overwrite_cache=True, post_process=False + ) + inverse_normalizer = InverseNormalizer(lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('hi/data_text_normalization/test_cases_math.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() From 7796efcc96bf43c5acab7045f942f98af6c416fb Mon Sep 17 00:00:00 2001 From: Shreyas Pawar Date: Fri, 11 Sep 2026 11:14:48 +0000 Subject: [PATCH 2/5] jenkinsfile data update Signed-off-by: Shreyas Pawar --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 39972c461..cebe1fddd 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -29,7 +29,7 @@ pipeline { MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1' JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1' KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6' - KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-29-26-1' + HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-11-26-0' DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' } stages { From 626a2ac77df810dc86d63f4b6d66bcb756075805 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 11 Sep 2026 11:22:04 +0000 Subject: [PATCH 3/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../text_normalization/hi/taggers/measure.py | 16 +++++----------- .../text_normalization/hi/verbalizers/measure.py | 12 ++++++------ 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/nemo_text_processing/text_normalization/hi/taggers/measure.py b/nemo_text_processing/text_normalization/hi/taggers/measure.py index bd38bab1e..cbfe145a0 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/measure.py +++ b/nemo_text_processing/text_normalization/hi/taggers/measure.py @@ -454,7 +454,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, ser + pynutil.insert("\"") ) - #Math Graph + # Math Graph math_operations = pynini.string_file(get_abs_path("data/measure/math_operation.tsv")) delimiter = pynini.accep(" ") | pynutil.insert(" ") math_alpha = capitalized_input_graph(letters_map) @@ -464,22 +464,16 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, ser fractional_bare = digit_or_zero + pynini.closure(insert_space + digit_or_zero) flat_decimal = ( - cardinal_graph - + pynutil.insert(" ") - + pynini.cross(".", "दशमलव") - + pynutil.insert(" ") - + fractional_bare + cardinal_graph + pynutil.insert(" ") + pynini.cross(".", "दशमलव") + pynutil.insert(" ") + fractional_bare ) operand = cardinal_graph | flat_decimal | math_alpha - + math_expr = operand + pynini.closure(delimiter + math_operations + delimiter + operand) - + math_expr_with_op = operand + pynini.closure(delimiter + math_operations + delimiter + operand, 1) - math = ( - math_expr_with_op + delimiter + equals + delimiter + math_expr - ) | ( + math = (math_expr_with_op + delimiter + equals + delimiter + math_expr) | ( math_expr + delimiter + equals + delimiter + math_expr_with_op ) diff --git a/nemo_text_processing/text_normalization/hi/verbalizers/measure.py b/nemo_text_processing/text_normalization/hi/verbalizers/measure.py index fcb8a2d23..e92d4e88a 100644 --- a/nemo_text_processing/text_normalization/hi/verbalizers/measure.py +++ b/nemo_text_processing/text_normalization/hi/verbalizers/measure.py @@ -42,11 +42,11 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst): ) unit = ( - pynutil.delete("units: \"") - + pynini.difference(pynini.closure(NEMO_NOT_QUOTE, 1), pynini.union("address", "math")) - + pynutil.delete("\"") - + delete_space - ) + pynutil.delete("units: \"") + + pynini.difference(pynini.closure(NEMO_NOT_QUOTE, 1), pynini.union("address", "math")) + + pynutil.delete("\"") + + delete_space + ) graph_decimal = ( pynutil.delete("decimal {") @@ -71,7 +71,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst): graph = (graph_cardinal | graph_decimal) + delete_space + insert_space + unit preserve_order = pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + delete_space - + address = ( pynutil.delete("units: \"address\" ") + delete_space From 34f00abb9b9a3555d22ab2de3a5eb67ba8c3c84f Mon Sep 17 00:00:00 2001 From: Shreyas Pawar Date: Fri, 11 Sep 2026 11:46:03 +0000 Subject: [PATCH 4/5] test case Signed-off-by: Shreyas Pawar --- .../hi/data_text_normalization/test_cases_math.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_math.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_math.txt index 88302fb07..2f32be474 100644 --- a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_math.txt +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_math.txt @@ -19,4 +19,5 @@ x +१ = y~एक्स प्लस एक बराबर वाई २ + ३ + ४ = ९~दो प्लस तीन प्लस चार बराबर नौ १० - २ - ३ = ५~दस माइनस दो माइनस तीन बराबर पाँच ४ = २ + २~चार बराबर दो प्लस दो -100 + 50 = 150~एक सौ प्लस पचास बराबर एक सौ पचास \ No newline at end of file +100 + 50 = 150~एक सौ प्लस पचास बराबर एक सौ पचास +1-2+5/3*4=5.66666666667~एक माइनस दो प्लस पाँच बटा तीन गुणा चार बराबर पाँच दशमलव छह छह छह छह छह छह छह छह छह छह सात \ No newline at end of file From ed1a91cb76a26915600d93abe1e5040dfbcef2b5 Mon Sep 17 00:00:00 2001 From: Shreyas Pawar Date: Tue, 15 Sep 2026 12:10:24 +0000 Subject: [PATCH 5/5] added math in SH tests Signed-off-by: Shreyas Pawar --- Jenkinsfile | 2 +- .../hi/test_sparrowhawk_normalization.sh | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index cebe1fddd..e91d0b64d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -29,7 +29,7 @@ pipeline { MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1' JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1' KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6' - HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-11-26-0' + HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-15-26-0' DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' } stages { diff --git a/tests/nemo_text_processing/hi/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/hi/test_sparrowhawk_normalization.sh index 74e1cf9c9..66f5d3d7c 100644 --- a/tests/nemo_text_processing/hi/test_sparrowhawk_normalization.sh +++ b/tests/nemo_text_processing/hi/test_sparrowhawk_normalization.sh @@ -112,10 +112,10 @@ testTNAddress() { runtest $input } -#testTNMath() { -# input=$PROJECT_DIR/en/data_text_normalization/test_cases_math.txt -# runtest $input -#} +testTNMath() { + input=$PROJECT_DIR/hi/data_text_normalization/test_cases_math.txt + runtest $input +} # Load shUnit2 . $PROJECT_DIR/../shunit2/shunit2