diff --git a/Jenkinsfile b/Jenkinsfile index ce326e63f..e91d0b64d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -29,7 +29,7 @@ pipeline { MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1' JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1' KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6' - HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-04-26-0' + HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-15-26-0' DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' } stages { diff --git a/nemo_text_processing/text_normalization/hi/data/measure/math_operation.tsv b/nemo_text_processing/text_normalization/hi/data/measure/math_operation.tsv new file mode 100644 index 000000000..8fdf49823 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/measure/math_operation.tsv @@ -0,0 +1,9 @@ ++ प्लस +- माइनस +/ बटा +÷ बटा +: बटा +× गुणा +* गुणा +· गुणा += बराबर \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/taggers/measure.py b/nemo_text_processing/text_normalization/hi/taggers/measure.py index 67043b727..cbfe145a0 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/measure.py +++ b/nemo_text_processing/text_normalization/hi/taggers/measure.py @@ -454,6 +454,33 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, ser + pynutil.insert("\"") ) + # Math Graph + math_operations = pynini.string_file(get_abs_path("data/measure/math_operation.tsv")) + delimiter = pynini.accep(" ") | pynutil.insert(" ") + math_alpha = capitalized_input_graph(letters_map) + equals = pynini.cross("=", "बराबर") + + digit_or_zero = digit | zero + fractional_bare = digit_or_zero + pynini.closure(insert_space + digit_or_zero) + + flat_decimal = ( + cardinal_graph + pynutil.insert(" ") + pynini.cross(".", "दशमलव") + pynutil.insert(" ") + fractional_bare + ) + + operand = cardinal_graph | flat_decimal | math_alpha + + math_expr = operand + pynini.closure(delimiter + math_operations + delimiter + operand) + + math_expr_with_op = operand + pynini.closure(delimiter + math_operations + delimiter + operand, 1) + + math = (math_expr_with_op + delimiter + equals + delimiter + math_expr) | ( + math_expr + delimiter + equals + delimiter + math_expr_with_op + ) + + graph_math = ( + pynutil.insert('units: "math" cardinal { integer: "') + math + pynutil.insert('" } preserve_order: true') + ) + address_graph = self.get_address_graph(cardinal, ordinal, serial, input_case) structured_address_graph = self.get_structured_address_graph(cardinal, ordinal, input_case) @@ -468,6 +495,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, ser | pynutil.add_weight(graph_savva, -0.1) | pynutil.add_weight(graph_sadhe, -0.1) | pynutil.add_weight(graph_paune, -0.5) + | pynutil.add_weight(graph_math, 0.1) | address_graph | structured_address_graph ) diff --git a/nemo_text_processing/text_normalization/hi/verbalizers/measure.py b/nemo_text_processing/text_normalization/hi/verbalizers/measure.py index cba08057d..e92d4e88a 100644 --- a/nemo_text_processing/text_normalization/hi/verbalizers/measure.py +++ b/nemo_text_processing/text_normalization/hi/verbalizers/measure.py @@ -43,7 +43,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst): unit = ( pynutil.delete("units: \"") - + pynini.difference(pynini.closure(NEMO_NOT_QUOTE, 1), pynini.accep("address")) + + pynini.difference(pynini.closure(NEMO_NOT_QUOTE, 1), pynini.union("address", "math")) + pynutil.delete("\"") + delete_space ) @@ -71,6 +71,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst): graph = (graph_cardinal | graph_decimal) + delete_space + insert_space + unit preserve_order = pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + delete_space + address = ( pynutil.delete("units: \"address\" ") + delete_space @@ -79,7 +80,25 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst): + pynini.closure(preserve_order) ) - graph |= address + math_cardinal = ( + pynutil.delete("cardinal {") + + delete_space + + pynutil.delete("integer: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + + delete_space + + pynutil.delete("}") + ) + + math_graph = ( + pynutil.delete("units: \"math\" ") + + delete_space + + math_cardinal + + delete_space + + pynini.closure(preserve_order) + ) + + graph |= address | math_graph self.decimal = graph_decimal delete_tokens = self.delete_tokens(graph) diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_math.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_math.txt new file mode 100644 index 000000000..2f32be474 --- /dev/null +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_math.txt @@ -0,0 +1,23 @@ +1-2=5~एक माइनस दो बराबर पाँच +1- 2 = 5~एक माइनस दो बराबर पाँच +y=x +1~वाई बराबर एक्स प्लस एक +x +1 = y~एक्स प्लस एक बराबर वाई +१-२=५~एक माइनस दो बराबर पाँच +१- २ = ५~एक माइनस दो बराबर पाँच +y=x +१~वाई बराबर एक्स प्लस एक +x +१ = y~एक्स प्लस एक बराबर वाई +२ + २ = ४~दो प्लस दो बराबर चार +७ + ३ = १०~सात प्लस तीन बराबर दस +१५ + ५ = २०~पंद्रह प्लस पाँच बराबर बीस +५ - ३ = २~पाँच माइनस तीन बराबर दो +२० - ८ = १२~बीस माइनस आठ बराबर बारह +२ * ३ = ६~दो गुणा तीन बराबर छह +४ * ५ = २०~चार गुणा पाँच बराबर बीस +९ * ३ = २७~नौ गुणा तीन बराबर सत्ताईस +६ / २ = ३~छह बटा दो बराबर तीन +१०० / ४ = २५~एक सौ बटा चार बराबर पच्चीस +२ + ३ + ४ = ९~दो प्लस तीन प्लस चार बराबर नौ +१० - २ - ३ = ५~दस माइनस दो माइनस तीन बराबर पाँच +४ = २ + २~चार बराबर दो प्लस दो +100 + 50 = 150~एक सौ प्लस पचास बराबर एक सौ पचास +1-2+5/3*4=5.66666666667~एक माइनस दो प्लस पाँच बटा तीन गुणा चार बराबर पाँच दशमलव छह छह छह छह छह छह छह छह छह छह सात \ No newline at end of file diff --git a/tests/nemo_text_processing/hi/test_math.py b/tests/nemo_text_processing/hi/test_math.py new file mode 100644 index 000000000..f2dc6cfc9 --- /dev/null +++ b/tests/nemo_text_processing/hi/test_math.py @@ -0,0 +1,35 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestMath: + normalizer = Normalizer( + input_case='cased', lang='hi', cache_dir=CACHE_DIR, overwrite_cache=True, post_process=False + ) + inverse_normalizer = InverseNormalizer(lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('hi/data_text_normalization/test_cases_math.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() diff --git a/tests/nemo_text_processing/hi/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/hi/test_sparrowhawk_normalization.sh index 74e1cf9c9..66f5d3d7c 100644 --- a/tests/nemo_text_processing/hi/test_sparrowhawk_normalization.sh +++ b/tests/nemo_text_processing/hi/test_sparrowhawk_normalization.sh @@ -112,10 +112,10 @@ testTNAddress() { runtest $input } -#testTNMath() { -# input=$PROJECT_DIR/en/data_text_normalization/test_cases_math.txt -# runtest $input -#} +testTNMath() { + input=$PROJECT_DIR/hi/data_text_normalization/test_cases_math.txt + runtest $input +} # Load shUnit2 . $PROJECT_DIR/../shunit2/shunit2