From 66ee78ee07beb10d1b18e7248c02a1640cefb955 Mon Sep 17 00:00:00 2001 From: Shreyas Pawar Date: Wed, 2 Sep 2026 12:02:28 +0000 Subject: [PATCH 1/7] percent and time bug fixes Signed-off-by: Shreyas Pawar --- Jenkinsfile | 2 +- .../hi/data/measure/unit.tsv | 3 +- .../text_normalization/hi/taggers/measure.py | 42 +++++++++++++++++-- .../text_normalization/hi/taggers/time.py | 9 ++-- .../test_cases_measure.txt | 4 +- .../test_cases_time.txt | 4 +- 6 files changed, 53 insertions(+), 11 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 39972c461..a2de3c8c4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -29,7 +29,7 @@ pipeline { MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1' JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1' KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6' - KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-29-26-1' + HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-02-26-0' DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' } stages { diff --git a/nemo_text_processing/text_normalization/hi/data/measure/unit.tsv b/nemo_text_processing/text_normalization/hi/data/measure/unit.tsv index d236dd51b..6598affc0 100644 --- a/nemo_text_processing/text_normalization/hi/data/measure/unit.tsv +++ b/nemo_text_processing/text_normalization/hi/data/measure/unit.tsv @@ -142,4 +142,5 @@ mi/hr मील प्रति घंटा mi/min मील प्रति मिनट ₹/ac रुपए प्रति एकड़ x बाई -* बाई \ No newline at end of file +* बाई +% प्रतिशत \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/taggers/measure.py b/nemo_text_processing/text_normalization/hi/taggers/measure.py index 67043b727..cf8068b42 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/measure.py +++ b/nemo_text_processing/text_normalization/hi/taggers/measure.py @@ -228,9 +228,22 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, ser year_informal = pynini.string_map([("yr", "साल")]) year_formal = pynini.string_file(get_abs_path("data/measure/unit_year_formal.tsv")) - # All units EXCEPT year - unit_inputs_except_yr = pynini.difference(pynini.project(unit_graph, "input"), pynini.accep("yr")) - unit_graph_no_year = pynini.compose(unit_inputs_except_yr, unit_graph) + # All units EXCEPT year and percent + unit_inputs_regular = pynini.difference( + pynini.project(unit_graph, "input"), + pynini.union(pynini.accep("yr"), pynini.accep("%")) + ) + unit_graph_no_year = pynini.compose(unit_inputs_regular, unit_graph) + + percent_graph = pynini.compose(pynini.accep("%"), unit_graph) + + percent_unit = ( + pynutil.insert(NEMO_SPACE) + + pynutil.insert("units: \"") + + percent_graph + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + ) # Load quarterly units from separate files: map (FST) and list (FSA) quarterly_units_map = pynini.string_file(get_abs_path("data/measure/quarterly_units_map.tsv")) @@ -454,14 +467,37 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, ser + pynutil.insert("\"") ) + graph_cardinal_percent = ( + pynutil.insert("cardinal { ") + + optional_graph_negative + + pynutil.insert("integer: \"") + + cardinal_graph + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + + pynutil.insert("}") + + pynini.closure(delete_space, 0, 1) + + percent_unit + ) + + graph_decimal_percent = ( + pynutil.insert("decimal { ") + + optional_graph_negative + + decimal_graph + + pynutil.insert(" }") + + pynini.closure(delete_space, 0, 1) + + percent_unit + ) + address_graph = self.get_address_graph(cardinal, ordinal, serial, input_case) structured_address_graph = self.get_structured_address_graph(cardinal, ordinal, input_case) graph = ( pynutil.add_weight(graph_decimal, 0.1) | pynutil.add_weight(graph_decimal_year_formal, 0.1) + | pynutil.add_weight(graph_decimal_percent, 0.1) | pynutil.add_weight(graph_cardinal, 0.1) | pynutil.add_weight(graph_cardinal_year_formal, 0.1) + | pynutil.add_weight(graph_cardinal_percent, 0.1) | pynutil.add_weight(graph_cardinal_year_informal, -0.1) # Higher priority for small numbers | pynutil.add_weight(graph_exceptions, 0.1) | pynutil.add_weight(graph_dedh_dhai, -0.2) diff --git a/nemo_text_processing/text_normalization/hi/taggers/time.py b/nemo_text_processing/text_normalization/hi/taggers/time.py index fc598c2b1..68568aa60 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/time.py +++ b/nemo_text_processing/text_normalization/hi/taggers/time.py @@ -55,9 +55,10 @@ def __init__(self, cardinal: GraphFst): super().__init__(name="time", kind="classify") delete_colon = pynutil.delete(":") - cardinal_graph = cardinal.digit | cardinal.teens_and_ties + delete_leading_zero = pynini.closure(pynutil.delete("0") | pynutil.delete("०"), 0, 1) + cardinal_graph = delete_leading_zero + (cardinal.digit | cardinal.teens_and_ties) - self.hours = pynutil.insert("hours: \"") + hours_graph + pynutil.insert("\" ") + self.hours = pynutil.insert("hours: \"") + delete_leading_zero + hours_graph + pynutil.insert("\" ") self.minutes = pynutil.insert("minutes: \"") + minutes_graph + pynutil.insert("\" ") self.seconds = pynutil.insert("seconds: \"") + seconds_graph + pynutil.insert("\" ") @@ -73,7 +74,7 @@ def __init__(self, cardinal: GraphFst): graph_h = self.hours + delete_colon + pynutil.delete(HI_DOUBLE_ZERO) # Support all combinations of Devanagari and Arabic digits for dedh/dhai patterns - dedh_dhai_graph = pynini.string_map( + dedh_dhai_graph = delete_leading_zero + pynini.string_map( [ ("१:३०", HI_DEDH), ("१:30", HI_DEDH), @@ -92,7 +93,7 @@ def __init__(self, cardinal: GraphFst): sadhe_numbers = cardinal_graph + pynini.cross(HI_TIME_THIRTY, "") sadhe_graph = pynutil.insert(HI_SADHE) + pynutil.insert(NEMO_SPACE) + sadhe_numbers - paune = pynini.string_file(get_abs_path("data/whitelist/paune_mappings.tsv")) + paune = delete_leading_zero + pynini.string_file(get_abs_path("data/whitelist/paune_mappings.tsv")) paune_numbers = paune + pynini.cross(HI_TIME_FORTYFIVE, "") paune_graph = pynutil.insert(HI_PAUNE) + pynutil.insert(NEMO_SPACE) + paune_numbers diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_measure.txt index 6afd66b7f..13f0cda5e 100644 --- a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_measure.txt +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_measure.txt @@ -66,4 +66,6 @@ ५ yr~पाँच साल 1.5 yr~डेढ़ साल २.५ yr~ढाई साल -3.5 yr~साढ़े तीन साल \ No newline at end of file +3.5 yr~साढ़े तीन साल +100%~एक सौ प्रतिशत +५०.५० %~पचास दशमलव पाँच शून्य प्रतिशत \ No newline at end of file diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt index 5bc796209..2e613ddae 100644 --- a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt @@ -15,4 +15,6 @@ दोपहर के 3:00~दोपहर के तीन बजे रात के १०:४८:५०~रात के दस बजकर अड़तालीस मिनट पचास सेकंड रात के 11:50~रात के ग्यारह बजकर पचास मिनट -रात के ८:००~रात के आठ बजे \ No newline at end of file +रात के ८:००~रात के आठ बजे +आज सुबह ०९:१५ बजे~आज सुबह सवा नौ बजे +08:00~आठ बजे \ No newline at end of file From 4e60c63fb34cfbab727c6121f5521bcf2e84eea9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:28:40 +0000 Subject: [PATCH 2/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../text_normalization/hi/taggers/measure.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/nemo_text_processing/text_normalization/hi/taggers/measure.py b/nemo_text_processing/text_normalization/hi/taggers/measure.py index cf8068b42..897f77fac 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/measure.py +++ b/nemo_text_processing/text_normalization/hi/taggers/measure.py @@ -230,13 +230,12 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, ser # All units EXCEPT year and percent unit_inputs_regular = pynini.difference( - pynini.project(unit_graph, "input"), - pynini.union(pynini.accep("yr"), pynini.accep("%")) + pynini.project(unit_graph, "input"), pynini.union(pynini.accep("yr"), pynini.accep("%")) ) unit_graph_no_year = pynini.compose(unit_inputs_regular, unit_graph) percent_graph = pynini.compose(pynini.accep("%"), unit_graph) - + percent_unit = ( pynutil.insert(NEMO_SPACE) + pynutil.insert("units: \"") @@ -475,7 +474,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, ser + pynutil.insert("\"") + pynutil.insert(NEMO_SPACE) + pynutil.insert("}") - + pynini.closure(delete_space, 0, 1) + + pynini.closure(delete_space, 0, 1) + percent_unit ) @@ -484,7 +483,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, ser + optional_graph_negative + decimal_graph + pynutil.insert(" }") - + pynini.closure(delete_space, 0, 1) + + pynini.closure(delete_space, 0, 1) + percent_unit ) From 5f4b14e116a7783ac654766e95f568fdf4b38d60 Mon Sep 17 00:00:00 2001 From: Shreyas Pawar Date: Tue, 8 Sep 2026 10:49:12 +0000 Subject: [PATCH 3/7] merge conflict jenkinsfile resolved Signed-off-by: Shreyas Pawar --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index a2de3c8c4..53faef15b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -29,7 +29,7 @@ pipeline { MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1' JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1' KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6' - HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-02-26-0' + HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-08-26-0' DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' } stages { From 5dd3d20ff4f8b48173e7a7b7ad47e91bed6c2ae3 Mon Sep 17 00:00:00 2001 From: Shreyas Pawar Date: Thu, 17 Sep 2026 13:36:41 +0000 Subject: [PATCH 4/7] review feedback changes and additional test cases Signed-off-by: Shreyas Pawar --- Jenkinsfile | 2 +- .../hi/data/measure/special_units.tsv | 2 + .../hi/data/whitelist/paune_mappings.tsv | 100 ++++++++++++++++++ .../text_normalization/hi/taggers/measure.py | 10 +- .../text_normalization/hi/taggers/time.py | 35 ++++-- .../test_cases_measure.txt | 16 ++- .../test_cases_time.txt | 31 +++++- 7 files changed, 183 insertions(+), 13 deletions(-) create mode 100644 nemo_text_processing/text_normalization/hi/data/measure/special_units.tsv diff --git a/Jenkinsfile b/Jenkinsfile index 53faef15b..6d546ae42 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -29,7 +29,7 @@ pipeline { MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1' JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1' KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6' - HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-08-26-0' + HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-17-26-0' DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' } stages { diff --git a/nemo_text_processing/text_normalization/hi/data/measure/special_units.tsv b/nemo_text_processing/text_normalization/hi/data/measure/special_units.tsv new file mode 100644 index 000000000..1072fad92 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/measure/special_units.tsv @@ -0,0 +1,2 @@ +yr +% diff --git a/nemo_text_processing/text_normalization/hi/data/whitelist/paune_mappings.tsv b/nemo_text_processing/text_normalization/hi/data/whitelist/paune_mappings.tsv index 3477871e4..b21396025 100644 --- a/nemo_text_processing/text_normalization/hi/data/whitelist/paune_mappings.tsv +++ b/nemo_text_processing/text_normalization/hi/data/whitelist/paune_mappings.tsv @@ -98,3 +98,103 @@ ९७ अट्ठानबे ९८ निन्यानबे ९९ एक सौ +0 एक +1 दो +2 तीन +3 चार +4 पाँच +5 छह +6 सात +7 आठ +8 नौ +9 दस +10 ग्यारह +11 बारह +12 तेरह +13 चौदह +14 पंद्रह +15 सोलह +16 सत्रह +17 अठारह +18 उन्नीस +19 बीस +20 इक्कीस +21 बाईस +22 तेईस +23 चौबीस +24 पच्चीस +25 छब्बीस +26 सत्ताईस +27 अट्ठाईस +28 उनतीस +29 तीस +30 इकतीस +31 बत्तीस +32 तैंतीस +33 चौंतीस +34 पैंतीस +35 छत्तीस +36 सैंतीस +37 अड़तीस +38 उनतालीस +39 चालीस +40 इकतालीस +41 बयालीस +42 तैंतालीस +43 चौवालीस +44 पैंतालीस +45 छियालीस +46 सैंतालीस +47 अड़तालीस +48 उनचास +49 पचास +50 इक्यावन +51 बावन +52 तिरेपन +53 चौवन +54 पचपन +55 छप्पन +56 सत्तावन +57 अट्ठावन +58 उनसठ +59 साठ +60 इकसठ +61 बासठ +62 तिरेसठ +63 चौंसठ +64 पैंसठ +65 छियासठ +66 सड़सठ +67 अड़सठ +68 उनहत्तर +69 सत्तर +70 इकहत्तर +71 बहत्तर +72 तिहत्तर +73 चौहत्तर +74 पचहत्तर +75 छिहत्तर +76 सतहत्तर +77 अठहत्तर +78 उनासी +79 अस्सी +80 इक्यासी +81 बयासी +82 तिरासी +83 चौरासी +84 पचासी +85 छियासी +86 सत्तासी +87 अट्ठासी +88 नवासी +89 नब्बे +90 इक्यानबे +91 बानबे +92 तिरानबे +93 चौरानबे +94 पंचानबे +95 छियानबे +96 सत्तानबे +97 अट्ठानबे +98 निन्यानबे +99 एक सौ \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/taggers/measure.py b/nemo_text_processing/text_normalization/hi/taggers/measure.py index 897f77fac..7513aae11 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/measure.py +++ b/nemo_text_processing/text_normalization/hi/taggers/measure.py @@ -228,10 +228,12 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, ser year_informal = pynini.string_map([("yr", "साल")]) year_formal = pynini.string_file(get_abs_path("data/measure/unit_year_formal.tsv")) - # All units EXCEPT year and percent + special_units_fst = pynini.string_file(get_abs_path("data/measure/special_units.tsv")) + unit_inputs_regular = pynini.difference( - pynini.project(unit_graph, "input"), pynini.union(pynini.accep("yr"), pynini.accep("%")) + pynini.project(unit_graph, "input"), pynini.project(special_units_fst, "input") ) + unit_graph_no_year = pynini.compose(unit_inputs_regular, unit_graph) percent_graph = pynini.compose(pynini.accep("%"), unit_graph) @@ -474,7 +476,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, ser + pynutil.insert("\"") + pynutil.insert(NEMO_SPACE) + pynutil.insert("}") - + pynini.closure(delete_space, 0, 1) + + delete_space + percent_unit ) @@ -483,7 +485,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, ser + optional_graph_negative + decimal_graph + pynutil.insert(" }") - + pynini.closure(delete_space, 0, 1) + + delete_space + percent_unit ) diff --git a/nemo_text_processing/text_normalization/hi/taggers/time.py b/nemo_text_processing/text_normalization/hi/taggers/time.py index 68568aa60..d58a8f648 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/time.py +++ b/nemo_text_processing/text_normalization/hi/taggers/time.py @@ -22,6 +22,9 @@ HI_SADHE, HI_SAVVA, NEMO_SPACE, + MIN_NEG_WEIGHT, + NEMO_DIGIT, + NEMO_HI_DIGIT, GraphFst, insert_space, ) @@ -90,7 +93,14 @@ def __init__(self, cardinal: GraphFst): savva_numbers = cardinal_graph + pynini.cross(HI_TIME_FIFTEEN, "") savva_graph = pynutil.insert(HI_SAVVA) + pynutil.insert(NEMO_SPACE) + savva_numbers - sadhe_numbers = cardinal_graph + pynini.cross(HI_TIME_THIRTY, "") + # Restrict 'sadhe' from accepting 1 or 2 so it doesn't conflict with dedh/dhai + exclude_dedh_dhai = pynini.union("1", "१", "01", "०१", "2", "२", "02", "०२").optimize() + + # Project cardinal_graph to an acceptor, subtract exceptions, then compose (@) back to the transducer + valid_sadhe_inputs = pynini.difference(pynini.project(cardinal_graph, "input"), exclude_dedh_dhai) + sadhe_cardinal = valid_sadhe_inputs @ cardinal_graph + + sadhe_numbers = sadhe_cardinal + pynini.cross(HI_TIME_THIRTY, "") sadhe_graph = pynutil.insert(HI_SADHE) + pynutil.insert(NEMO_SPACE) + sadhe_numbers paune = delete_leading_zero + pynini.string_file(get_abs_path("data/whitelist/paune_mappings.tsv")) @@ -125,15 +135,28 @@ def __init__(self, cardinal: GraphFst): + pynutil.insert(NEMO_SPACE) ) - final_graph = ( + arabic_1_2 = pynini.closure(NEMO_DIGIT, 1, 2) + arabic_2 = pynini.closure(NEMO_DIGIT, 2, 2) + arabic_valid_time = arabic_1_2 + pynini.accep(":") + arabic_2 + pynini.closure(pynini.accep(":") + arabic_2, 0, 1) + + deva_1_2 = pynini.closure(NEMO_HI_DIGIT, 1, 2) + deva_2 = pynini.closure(NEMO_HI_DIGIT, 2, 2) + deva_valid_time = deva_1_2 + pynini.accep(":") + deva_2 + pynini.closure(pynini.accep(":") + deva_2, 0, 1) + + valid_time_pattern = pynini.union(arabic_valid_time, deva_valid_time).optimize() + + # 2. Give special patterns a minimal negative weight so they safely beat the fallback graph_hm + unfiltered_graph = ( graph_hms | pynutil.add_weight(graph_hm, 0.3) | pynutil.add_weight(graph_h, 0.3) - | pynutil.add_weight(graph_dedh_dhai, 0.1) - | pynutil.add_weight(graph_savva, 0.2) - | pynutil.add_weight(graph_sadhe, 0.2) - | pynutil.add_weight(graph_paune, 0.1) + | pynutil.add_weight(graph_dedh_dhai, MIN_NEG_WEIGHT) + | pynutil.add_weight(graph_savva, MIN_NEG_WEIGHT) + | pynutil.add_weight(graph_sadhe, MIN_NEG_WEIGHT) + | pynutil.add_weight(graph_paune, MIN_NEG_WEIGHT) ) + final_graph = pynini.compose(valid_time_pattern, unfiltered_graph) + final_graph = self.add_tokens(final_graph) self.fst = final_graph.optimize() diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_measure.txt index 13f0cda5e..b2bc95f88 100644 --- a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_measure.txt +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_measure.txt @@ -68,4 +68,18 @@ २.५ yr~ढाई साल 3.5 yr~साढ़े तीन साल 100%~एक सौ प्रतिशत -५०.५० %~पचास दशमलव पाँच शून्य प्रतिशत \ No newline at end of file +५०.५० %~पचास दशमलव पाँच शून्य प्रतिशत +50%~पचास प्रतिशत +५०%~पचास प्रतिशत +50 %~पचास प्रतिशत +५० %~पचास प्रतिशत +0%~शून्य प्रतिशत +०%~शून्य प्रतिशत +5.5%~पाँच दशमलव पाँच प्रतिशत +५.५%~पाँच दशमलव पाँच प्रतिशत +12.75%~बारह दशमलव सात पाँच प्रतिशत +१२.७५%~बारह दशमलव सात पाँच प्रतिशत +999%~नौ सौ निन्यानबे प्रतिशत +5% से 10% तक~पाँच प्रतिशत से दस प्रतिशत तक +GDP ७.५% बढ़ी~GDP सात दशमलव पाँच प्रतिशत बढ़ी +मुद्रास्फीति 5% रही।~मुद्रास्फीति पाँच प्रतिशत रही । \ No newline at end of file diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt index 2e613ddae..e2ae626fa 100644 --- a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt @@ -17,4 +17,33 @@ रात के 11:50~रात के ग्यारह बजकर पचास मिनट रात के ८:००~रात के आठ बजे आज सुबह ०९:१५ बजे~आज सुबह सवा नौ बजे -08:00~आठ बजे \ No newline at end of file +08:00~आठ बजे +0९:१५~शून्य नौ : पंद्रह +०9:15~शून्य नौ : पंद्रह +0८:४५~शून्य आठ : पैंतालीस +०8:00~शून्य आठ : शून्य शून्य +0१:३०~शून्य एक : तीस +०1:30~शून्य एक : तीस +012:30~शून्य एक दो : तीस +०१२:३०~शून्य एक दो : तीस +010:30~शून्य एक शून्य : तीस +०१०:३०~शून्य एक शून्य : तीस +023:45~शून्य दो तीन : पैंतालीस +०२३:४५~शून्य दो तीन : पैंतालीस +8:45~पौने नौ +08:45~पौने नौ +5:45~पौने छह +05:45~पौने छह +08:30~साढ़े आठ +०८:३०~साढ़े आठ +०८:४५~पौने नौ +01:30~डेढ़ +०१:३०~डेढ़ +02:30~ढाई +०२:३०~ढाई +07:15~सवा सात +०७:१५~सवा सात +09:20~नौ बजकर बीस मिनट +०९:२०~नौ बजकर बीस मिनट +00:00~शून्य बजे +००:००~शून्य बजे \ No newline at end of file From 2f7f9eb4fb7cd004dc10cda910bf89acc78a2b77 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 17 Sep 2026 13:37:30 +0000 Subject: [PATCH 5/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../text_normalization/hi/taggers/measure.py | 4 ++-- .../text_normalization/hi/taggers/time.py | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/nemo_text_processing/text_normalization/hi/taggers/measure.py b/nemo_text_processing/text_normalization/hi/taggers/measure.py index 7513aae11..97d4d32ac 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/measure.py +++ b/nemo_text_processing/text_normalization/hi/taggers/measure.py @@ -229,11 +229,11 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, ser year_formal = pynini.string_file(get_abs_path("data/measure/unit_year_formal.tsv")) special_units_fst = pynini.string_file(get_abs_path("data/measure/special_units.tsv")) - + unit_inputs_regular = pynini.difference( pynini.project(unit_graph, "input"), pynini.project(special_units_fst, "input") ) - + unit_graph_no_year = pynini.compose(unit_inputs_regular, unit_graph) percent_graph = pynini.compose(pynini.accep("%"), unit_graph) diff --git a/nemo_text_processing/text_normalization/hi/taggers/time.py b/nemo_text_processing/text_normalization/hi/taggers/time.py index d58a8f648..cc35fb1f0 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/time.py +++ b/nemo_text_processing/text_normalization/hi/taggers/time.py @@ -21,10 +21,10 @@ HI_PAUNE, HI_SADHE, HI_SAVVA, - NEMO_SPACE, MIN_NEG_WEIGHT, NEMO_DIGIT, NEMO_HI_DIGIT, + NEMO_SPACE, GraphFst, insert_space, ) @@ -95,11 +95,11 @@ def __init__(self, cardinal: GraphFst): # Restrict 'sadhe' from accepting 1 or 2 so it doesn't conflict with dedh/dhai exclude_dedh_dhai = pynini.union("1", "१", "01", "०१", "2", "२", "02", "०२").optimize() - + # Project cardinal_graph to an acceptor, subtract exceptions, then compose (@) back to the transducer valid_sadhe_inputs = pynini.difference(pynini.project(cardinal_graph, "input"), exclude_dedh_dhai) sadhe_cardinal = valid_sadhe_inputs @ cardinal_graph - + sadhe_numbers = sadhe_cardinal + pynini.cross(HI_TIME_THIRTY, "") sadhe_graph = pynutil.insert(HI_SADHE) + pynutil.insert(NEMO_SPACE) + sadhe_numbers @@ -137,7 +137,9 @@ def __init__(self, cardinal: GraphFst): arabic_1_2 = pynini.closure(NEMO_DIGIT, 1, 2) arabic_2 = pynini.closure(NEMO_DIGIT, 2, 2) - arabic_valid_time = arabic_1_2 + pynini.accep(":") + arabic_2 + pynini.closure(pynini.accep(":") + arabic_2, 0, 1) + arabic_valid_time = ( + arabic_1_2 + pynini.accep(":") + arabic_2 + pynini.closure(pynini.accep(":") + arabic_2, 0, 1) + ) deva_1_2 = pynini.closure(NEMO_HI_DIGIT, 1, 2) deva_2 = pynini.closure(NEMO_HI_DIGIT, 2, 2) From 879a73dd6409dd84034ed5515372ef281e7ad898 Mon Sep 17 00:00:00 2001 From: Shreyas Pawar Date: Fri, 18 Sep 2026 12:37:24 +0000 Subject: [PATCH 6/7] review feedback changes 2 Signed-off-by: Shreyas Pawar --- Jenkinsfile | 2 +- .../hi/data/time/exclude_dedh_dhai.tsv | 8 ++++++++ .../text_normalization/hi/data/time/minutes.tsv | 2 ++ .../text_normalization/hi/data/time/seconds.tsv | 2 ++ .../text_normalization/hi/taggers/time.py | 14 ++++++++++---- .../hi/data_text_normalization/test_cases_time.txt | 4 +++- 6 files changed, 26 insertions(+), 6 deletions(-) create mode 100644 nemo_text_processing/text_normalization/hi/data/time/exclude_dedh_dhai.tsv diff --git a/Jenkinsfile b/Jenkinsfile index 6d546ae42..04b51c044 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -29,7 +29,7 @@ pipeline { MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1' JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1' KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6' - HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-17-26-0' + HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-18-26-0' DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' } stages { diff --git a/nemo_text_processing/text_normalization/hi/data/time/exclude_dedh_dhai.tsv b/nemo_text_processing/text_normalization/hi/data/time/exclude_dedh_dhai.tsv new file mode 100644 index 000000000..7584b18ff --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/time/exclude_dedh_dhai.tsv @@ -0,0 +1,8 @@ +१ १ +२ २ +०१ ०१ +०२ ०२ +1 1 +2 2 +01 01 +02 02 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/time/minutes.tsv b/nemo_text_processing/text_normalization/hi/data/time/minutes.tsv index 895ab481a..85b9cad4a 100644 --- a/nemo_text_processing/text_normalization/hi/data/time/minutes.tsv +++ b/nemo_text_processing/text_normalization/hi/data/time/minutes.tsv @@ -1,3 +1,4 @@ +०० शून्य ०१ एक ०२ दो ०३ तीन @@ -58,6 +59,7 @@ ५८ अट्ठावन ५९ उनसठ ६० साठ +00 शून्य 01 एक 02 दो 03 तीन diff --git a/nemo_text_processing/text_normalization/hi/data/time/seconds.tsv b/nemo_text_processing/text_normalization/hi/data/time/seconds.tsv index 895ab481a..85b9cad4a 100644 --- a/nemo_text_processing/text_normalization/hi/data/time/seconds.tsv +++ b/nemo_text_processing/text_normalization/hi/data/time/seconds.tsv @@ -1,3 +1,4 @@ +०० शून्य ०१ एक ०२ दो ०३ तीन @@ -58,6 +59,7 @@ ५८ अट्ठावन ५९ उनसठ ६० साठ +00 शून्य 01 एक 02 दो 03 तीन diff --git a/nemo_text_processing/text_normalization/hi/taggers/time.py b/nemo_text_processing/text_normalization/hi/taggers/time.py index cc35fb1f0..1df6b7f6d 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/time.py +++ b/nemo_text_processing/text_normalization/hi/taggers/time.py @@ -65,13 +65,18 @@ def __init__(self, cardinal: GraphFst): self.minutes = pynutil.insert("minutes: \"") + minutes_graph + pynutil.insert("\" ") self.seconds = pynutil.insert("seconds: \"") + seconds_graph + pynutil.insert("\" ") - # hour minute seconds + # hour minute seconds (allows 00 minutes and 00 seconds) graph_hms = ( self.hours + delete_colon + insert_space + self.minutes + delete_colon + insert_space + self.seconds ) + # Restrict graph_hm from accepting 00 minutes so H:00 falls back to graph_h + exclude_double_zero = pynini.union("00", "००").optimize() + minutes_no_zero_graph = pynini.difference(pynini.project(minutes_graph, "input"), exclude_double_zero) @ minutes_graph + hm_minutes_restricted = pynutil.insert("minutes: \"") + minutes_no_zero_graph + pynutil.insert("\" ") + # hour minute - graph_hm = self.hours + delete_colon + insert_space + self.minutes + graph_hm = self.hours + delete_colon + insert_space + hm_minutes_restricted # hour graph_h = self.hours + delete_colon + pynutil.delete(HI_DOUBLE_ZERO) @@ -94,8 +99,9 @@ def __init__(self, cardinal: GraphFst): savva_graph = pynutil.insert(HI_SAVVA) + pynutil.insert(NEMO_SPACE) + savva_numbers # Restrict 'sadhe' from accepting 1 or 2 so it doesn't conflict with dedh/dhai - exclude_dedh_dhai = pynini.union("1", "१", "01", "०१", "2", "२", "02", "०२").optimize() - + exclude_tsv = pynini.string_file(get_abs_path("data/time/exclude_dedh_dhai.tsv")) + exclude_dedh_dhai = pynini.project(exclude_tsv, "input").optimize() + # Project cardinal_graph to an acceptor, subtract exceptions, then compose (@) back to the transducer valid_sadhe_inputs = pynini.difference(pynini.project(cardinal_graph, "input"), exclude_dedh_dhai) sadhe_cardinal = valid_sadhe_inputs @ cardinal_graph diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt index e2ae626fa..24e90dd90 100644 --- a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt @@ -46,4 +46,6 @@ 09:20~नौ बजकर बीस मिनट ०९:२०~नौ बजकर बीस मिनट 00:00~शून्य बजे -००:००~शून्य बजे \ No newline at end of file +००:००~शून्य बजे +10:30:00~दस बजकर तीस मिनट शून्य सेकंड +12:00:00~बारह बजकर शून्य मिनट शून्य सेकंड \ No newline at end of file From 09350333063c4cf78cc2e6352deaa9b53c424a47 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2026 12:44:03 +0000 Subject: [PATCH 7/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../text_normalization/hi/taggers/time.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/nemo_text_processing/text_normalization/hi/taggers/time.py b/nemo_text_processing/text_normalization/hi/taggers/time.py index 1df6b7f6d..c840622bb 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/time.py +++ b/nemo_text_processing/text_normalization/hi/taggers/time.py @@ -65,14 +65,16 @@ def __init__(self, cardinal: GraphFst): self.minutes = pynutil.insert("minutes: \"") + minutes_graph + pynutil.insert("\" ") self.seconds = pynutil.insert("seconds: \"") + seconds_graph + pynutil.insert("\" ") - # hour minute seconds (allows 00 minutes and 00 seconds) + # hour minute seconds (allows 00 minutes and 00 seconds) graph_hms = ( self.hours + delete_colon + insert_space + self.minutes + delete_colon + insert_space + self.seconds ) # Restrict graph_hm from accepting 00 minutes so H:00 falls back to graph_h exclude_double_zero = pynini.union("00", "००").optimize() - minutes_no_zero_graph = pynini.difference(pynini.project(minutes_graph, "input"), exclude_double_zero) @ minutes_graph + minutes_no_zero_graph = ( + pynini.difference(pynini.project(minutes_graph, "input"), exclude_double_zero) @ minutes_graph + ) hm_minutes_restricted = pynutil.insert("minutes: \"") + minutes_no_zero_graph + pynutil.insert("\" ") # hour minute @@ -101,7 +103,7 @@ def __init__(self, cardinal: GraphFst): # Restrict 'sadhe' from accepting 1 or 2 so it doesn't conflict with dedh/dhai exclude_tsv = pynini.string_file(get_abs_path("data/time/exclude_dedh_dhai.tsv")) exclude_dedh_dhai = pynini.project(exclude_tsv, "input").optimize() - + # Project cardinal_graph to an acceptor, subtract exceptions, then compose (@) back to the transducer valid_sadhe_inputs = pynini.difference(pynini.project(cardinal_graph, "input"), exclude_dedh_dhai) sadhe_cardinal = valid_sadhe_inputs @ cardinal_graph