diff --git a/Jenkinsfile b/Jenkinsfile index ce326e63f..04b51c044 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -29,7 +29,7 @@ pipeline { MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1' JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1' KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6' - HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-04-26-0' + HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-18-26-0' DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' } stages { diff --git a/nemo_text_processing/text_normalization/hi/data/measure/special_units.tsv b/nemo_text_processing/text_normalization/hi/data/measure/special_units.tsv new file mode 100644 index 000000000..1072fad92 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/measure/special_units.tsv @@ -0,0 +1,2 @@ +yr +% diff --git a/nemo_text_processing/text_normalization/hi/data/measure/unit.tsv b/nemo_text_processing/text_normalization/hi/data/measure/unit.tsv index d236dd51b..6598affc0 100644 --- a/nemo_text_processing/text_normalization/hi/data/measure/unit.tsv +++ b/nemo_text_processing/text_normalization/hi/data/measure/unit.tsv @@ -142,4 +142,5 @@ mi/hr मील प्रति घंटा mi/min मील प्रति मिनट ₹/ac रुपए प्रति एकड़ x बाई -* बाई \ No newline at end of file +* बाई +% प्रतिशत \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/time/exclude_dedh_dhai.tsv b/nemo_text_processing/text_normalization/hi/data/time/exclude_dedh_dhai.tsv new file mode 100644 index 000000000..7584b18ff --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/time/exclude_dedh_dhai.tsv @@ -0,0 +1,8 @@ +१ १ +२ २ +०१ ०१ +०२ ०२ +1 1 +2 2 +01 01 +02 02 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/time/minutes.tsv b/nemo_text_processing/text_normalization/hi/data/time/minutes.tsv index 895ab481a..85b9cad4a 100644 --- a/nemo_text_processing/text_normalization/hi/data/time/minutes.tsv +++ b/nemo_text_processing/text_normalization/hi/data/time/minutes.tsv @@ -1,3 +1,4 @@ +०० शून्य ०१ एक ०२ दो ०३ तीन @@ -58,6 +59,7 @@ ५८ अट्ठावन ५९ उनसठ ६० साठ +00 शून्य 01 एक 02 दो 03 तीन diff --git a/nemo_text_processing/text_normalization/hi/data/time/seconds.tsv b/nemo_text_processing/text_normalization/hi/data/time/seconds.tsv index 895ab481a..85b9cad4a 100644 --- a/nemo_text_processing/text_normalization/hi/data/time/seconds.tsv +++ b/nemo_text_processing/text_normalization/hi/data/time/seconds.tsv @@ -1,3 +1,4 @@ +०० शून्य ०१ एक ०२ दो ०३ तीन @@ -58,6 +59,7 @@ ५८ अट्ठावन ५९ उनसठ ६० साठ +00 शून्य 01 एक 02 दो 03 तीन diff --git a/nemo_text_processing/text_normalization/hi/data/whitelist/paune_mappings.tsv b/nemo_text_processing/text_normalization/hi/data/whitelist/paune_mappings.tsv index 3477871e4..b21396025 100644 --- a/nemo_text_processing/text_normalization/hi/data/whitelist/paune_mappings.tsv +++ b/nemo_text_processing/text_normalization/hi/data/whitelist/paune_mappings.tsv @@ -98,3 +98,103 @@ ९७ अट्ठानबे ९८ निन्यानबे ९९ एक सौ +0 एक +1 दो +2 तीन +3 चार +4 पाँच +5 छह +6 सात +7 आठ +8 नौ +9 दस +10 ग्यारह +11 बारह +12 तेरह +13 चौदह +14 पंद्रह +15 सोलह +16 सत्रह +17 अठारह +18 उन्नीस +19 बीस +20 इक्कीस +21 बाईस +22 तेईस +23 चौबीस +24 पच्चीस +25 छब्बीस +26 सत्ताईस +27 अट्ठाईस +28 उनतीस +29 तीस +30 इकतीस +31 बत्तीस +32 तैंतीस +33 चौंतीस +34 पैंतीस +35 छत्तीस +36 सैंतीस +37 अड़तीस +38 उनतालीस +39 चालीस +40 इकतालीस +41 बयालीस +42 तैंतालीस +43 चौवालीस +44 पैंतालीस +45 छियालीस +46 सैंतालीस +47 अड़तालीस +48 उनचास +49 पचास +50 इक्यावन +51 बावन +52 तिरेपन +53 चौवन +54 पचपन +55 छप्पन +56 सत्तावन +57 अट्ठावन +58 उनसठ +59 साठ +60 इकसठ +61 बासठ +62 तिरेसठ +63 चौंसठ +64 पैंसठ +65 छियासठ +66 सड़सठ +67 अड़सठ +68 उनहत्तर +69 सत्तर +70 इकहत्तर +71 बहत्तर +72 तिहत्तर +73 चौहत्तर +74 पचहत्तर +75 छिहत्तर +76 सतहत्तर +77 अठहत्तर +78 उनासी +79 अस्सी +80 इक्यासी +81 बयासी +82 तिरासी +83 चौरासी +84 पचासी +85 छियासी +86 सत्तासी +87 अट्ठासी +88 नवासी +89 नब्बे +90 इक्यानबे +91 बानबे +92 तिरानबे +93 चौरानबे +94 पंचानबे +95 छियानबे +96 सत्तानबे +97 अट्ठानबे +98 निन्यानबे +99 एक सौ \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/taggers/measure.py b/nemo_text_processing/text_normalization/hi/taggers/measure.py index 67043b727..97d4d32ac 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/measure.py +++ b/nemo_text_processing/text_normalization/hi/taggers/measure.py @@ -228,9 +228,23 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, ser year_informal = pynini.string_map([("yr", "साल")]) year_formal = pynini.string_file(get_abs_path("data/measure/unit_year_formal.tsv")) - # All units EXCEPT year - unit_inputs_except_yr = pynini.difference(pynini.project(unit_graph, "input"), pynini.accep("yr")) - unit_graph_no_year = pynini.compose(unit_inputs_except_yr, unit_graph) + special_units_fst = pynini.string_file(get_abs_path("data/measure/special_units.tsv")) + + unit_inputs_regular = pynini.difference( + pynini.project(unit_graph, "input"), pynini.project(special_units_fst, "input") + ) + + unit_graph_no_year = pynini.compose(unit_inputs_regular, unit_graph) + + percent_graph = pynini.compose(pynini.accep("%"), unit_graph) + + percent_unit = ( + pynutil.insert(NEMO_SPACE) + + pynutil.insert("units: \"") + + percent_graph + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + ) # Load quarterly units from separate files: map (FST) and list (FSA) quarterly_units_map = pynini.string_file(get_abs_path("data/measure/quarterly_units_map.tsv")) @@ -454,14 +468,37 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, ser + pynutil.insert("\"") ) + graph_cardinal_percent = ( + pynutil.insert("cardinal { ") + + optional_graph_negative + + pynutil.insert("integer: \"") + + cardinal_graph + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + + pynutil.insert("}") + + delete_space + + percent_unit + ) + + graph_decimal_percent = ( + pynutil.insert("decimal { ") + + optional_graph_negative + + decimal_graph + + pynutil.insert(" }") + + delete_space + + percent_unit + ) + address_graph = self.get_address_graph(cardinal, ordinal, serial, input_case) structured_address_graph = self.get_structured_address_graph(cardinal, ordinal, input_case) graph = ( pynutil.add_weight(graph_decimal, 0.1) | pynutil.add_weight(graph_decimal_year_formal, 0.1) + | pynutil.add_weight(graph_decimal_percent, 0.1) | pynutil.add_weight(graph_cardinal, 0.1) | pynutil.add_weight(graph_cardinal_year_formal, 0.1) + | pynutil.add_weight(graph_cardinal_percent, 0.1) | pynutil.add_weight(graph_cardinal_year_informal, -0.1) # Higher priority for small numbers | pynutil.add_weight(graph_exceptions, 0.1) | pynutil.add_weight(graph_dedh_dhai, -0.2) diff --git a/nemo_text_processing/text_normalization/hi/taggers/time.py b/nemo_text_processing/text_normalization/hi/taggers/time.py index fc598c2b1..c840622bb 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/time.py +++ b/nemo_text_processing/text_normalization/hi/taggers/time.py @@ -21,6 +21,9 @@ HI_PAUNE, HI_SADHE, HI_SAVVA, + MIN_NEG_WEIGHT, + NEMO_DIGIT, + NEMO_HI_DIGIT, NEMO_SPACE, GraphFst, insert_space, @@ -55,25 +58,33 @@ def __init__(self, cardinal: GraphFst): super().__init__(name="time", kind="classify") delete_colon = pynutil.delete(":") - cardinal_graph = cardinal.digit | cardinal.teens_and_ties + delete_leading_zero = pynini.closure(pynutil.delete("0") | pynutil.delete("०"), 0, 1) + cardinal_graph = delete_leading_zero + (cardinal.digit | cardinal.teens_and_ties) - self.hours = pynutil.insert("hours: \"") + hours_graph + pynutil.insert("\" ") + self.hours = pynutil.insert("hours: \"") + delete_leading_zero + hours_graph + pynutil.insert("\" ") self.minutes = pynutil.insert("minutes: \"") + minutes_graph + pynutil.insert("\" ") self.seconds = pynutil.insert("seconds: \"") + seconds_graph + pynutil.insert("\" ") - # hour minute seconds + # hour minute seconds (allows 00 minutes and 00 seconds) graph_hms = ( self.hours + delete_colon + insert_space + self.minutes + delete_colon + insert_space + self.seconds ) + # Restrict graph_hm from accepting 00 minutes so H:00 falls back to graph_h + exclude_double_zero = pynini.union("00", "००").optimize() + minutes_no_zero_graph = ( + pynini.difference(pynini.project(minutes_graph, "input"), exclude_double_zero) @ minutes_graph + ) + hm_minutes_restricted = pynutil.insert("minutes: \"") + minutes_no_zero_graph + pynutil.insert("\" ") + # hour minute - graph_hm = self.hours + delete_colon + insert_space + self.minutes + graph_hm = self.hours + delete_colon + insert_space + hm_minutes_restricted # hour graph_h = self.hours + delete_colon + pynutil.delete(HI_DOUBLE_ZERO) # Support all combinations of Devanagari and Arabic digits for dedh/dhai patterns - dedh_dhai_graph = pynini.string_map( + dedh_dhai_graph = delete_leading_zero + pynini.string_map( [ ("१:३०", HI_DEDH), ("१:30", HI_DEDH), @@ -89,10 +100,18 @@ def __init__(self, cardinal: GraphFst): savva_numbers = cardinal_graph + pynini.cross(HI_TIME_FIFTEEN, "") savva_graph = pynutil.insert(HI_SAVVA) + pynutil.insert(NEMO_SPACE) + savva_numbers - sadhe_numbers = cardinal_graph + pynini.cross(HI_TIME_THIRTY, "") + # Restrict 'sadhe' from accepting 1 or 2 so it doesn't conflict with dedh/dhai + exclude_tsv = pynini.string_file(get_abs_path("data/time/exclude_dedh_dhai.tsv")) + exclude_dedh_dhai = pynini.project(exclude_tsv, "input").optimize() + + # Project cardinal_graph to an acceptor, subtract exceptions, then compose (@) back to the transducer + valid_sadhe_inputs = pynini.difference(pynini.project(cardinal_graph, "input"), exclude_dedh_dhai) + sadhe_cardinal = valid_sadhe_inputs @ cardinal_graph + + sadhe_numbers = sadhe_cardinal + pynini.cross(HI_TIME_THIRTY, "") sadhe_graph = pynutil.insert(HI_SADHE) + pynutil.insert(NEMO_SPACE) + sadhe_numbers - paune = pynini.string_file(get_abs_path("data/whitelist/paune_mappings.tsv")) + paune = delete_leading_zero + pynini.string_file(get_abs_path("data/whitelist/paune_mappings.tsv")) paune_numbers = paune + pynini.cross(HI_TIME_FORTYFIVE, "") paune_graph = pynutil.insert(HI_PAUNE) + pynutil.insert(NEMO_SPACE) + paune_numbers @@ -124,15 +143,30 @@ def __init__(self, cardinal: GraphFst): + pynutil.insert(NEMO_SPACE) ) - final_graph = ( + arabic_1_2 = pynini.closure(NEMO_DIGIT, 1, 2) + arabic_2 = pynini.closure(NEMO_DIGIT, 2, 2) + arabic_valid_time = ( + arabic_1_2 + pynini.accep(":") + arabic_2 + pynini.closure(pynini.accep(":") + arabic_2, 0, 1) + ) + + deva_1_2 = pynini.closure(NEMO_HI_DIGIT, 1, 2) + deva_2 = pynini.closure(NEMO_HI_DIGIT, 2, 2) + deva_valid_time = deva_1_2 + pynini.accep(":") + deva_2 + pynini.closure(pynini.accep(":") + deva_2, 0, 1) + + valid_time_pattern = pynini.union(arabic_valid_time, deva_valid_time).optimize() + + # 2. Give special patterns a minimal negative weight so they safely beat the fallback graph_hm + unfiltered_graph = ( graph_hms | pynutil.add_weight(graph_hm, 0.3) | pynutil.add_weight(graph_h, 0.3) - | pynutil.add_weight(graph_dedh_dhai, 0.1) - | pynutil.add_weight(graph_savva, 0.2) - | pynutil.add_weight(graph_sadhe, 0.2) - | pynutil.add_weight(graph_paune, 0.1) + | pynutil.add_weight(graph_dedh_dhai, MIN_NEG_WEIGHT) + | pynutil.add_weight(graph_savva, MIN_NEG_WEIGHT) + | pynutil.add_weight(graph_sadhe, MIN_NEG_WEIGHT) + | pynutil.add_weight(graph_paune, MIN_NEG_WEIGHT) ) + final_graph = pynini.compose(valid_time_pattern, unfiltered_graph) + final_graph = self.add_tokens(final_graph) self.fst = final_graph.optimize() diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_measure.txt index 6afd66b7f..b2bc95f88 100644 --- a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_measure.txt +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_measure.txt @@ -66,4 +66,20 @@ ५ yr~पाँच साल 1.5 yr~डेढ़ साल २.५ yr~ढाई साल -3.5 yr~साढ़े तीन साल \ No newline at end of file +3.5 yr~साढ़े तीन साल +100%~एक सौ प्रतिशत +५०.५० %~पचास दशमलव पाँच शून्य प्रतिशत +50%~पचास प्रतिशत +५०%~पचास प्रतिशत +50 %~पचास प्रतिशत +५० %~पचास प्रतिशत +0%~शून्य प्रतिशत +०%~शून्य प्रतिशत +5.5%~पाँच दशमलव पाँच प्रतिशत +५.५%~पाँच दशमलव पाँच प्रतिशत +12.75%~बारह दशमलव सात पाँच प्रतिशत +१२.७५%~बारह दशमलव सात पाँच प्रतिशत +999%~नौ सौ निन्यानबे प्रतिशत +5% से 10% तक~पाँच प्रतिशत से दस प्रतिशत तक +GDP ७.५% बढ़ी~GDP सात दशमलव पाँच प्रतिशत बढ़ी +मुद्रास्फीति 5% रही।~मुद्रास्फीति पाँच प्रतिशत रही । \ No newline at end of file diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt index 5bc796209..24e90dd90 100644 --- a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt @@ -15,4 +15,37 @@ दोपहर के 3:00~दोपहर के तीन बजे रात के १०:४८:५०~रात के दस बजकर अड़तालीस मिनट पचास सेकंड रात के 11:50~रात के ग्यारह बजकर पचास मिनट -रात के ८:००~रात के आठ बजे \ No newline at end of file +रात के ८:००~रात के आठ बजे +आज सुबह ०९:१५ बजे~आज सुबह सवा नौ बजे +08:00~आठ बजे +0९:१५~शून्य नौ : पंद्रह +०9:15~शून्य नौ : पंद्रह +0८:४५~शून्य आठ : पैंतालीस +०8:00~शून्य आठ : शून्य शून्य +0१:३०~शून्य एक : तीस +०1:30~शून्य एक : तीस +012:30~शून्य एक दो : तीस +०१२:३०~शून्य एक दो : तीस +010:30~शून्य एक शून्य : तीस +०१०:३०~शून्य एक शून्य : तीस +023:45~शून्य दो तीन : पैंतालीस +०२३:४५~शून्य दो तीन : पैंतालीस +8:45~पौने नौ +08:45~पौने नौ +5:45~पौने छह +05:45~पौने छह +08:30~साढ़े आठ +०८:३०~साढ़े आठ +०८:४५~पौने नौ +01:30~डेढ़ +०१:३०~डेढ़ +02:30~ढाई +०२:३०~ढाई +07:15~सवा सात +०७:१५~सवा सात +09:20~नौ बजकर बीस मिनट +०९:२०~नौ बजकर बीस मिनट +00:00~शून्य बजे +००:००~शून्य बजे +10:30:00~दस बजकर तीस मिनट शून्य सेकंड +12:00:00~बारह बजकर शून्य मिनट शून्य सेकंड \ No newline at end of file