diff --git a/nemo_text_processing/text_normalization/hi/data/roman/roman_components.tsv b/nemo_text_processing/text_normalization/hi/data/roman/roman_components.tsv new file mode 100644 index 000000000..b6cbb191f --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/roman/roman_components.tsv @@ -0,0 +1,30 @@ +I 1 +II 2 +III 3 +IV 4 +V 5 +VI 6 +VII 7 +VIII 8 +IX 9 +X 1 +XX 2 +XXX 3 +XL 4 +L 5 +LX 6 +LXX 7 +LXXX 8 +XC 9 +C 1 +CC 2 +CCC 3 +CD 4 +D 5 +DC 6 +DCC 7 +DCCC 8 +CM 9 +M 1 +MM 2 +MMM 3 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/taggers/roman.py b/nemo_text_processing/text_normalization/hi/taggers/roman.py index ea8d259fe..566c28efc 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/roman.py +++ b/nemo_text_processing/text_normalization/hi/taggers/roman.py @@ -32,11 +32,25 @@ class RomanFst(GraphFst): for False multiple transduction are generated (used for audio-based normalization) """ - def __init__(self, deterministic: bool = True): + def __init__(self, cardinal: GraphFst, deterministic: bool = True): super().__init__(name="roman", kind="classify", deterministic=deterministic) - roman_graph = pynini.string_file(get_abs_path("data/roman/roman_to_spoken.tsv")).optimize() - roman_numeral_only = pynini.project(roman_graph, "input").optimize() + components = load_labels(get_abs_path("data/roman/roman_components.tsv")) + ones = pynini.string_map(components[0:9]).optimize() + tens = pynini.string_map(components[9:18]).optimize() + hundreds = pynini.string_map(components[18:27]).optimize() + thousands = pynini.string_map(components[27:30]).optimize() + + zero = pynutil.insert("0") + opt_hundreds = hundreds | zero + opt_tens = tens | zero + opt_ones = ones | zero + + roman_to_arabic = pynini.union( + thousands + opt_hundreds + opt_tens + opt_ones, hundreds + opt_tens + opt_ones, tens + opt_ones, ones + ).optimize() + + roman_to_spoken_fst = pynini.compose(roman_to_arabic, cardinal.graph_without_leading_zeros).optimize() devanagari_chars = pynini.project( pynini.string_file(get_abs_path("data/serial/chars.tsv")), "input" @@ -58,14 +72,14 @@ def __init__(self, deterministic: bool = True): + pynutil.delete(separator) + insert_space + pynutil.insert('integer: "') - + roman_numeral_only + + roman_to_spoken_fst + pynutil.insert('"') ).optimize() numeral_before_key = ( pynutil.insert("preserve_order: true ") + pynutil.insert('integer: "') - + roman_numeral_only + + roman_to_spoken_fst + pynutil.insert('"') + pynutil.delete(separator) + insert_space @@ -74,45 +88,30 @@ def __init__(self, deterministic: bool = True): + pynutil.insert('"') ).optimize() - roman_rows = load_labels(get_abs_path("data/roman/roman_to_spoken.tsv")) - numerals_by_len_desc = sorted((n for n, _ in roman_rows), key=len, reverse=True) - exception_rows = load_labels(get_abs_path("data/roman/roman_ordinal_exceptions.tsv")) - exception_fused_set = {fused for fused, _ in exception_rows} - - suffix_rows_raw = load_labels(get_abs_path("data/ordinal/suffixes.tsv")) + load_labels( - get_abs_path("data/ordinal/suffixes_map.tsv") - ) exception_graphs = [] for fused, spoken_word in exception_rows: - matched_numeral = next(c for c in numerals_by_len_desc if fused.startswith(c)) exception_graphs.append( - pynutil.insert('integer: "' + matched_numeral + '"') + pynutil.insert('integer: "-"') + insert_space + pynutil.insert('default_ordinal: "' + spoken_word + '"') + pynutil.delete(fused) ) glued_ordinal_exceptions_graph = pynini.union(*exception_graphs).optimize() - regular_row_graphs = [] - for numeral, spoken in roman_rows: - for row in suffix_rows_raw: - - suffix_input = row[0] - suffix_output = row[1] if len(row) > 1 else row[0] - - fused = numeral + suffix_input - if fused in exception_fused_set: - continue - spoken_ordinal = spoken + suffix_output - regular_row_graphs.append( - pynutil.insert('integer: "' + numeral + '"') - + insert_space - + pynutil.insert('default_ordinal: "' + spoken_ordinal + '"') - + pynutil.delete(fused) - ) - glued_ordinal_regular_graph = pynini.union(*regular_row_graphs).optimize() + suffixes_fst = pynini.union( + pynini.string_file(get_abs_path("data/ordinal/suffixes.tsv")), + pynini.string_file(get_abs_path("data/ordinal/suffixes_map.tsv")), + ).optimize() + + glued_ordinal_regular_graph = ( + pynutil.insert('integer: "-"') + + insert_space + + pynutil.insert('default_ordinal: "') + + (roman_to_spoken_fst + suffixes_fst) + + pynutil.insert('"') + ).optimize() roman_glued_ordinal_fields = pynini.union( pynutil.add_weight(glued_ordinal_exceptions_graph, -0.1), diff --git a/nemo_text_processing/text_normalization/hi/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/hi/taggers/tokenize_and_classify.py index 04124635e..8e1ec6df2 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/text_normalization/hi/taggers/tokenize_and_classify.py @@ -121,7 +121,7 @@ def __init__( word = WordFst(punctuation=punctuation, deterministic=deterministic) word_graph = word.fst - roman = RomanFst(deterministic=deterministic) + roman = RomanFst(cardinal=cardinal, deterministic=deterministic) roman_graph = roman.fst telephone = TelephoneFst() diff --git a/nemo_text_processing/text_normalization/hi/verbalizers/roman.py b/nemo_text_processing/text_normalization/hi/verbalizers/roman.py index c28084a77..adb6e61c5 100644 --- a/nemo_text_processing/text_normalization/hi/verbalizers/roman.py +++ b/nemo_text_processing/text_normalization/hi/verbalizers/roman.py @@ -40,13 +40,11 @@ class RomanFst(GraphFst): def __init__(self, deterministic: bool = True): super().__init__(name="roman", kind="verbalize", deterministic=deterministic) - roman_to_spoken = pynini.string_file(get_abs_path("data/roman/roman_to_spoken.tsv")).optimize() - key_cardinal = ( pynutil.delete('key_cardinal: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') ).optimize() - integer = (pynutil.delete('integer: "') + roman_to_spoken + pynutil.delete('"')).optimize() + integer = (pynutil.delete('integer: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"')).optimize() default_ordinal = ( pynutil.delete('default_ordinal: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_roman.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_roman.txt index 340c754ed..6b334161c 100644 --- a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_roman.txt +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_roman.txt @@ -19,5 +19,11 @@ XIIवीं कक्षा की परीक्षा~बारहवीं अध्याय IV के प्रश्न~अध्याय चार के प्रश्न IVथी कक्षा के विद्यार्थी~चौथी कक्षा के विद्यार्थी XC विद्यार्थी~नब्बे विद्यार्थी -LIII वा गणतंत्र दिन~तिरपन वा गणतंत्र दिन -भाग-XCIX~भाग निन्यानवे \ No newline at end of file +LIII वा गणतंत्र दिन~तिरेपन वा गणतंत्र दिन +भाग-XCIX~भाग निन्यानबे +भाग-CDXLIV~भाग चार सौ चौवालीस +रॉकेट-MMM~रॉकेट तीन हज़ार +अध्याय MCMXCIX~अध्याय एक हज़ार नौ सौ निन्यानबे +MMXXIVवीं वर्षगांठ~दो हज़ार चौबीसवीं वर्षगांठ +MCMXCIX वा स्थापना दिवस~एक हज़ार नौ सौ निन्यानबे वा स्थापना दिवस +MMMCMXCIX सैनिक~तीन हज़ार नौ सौ निन्यानबे सैनिक