diff --git a/Jenkinsfile b/Jenkinsfile index 39972c461..3fde1f2b1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -30,6 +30,7 @@ pipeline { JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1' KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6' KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-29-26-1' + TA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-16-26-0' DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' } stages { @@ -113,6 +114,30 @@ pipeline { } } + stage('L0: Create TA TN/ITN Grammars') { + when { + anyOf { + branch 'main' + branch 'staging/**' + branch 'staging_*' + changeRequest target: 'main' + } + } + failFast true + parallel { + stage('L0: Ta TN grammars') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=ta --text="௧" --cache_dir ${TA_TN_CACHE}' + } + } + stage('L0: Ta ITN grammars') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=ta --text="ஒன்று" --cache_dir ${TA_TN_CACHE}' + } + } + } + } + stage('L0: Create DE/ES TN/ITN Grammars') { when { anyOf { @@ -408,6 +433,11 @@ pipeline { sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/hi/ -m "not pleasefixme" --cpu --tn_cache_dir ${HI_TN_CACHE}' } } + stage('L1: Run all TA TN/ITN tests (restore grammars from cache)') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/ta/ -m "not pleasefixme" --cpu --tn_cache_dir ${TA_TN_CACHE}' + } + } stage('L1: Run all Codeswitched ES/EN TN/ITN tests (restore grammars from cache)') { steps { sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/es_en/ -m "not pleasefixme" --cpu --tn_cache_dir ${ES_EN_TN_CACHE}' diff --git a/nemo_text_processing/inverse_text_normalization/inverse_normalize.py b/nemo_text_processing/inverse_text_normalization/inverse_normalize.py index 9a6fcc64c..851216cc3 100644 --- a/nemo_text_processing/inverse_text_normalization/inverse_normalize.py +++ b/nemo_text_processing/inverse_text_normalization/inverse_normalize.py @@ -146,6 +146,11 @@ def __init__( from nemo_text_processing.inverse_text_normalization.ko.verbalizers.verbalize_final import ( VerbalizeFinalFst, ) + elif lang == 'ta': # Tamil + from nemo_text_processing.inverse_text_normalization.ta.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.inverse_text_normalization.ta.verbalizers.verbalize_final import ( + VerbalizeFinalFst, + ) else: raise NotImplementedError(f"Language {lang} has not been supported yet.") @@ -211,6 +216,7 @@ def parse_args(): 'mr', 'ja', 'ko', + 'ta', ], default="en", type=str, diff --git a/nemo_text_processing/inverse_text_normalization/run_evaluate.py b/nemo_text_processing/inverse_text_normalization/run_evaluate.py index cf9b29fce..a47c2acd2 100644 --- a/nemo_text_processing/inverse_text_normalization/run_evaluate.py +++ b/nemo_text_processing/inverse_text_normalization/run_evaluate.py @@ -53,6 +53,7 @@ def parse_args(): "vi", "zh", 'ja', + "ta", ], default="en", type=str, diff --git a/nemo_text_processing/inverse_text_normalization/ta/__init__.py b/nemo_text_processing/inverse_text_normalization/ta/__init__.py new file mode 100644 index 000000000..b03e15023 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.ta.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.inverse_text_normalization.ta.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.inverse_text_normalization.ta.verbalizers.verbalize_final import VerbalizeFinalFst diff --git a/nemo_text_processing/inverse_text_normalization/ta/data/__init__.py b/nemo_text_processing/inverse_text_normalization/ta/data/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/ta/data/fraction/__init__.py b/nemo_text_processing/inverse_text_normalization/ta/data/fraction/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/data/fraction/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/ta/data/fraction/denominator_locative.tsv b/nemo_text_processing/inverse_text_normalization/ta/data/fraction/denominator_locative.tsv new file mode 100644 index 000000000..416fe8c99 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/data/fraction/denominator_locative.tsv @@ -0,0 +1,28 @@ +ஒன்றில் ஒன்று +இரண்டில் இரண்டு +மூன்றில் மூன்று +நான்கில் நான்கு +ஐந்தில் ஐந்து +ஆறில் ஆறு +ஏழில் ஏழு +எட்டில் எட்டு +ஒன்பதில் ஒன்பது +பத்தில் பத்து +பதினொன்றில் பதினொன்று +பன்னிரண்டில் பன்னிரண்டு +இருபதில் இருபது +முப்பதில் முப்பது +நாற்பதில் நாற்பது +ஐம்பதில் ஐம்பது +அறுபதில் அறுபது +எழுபதில் எழுபது +எண்பதில் எண்பது +தொண்ணூறில் தொண்ணூறு +நூறில் நூறு +பதின்மூன்றில் பதின்மூன்று +பதினான்கில் பதினான்கு +பதினைந்தில் பதினைந்து +பதினாறில் பதினாறு +பதினேழில் பதினேழு +பதினெட்டில் பதினெட்டு +பத்தொன்பதில் பத்தொன்பது diff --git a/nemo_text_processing/inverse_text_normalization/ta/data/money/__init__.py b/nemo_text_processing/inverse_text_normalization/ta/data/money/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/data/money/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/ta/data/money/currency.tsv b/nemo_text_processing/inverse_text_normalization/ta/data/money/currency.tsv new file mode 100644 index 000000000..a9276c409 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/data/money/currency.tsv @@ -0,0 +1,13 @@ +ரூபாய் ₹ +ரூபாய்கள் ₹ +டாலர் $ +டாலர்கள் $ +பவுண்ட் £ +பவுண்ட்கள் £ +யூரோ € +யூரோக்கள் € +யென் ¥ +வான் ₩ +லீரா ₺ +டாக்கா ৳ +நைரா ₦ diff --git a/nemo_text_processing/inverse_text_normalization/ta/data/money/minor_units.tsv b/nemo_text_processing/inverse_text_normalization/ta/data/money/minor_units.tsv new file mode 100644 index 000000000..80fc5b636 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/data/money/minor_units.tsv @@ -0,0 +1,4 @@ +காசு ₹ +காசுகள் ₹ +பைசாக்கள் ₹ +சென்ட்கள் $ diff --git a/nemo_text_processing/inverse_text_normalization/ta/data/numbers/__init__.py b/nemo_text_processing/inverse_text_normalization/ta/data/numbers/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/ta/data/numbers/ambiguous.tsv b/nemo_text_processing/inverse_text_normalization/ta/data/numbers/ambiguous.tsv new file mode 100644 index 000000000..0aa4620d6 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/data/numbers/ambiguous.tsv @@ -0,0 +1,5 @@ +ஒரு licensed 1 +ஓர் licensed 1 +கால் standalone 0.25 +அரை standalone 0.5 +முக்கால் standalone 0.75 diff --git a/nemo_text_processing/inverse_text_normalization/ta/data/numbers/half_forms.tsv b/nemo_text_processing/inverse_text_normalization/ta/data/numbers/half_forms.tsv new file mode 100644 index 000000000..5b14d78a7 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/data/numbers/half_forms.tsv @@ -0,0 +1,19 @@ +அரை 0 5 +ஒன்றரை 1 5 +இரண்டரை 2 5 +மூன்றரை 3 5 +நான்கரை 4 5 +நாலரை 4 5 +ஐந்தரை 5 5 +ஆறரை 6 5 +ஏழரை 7 5 +எட்டரை 8 5 +ஒன்பதரை 9 5 +பத்தரை 10 5 +கால் 0 25 +முக்கால் 0 75 +ஒண்ணரை 1 5 +ரெண்டரை 2 5 +மூணரை 3 5 +அஞ்சரை 5 5 +ஒன்னரை 1 5 diff --git a/nemo_text_processing/inverse_text_normalization/ta/data/numbers/scale_words.tsv b/nemo_text_processing/inverse_text_normalization/ta/data/numbers/scale_words.tsv new file mode 100644 index 000000000..0375af428 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/data/numbers/scale_words.tsv @@ -0,0 +1,7 @@ +ஆயிரம் 3 expand +இலட்சம் 5 keep +லட்சம் 5 keep +கோடி 7 keep +மில்லியன் 6 keep +பில்லியன் 9 keep +டிரில்லியன் 12 keep diff --git a/nemo_text_processing/inverse_text_normalization/ta/data/whitelist/__init__.py b/nemo_text_processing/inverse_text_normalization/ta/data/whitelist/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/data/whitelist/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/ta/data/whitelist/prose_phrases.tsv b/nemo_text_processing/inverse_text_normalization/ta/data/whitelist/prose_phrases.tsv new file mode 100644 index 000000000..9df40aa7e --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/data/whitelist/prose_phrases.tsv @@ -0,0 +1,9 @@ +ஒன்று சேர் idiomatic verb "unite", not the numeral one +ஒன்று சேர்ந்து idiomatic verb "unite", not the numeral one +ஒன்று சேர்ந்த idiomatic verb "unite", not the numeral one +ஒன்று சேர்க்க idiomatic verb "unite", not the numeral one +ஒன்று கூடி idiomatic verb "assemble", not the numeral one +ஒன்று கூடு idiomatic verb "assemble", not the numeral one +ஒன்று கூடிய idiomatic verb "assemble", not the numeral one +ஒன்று போல "alike"; ஒன்று is the pronoun here +எல்லாம் ஒன்று "all is one"; ஒன்று is the pronoun here diff --git a/nemo_text_processing/inverse_text_normalization/ta/graph_utils.py b/nemo_text_processing/inverse_text_normalization/ta/graph_utils.py new file mode 100644 index 000000000..da287f8a8 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/graph_utils.py @@ -0,0 +1,110 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.export import export +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, delete_space + + +def generator_main(file_name: str, graphs: Dict[str, 'pynini.FstLike']): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logging.info(f'Created {file_name}') + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + '/grammars/' + kind + '/' + name + '.far') + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> 'pynini.FstLike': + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> 'pynini.FstLike': + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> 'pynini.FstLike': + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross(" ", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/inverse_text_normalization/ta/taggers/__init__.py b/nemo_text_processing/inverse_text_normalization/ta/taggers/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/ta/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/ta/taggers/cardinal.py new file mode 100644 index 000000000..28b4ee673 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/taggers/cardinal.py @@ -0,0 +1,346 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, List + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ta.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.ta.utils import get_abs_path, load_rows +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_DIGIT, NEMO_SIGMA +from nemo_text_processing.text_normalization.ta.graph_utils import ( + MINUS_WORD, + NEMO_TA_DIGIT, + NEMO_TA_LETTER, + PLUS_WORD, + POINT_WORD, + sequential, +) + +# Colloquial (spoken/ASR) forms rewritten to the formal words the grammar knows. +_COLLOQUIAL = [ + ("ஒன்னு", "ஒன்று"), + ("ஒண்ணு", "ஒன்று"), + ("ரெண்டு", "இரண்டு"), + ("மூணு", "மூன்று"), + ("நாலு", "நான்கு"), + ("அஞ்சு", "ஐந்து"), + ("ஒம்பது", "ஒன்பது"), + ("பூஜ்ஜியம்", "பூஜ்யம்"), + ("பன்னெண்டு", "பன்னிரண்டு"), + ("பன்னிரெண்டு", "பன்னிரண்டு"), + ("அம்பது", "ஐம்பது"), + ("ஐநூறு", "ஐந்நூறு"), + # U+0BA9 TAMIL LETTER NNA spelling of 300 (முந்நூறு is the grammar's form). + ("முன்னூறு", "முந்நூறு"), + # U+0BA9 TAMIL LETTER NNA misspelling of 90 (தொண்ணூறு is the grammar's form). + ("தொன்ணூறு", "தொண்ணூறு"), + # 13 without its U+0BA9 TAMIL LETTER NNA (பதின்மூன்று is the grammar's form). + ("பதிமூன்று", "பதின்மூன்று"), + ("முன்னூற்று", "முந்நூற்று"), +] + +# Colloquial/formal -த்தி tens joints normalized to the -த்து stems. +_TENS_JOINTS = [ + ("இருவத்தி", "இருபத்து"), + ("இருபத்தி", "இருபத்து"), + ("முப்பத்தி", "முப்பத்து"), + ("நாப்பத்தி", "நாற்பத்து"), + ("நாற்பத்தி", "நாற்பத்து"), + ("அம்பத்தி", "ஐம்பத்து"), + ("ஐம்பத்தி", "ஐம்பத்து"), + ("அறுபத்தி", "அறுபத்து"), + ("எழுபத்தி", "எழுபத்து"), + ("எண்பத்தி", "எண்பத்து"), + ("தொண்ணூத்தி", "தொண்ணூற்று"), + ("தொண்ணூற்றி", "தொண்ணூற்று"), + ("நூத்தி", "நூற்று"), + ("இருநூத்தி", "இருநூற்று"), + ("முன்னூத்தி", "முந்நூற்று"), + ("முன்னூற்றி", "முந்நூற்று"), + ("ஆயிரத்தி", "ஆயிரத்து"), + ("ரெண்டாயிர", "இரண்டாயிர"), + ("மூணாயிர", "மூன்றாயிர"), +] + +# Compound linkers and scale-word spellings normalized to the grammar's own forms. +_SCALE_LINKS = [ + ("கோடியே", "கோடி"), + ("இலட்சத்து", "இலட்சம்"), + ("லட்சத்து", "இலட்சம்"), + ("லட்சம்", "இலட்சம்"), + ("ஓராயிரம்", "ஆயிரம்"), + ("ஓர் ஆயிரம்", "ஆயிரம்"), + ("ஒரு ஆயிரம்", "ஆயிரம்"), +] + +_TENS_STEMS = [ + "இருபத்து", + "முப்பத்து", + "நாற்பத்து", + "ஐம்பத்து", + "அறுபத்து", + "எழுபத்து", + "எண்பத்து", + "தொண்ணூற்று", +] + +# Dependent vowel sign paired with the independent vowel it stands for, used both to fuse a +# spaced tens+digit pair and to split a solid one back apart. +_VOWEL_SIGNS = [("ொ", "ஒ"), ("ி", "இ"), ("ெ", "எ"), ("ே", "ஏ"), ("ை", "ஐ"), ("ா", "ஆ")] +_VOWELS = [vowel for _, vowel in _VOWEL_SIGNS] + +# Every string in the language, kept away from the Tamil digits an ASCII-only grammar never +# writes. +_NO_NATIVE_DIGITS = pynini.closure(pynini.difference(NEMO_CHAR, NEMO_TA_DIGIT)).optimize() + + +def _boundary_rewrite(pairs: List) -> 'pynini.FstLike': + """ + Word-boundary-anchored rewrite for the given (spoken, formal) pairs. + """ + tau = pynini.union(*[pynini.cross(a, b) for a, b in pairs]) + edge = pynini.union("[BOS]", " ") + right = pynini.union("[EOS]", " ") + return pynini.cdrewrite(tau, edge, right, NEMO_SIGMA).optimize() + + +def _colloquial_chain() -> 'pynini.FstLike': + """ + Normalizes spoken/colloquial number phrasing to the forms the TN grammar emits. + """ + edge = pynini.union("[BOS]", " ") + colloquial = _boundary_rewrite(_COLLOQUIAL) + joints = pynini.cdrewrite(pynini.union(*[pynini.cross(a, b) for a, b in _TENS_JOINTS]), edge, "", NEMO_SIGMA) + scale_links = _boundary_rewrite(_SCALE_LINKS) + # Fused thousands split back to the spaced reading: அறுபதாயிரம் -> அறுபது ஆயிரம். + # தொள்ளாயிரம் (900) also contains ாயிரம், so a ள just before blocks the split. + split_thousands = pynini.cdrewrite( + pynini.union(pynini.cross("ாயிரத்து", "ு ஆயிரம்"), pynini.cross("ாயிரம்", "ு ஆயிரம்")), + pynini.difference(NEMO_CHAR, pynini.accep("ள")), + "", + NEMO_SIGMA, + ) + # Colloquial -ஞ்சு endings after த/ன read as -ைந்து (பதினஞ்சு -> பதினைந்து). + nju = pynini.cdrewrite(pynini.cross("ஞ்சு", "ைந்து"), pynini.union("த", "ன"), pynini.union("[EOS]", " "), NEMO_SIGMA) + # A -தி joint written solid onto a vowel-initial digit takes a ய glide (எண்பத்தியொன்று) or + # fuses ந to ன (எண்பத்தினான்கு); split it back to the spaced reading so the joining stages + # below can rebuild the grammar's own sandhi form. + stems = pynini.union(*_TENS_STEMS) + glide_split = pynini.union(*[pynini.cross(f"ய{sign}", f" {vowel}") for sign, vowel in _VOWEL_SIGNS]) + unfuse = ( + pynini.cdrewrite(glide_split, stems, "", NEMO_SIGMA) + @ pynini.cdrewrite(pynini.cross("ன", " ந"), stems, "ா", NEMO_SIGMA) + @ pynini.cdrewrite(pynutil.insert(" "), stems, pynini.union(*_VOWELS), NEMO_SIGMA) + ) + + # A spaced tens+digit pair joins into the fused sandhi form the grammar accepts: + # consonant-initial digits join directly, vowel-initial digits merge the tens-final ு with + # their initial vowel (நாற்பத்து ஒன்று -> நாற்பத்தொன்று). + stems_lopped = pynini.union(*[stem[:-1] for stem in _TENS_STEMS]) + join_consonant = pynini.cdrewrite(pynutil.delete(" "), edge + stems, pynini.union("மூன்று", "நான்கு"), NEMO_SIGMA) + vowel_merge = pynini.union(*[pynini.cross(f"ு {vowel}", sign) for sign, vowel in _VOWEL_SIGNS]) + join_vowel = pynini.cdrewrite(vowel_merge, edge + stems_lopped, "", NEMO_SIGMA) + return ( + colloquial @ nju @ joints @ unfuse @ scale_links @ split_thousands @ join_consonant @ join_vowel + ).optimize() + + +def _colloquial_domain() -> 'pynini.FstLike': + """ + Strings some colloquial stage can rewrite; the chain is the identity on anything else, which + the raw reading already covers. + """ + triggers = ( + [spoken for spoken, _ in _COLLOQUIAL + _TENS_JOINTS + _SCALE_LINKS] + + _TENS_STEMS + + ["ஞ்சு", "ாயிரத்து", "ாயிரம்"] + + [f"ு {vowel}" for vowel in _VOWELS] + ) + return (pynini.closure(NEMO_CHAR) + pynini.union(*triggers) + pynini.closure(NEMO_CHAR)).optimize() + + +def _hundreds_split() -> 'pynini.FstLike': + """ + Splits the spoken hundreds sandhi back into the spaced form: நூற்றிரண்டு -> நூற்று இரண்டு. + """ + signs = ["ி", "ொ", "ெ", "ே", "ை", "ா"] + vowels = ["இ", "ஒ", "எ", "ஏ", "ஐ", "ஆ"] + stems = pynini.union("நூற்ற", "ஆயிரத்த") + unmerge = pynini.union(*[pynini.cross(s, f"ு {v}") for s, v in zip(signs, vowels)]) + rewrite = pynini.cdrewrite(unmerge, stems, "", NEMO_SIGMA) + # Restricted to strings that actually carry the sandhi, so this second reading of the input + # costs a small composition instead of a whole extra copy of the grammar. + domain = (pynini.closure(NEMO_CHAR) + stems + pynini.union(*signs) + pynini.closure(NEMO_CHAR)).optimize() + return pynini.compose(domain, rewrite).optimize() + + +def spoken_pre_map() -> 'pynini.FstLike': + """ + Every reading of the spoken words the cardinal tries: the words as spoken (preferred, since + the colloquial rewrites would destroy the sandhi forms TN itself emits), the colloquial + chain, and the hundreds sandhi split back apart. Each rewrite is restricted to the strings it + can change, so it costs a small composition rather than a second copy of the number grammar. + """ + raw = pynutil.add_weight(pynini.closure(NEMO_CHAR), -0.01) + colloquial = pynini.compose(_colloquial_domain(), _colloquial_chain()) + return pynini.union(raw, colloquial, _hundreds_split()).optimize() + + +def ambiguous_words(condition: str) -> List: + """ + The ``(word, reading)`` pairs of ``data/numbers/ambiguous.tsv`` whose admission condition is + ``condition``: ``licensed`` words (ஒரு, ஓர், also the indefinite article) count as a number + only inside a money or clock reading; ``standalone`` words (கால், அரை, முக்கால்) read as a + fraction on their own but not before another Tamil word. + """ + rows = load_rows(get_abs_path("data/numbers/ambiguous.tsv"), 3) + return [(word, reading) for word, row_condition, reading, *_ in rows if row_condition == condition] + + +def licensed_words() -> 'pynini.FstLike': + """ + ஒரு / ஓர் to 1: also the indefinite article, so a number only inside a money or clock reading. + """ + return pynini.string_map(ambiguous_words("licensed")).optimize() + + +def optional_sign_field() -> 'pynini.FstLike': + """ + Consumes a leading spoken sign word, emitting the ``negative``/``positive`` field. + """ + negative = pynini.cross(MINUS_WORD + " ", "\"true\" ") + positive = pynini.cross(PLUS_WORD + " ", "\"true\" ") + return pynini.closure(pynutil.insert("negative: ") + negative | pynutil.insert("positive: ") + positive, 0, 1) + + +def half_form_rows() -> List[List[str]]: + """ + Rows of ``data/numbers/half_forms.tsv`` (fused word, integer digits, fraction digits). + """ + return load_rows(get_abs_path("data/numbers/half_forms.tsv"), 3) + + +def scale_word_rows() -> List[List[str]]: + """ + Rows of ``data/numbers/scale_words.tsv`` (scale word, trailing zeros, expand|keep). + """ + return load_rows(get_abs_path("data/numbers/scale_words.tsv"), 3) + + +def kept_scale_words() -> List[str]: + """ + Scale words a written amount keeps as a word (5.5 லட்சம், ₹2.5 கோடி). + """ + return [word for word, _, policy, *_ in scale_word_rows() if policy != "expand"] + + +def _scale_expanded(plain: 'pynini.FstLike') -> 'pynini.FstLike': + """ + Multiplies out a scale word small enough for it, e.g. ஐந்து புள்ளி ஐந்து ஆயிரம் -> 5500. + """ + words_by_zeros: Dict[int, List[str]] = {} + for word, zeros, policy, *_ in scale_word_rows(): + if policy == "expand": + words_by_zeros.setdefault(int(zeros), []).append(word) + if not words_by_zeros: + return pynini.Fst() + + half_rows = half_form_rows() + point = pynutil.delete(" " + POINT_WORD + " ") + graphs = [] + for zeros, words in words_by_zeros.items(): + tail = pynutil.delete(" " + pynini.union(*words)) + # The fractional digits shift left by the scale's zero count, so the padding inserted + # after them follows the width that matched. + shifted = pynini.union( + *[(plain @ (NEMO_DIGIT**width)) + pynutil.insert("0" * (zeros - width)) for width in range(1, zeros + 1)] + ) + digits_1_3 = pynini.closure(NEMO_DIGIT, 1, 3) + graphs.append((plain @ pynini.difference(digits_1_3, pynini.accep("0"))) + point + shifted + tail) + # A zero integer part is dropped, not kept as a leading zero, and an all-zero result + # collapses to a single 0. + drop_zero = pynutil.delete((plain @ pynini.accep("0")).project("input")) + all_zeros = pynini.accep("0" * zeros) + graphs.append(drop_zero + point + (shifted @ pynini.difference(NEMO_DIGIT**zeros, all_zeros)) + tail) + graphs.append(drop_zero + point + (shifted @ pynini.cross("0" * zeros, "0")) + tail) + # The fused half words scale the same way: ஒன்றரை ஆயிரம் -> 1500. + graphs.append( + pynini.union( + *[ + pynini.cross(f"{fused} {word}", str(int(ip + fp.ljust(zeros, "0")))) + for fused, ip, fp, *_ in half_rows + for word in words + ] + ) + ) + return pynini.union(*graphs).optimize() + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying spoken cardinals, e.g. + இருபத்துமூன்று -> cardinal { integer: "23" } + இரண்டாயிரத்து இருபத்துநான்கில் -> cardinal { integer: "2024ல்" } + மைனஸ் நூற்று இருபது -> cardinal { negative: "true" integer: "120" } + + The spoken forms are the TN cardinal's own number grammar inverted, so the two directions + cannot drift apart, read through a pre-map that normalizes colloquial phrasing (ரெண்டு, + இருவத்தி மூணு, நூத்தி ஐம்பது) to the forms TN emits. + + Args: + tn_cardinal: the Tamil TN CardinalFst, whose number grammar is inverted here + """ + + def __init__(self, tn_cardinal: 'GraphFst'): + super().__init__(name="cardinal", kind="classify") + + # Every written form the TN grammar accepts, restricted to ASCII digits and inverted. + inverted = pynini.invert(pynini.compose(_NO_NATIVE_DIGITS, tn_cardinal.itn_input_graph())).optimize() + + self.pre_map = spoken_pre_map() + plain = self.read(inverted) + # A decimal amount times a small scale word is one number: ஐந்து புள்ளி ஐந்து ஆயிரம் -> 5500. + # The two readings are made sequential separately: determinizing their union re-times + # every delayed output and blows up. + scaled = _scale_expanded(plain) + if scaled.num_states() > 0: + scaled = sequential(scaled) + self.words_to_digits = pynini.union(plain, scaled).optimize() + self.words_to_digits_licensed = sequential(pynini.union(self.words_to_digits, licensed_words())) + + # A case suffix on the last number word is carried into the written form, spelled with + # its independent vowel (ஐந்தால் -> 5ஆல்), never as a bare vowel sign. + keep_suffix = pynini.closure(NEMO_DIGIT) + pynini.closure(NEMO_TA_LETTER) + suffixed = ( + pynini.invert( + pynini.compose( + _NO_NATIVE_DIGITS, + tn_cardinal.attach_case_suffix(tn_cardinal.readable_years(), include_vowel=False), + ) + ) + @ keep_suffix + ).optimize() + self.words_to_digits_suffixed = self.read(suffixed) + + graph = self.words_to_digits | pynutil.add_weight(self.words_to_digits_suffixed, 0.1) + graph = optional_sign_field() + pynutil.insert("integer: \"") + graph + pynutil.insert("\"") + self.fst = self.add_tokens(graph).optimize() + + def read(self, lexicon: 'pynini.FstLike') -> 'pynini.FstLike': + """ + Reads spoken words through the pre-map into ``lexicon``, input-deterministically. + """ + return sequential(self.pre_map @ lexicon) diff --git a/nemo_text_processing/inverse_text_normalization/ta/taggers/date.py b/nemo_text_processing/inverse_text_normalization/ta/taggers/date.py new file mode 100644 index 000000000..9557300cc --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/taggers/date.py @@ -0,0 +1,59 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ta.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.ta.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, delete_space, insert_space +from nemo_text_processing.text_normalization.ta.graph_utils import NEMO_TA_LETTER +from nemo_text_processing.text_normalization.ta.utils import get_abs_path as tn_abs_path + + +class DateFst(GraphFst): + """ + Finite state transducer for classifying spoken dates, e.g. + பதினைந்து ஜூன் இரண்டாயிரத்து இருபத்துநான்கு -> date { day: "15" month: "ஜூன்" year: "2024" preserve_order: true } + இரண்டாயிரத்து இருபத்துநான்கு ஜூன் பதினைந்து -> date { year: "2024" month: "ஜூன்" day: "15" preserve_order: true } + + The month names are the spoken side of the TN months table, so the two directions share + one list. + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: CardinalFst): + super().__init__(name="date", kind="classify") + + month_names = pynini.project(pynini.string_file(tn_abs_path("data/date/months.tsv")), "output").optimize() + + # Days are 1-31 and years four digits, so இரண்டாயிரத்து இருபத்துநான்கு is never a day. + valid_day = pynini.union(*[str(n) for n in range(1, 32)]).optimize() + four_digits = NEMO_DIGIT**4 + pynini.closure(NEMO_TA_LETTER) + day = pynutil.insert("day: \"") + (cardinal.words_to_digits @ valid_day) + pynutil.insert("\"") + month = pynutil.insert("month: \"") + month_names + pynutil.insert("\"") + # A case suffix on the year travels into the written form (... 2024ல்). + year_words = pynini.union(cardinal.words_to_digits, pynutil.add_weight(cardinal.words_to_digits_suffixed, 0.1)) + year = pynutil.insert("year: \"") + (year_words @ four_digits) + pynutil.insert("\"") + sep = delete_space + insert_space + + graph_dmy = day + sep + month + pynini.closure(sep + year, 0, 1) + graph_my = month + sep + year + graph_ymd = year + sep + month + sep + day + graph_mdy = month + sep + day + pynini.closure(sep + year, 0, 1) + + graph = (graph_dmy | graph_my | graph_ymd | graph_mdy) + pynutil.insert(" preserve_order: true") + self.fst = self.add_tokens(graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ta/taggers/decimal.py b/nemo_text_processing/inverse_text_normalization/ta/taggers/decimal.py new file mode 100644 index 000000000..c9ea25e78 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/taggers/decimal.py @@ -0,0 +1,165 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ta.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.ta.taggers.cardinal import ( + CardinalFst, + half_form_rows, + kept_scale_words, + optional_sign_field, +) +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_DIGIT, delete_space, insert_space +from nemo_text_processing.text_normalization.ta.graph_utils import POINT_WORD, TA_ARAI, TA_KAAL, TA_MUKKAL + +# Fraction digits of the quarter words, and the same quantity read as clock minutes. +QUARTER_FRACTION = {TA_KAAL: "25", TA_ARAI: "5", TA_MUKKAL: "75"} +FRACTION_MINUTES = {"5": "30", "25": "15", "75": "45"} + + +def half_form_graph( + number: 'pynini.FstLike', prefix: str, infix: str, suffix: Callable[[str], str] +) -> 'pynini.FstLike': + """ + Maps a fused -ரை half word (இருபத்தைந்தரை) to ``prefix INT infix suffix("5")``. + + ``data/numbers/half_forms.tsv`` lists these only up to பத்தரை; the rule is regular, so this + covers the rest (TN writes the fused form for any integer ending in -உ). + """ + stem = ((pynini.closure(NEMO_CHAR) + pynini.cross(TA_ARAI[1:], "ு")) @ number).optimize() + return (pynutil.insert(prefix) + stem + pynutil.insert(infix + suffix(QUARTER_FRACTION[TA_ARAI]))).optimize() + + +def quarter_form_graph( + number: 'pynini.FstLike', prefix: str, infix: str, suffix: Callable[[str], str] +) -> 'pynini.FstLike': + """ + Maps an -ே linked quarter phrase (பத்தே கால், ஒன்றேகால்) to ``prefix INT infix suffix(frac)``. + """ + stem = ((pynini.closure(NEMO_CHAR) + pynini.cross("ே", "ு")) @ number).optimize() + optional_space = pynini.closure(pynutil.delete(" "), 0, 1) + return pynini.union( + *[ + pynutil.insert(prefix) + + stem + + pynutil.insert(infix) + + optional_space + + pynutil.delete(word) + + pynutil.insert(suffix(fraction)) + for word, fraction in QUARTER_FRACTION.items() + ] + ).optimize() + + +def decimal_fused(small: 'pynini.FstLike') -> 'pynini.FstLike': + """ + The fused fractional words beyond the table as decimal fields: பத்தே கால் -> 10.25, + இருபத்தைந்தரை -> 25.5. ``small`` is the spoken integer part, at most three digits. + """ + + def suffix(fraction: str) -> str: + return f" fractional_part: \"{fraction}\"" + + return pynini.union( + quarter_form_graph(small, "integer_part: \"", "\"", suffix), + half_form_graph(small, "integer_part: \"", "\"", suffix), + ).optimize() + + +def money_fused(short: 'pynini.FstLike') -> 'pynini.FstLike': + """ + The fused fractional words as a digit amount with a point (ஒன்றரை -> 1.5, பத்தே கால் -> + 10.25), for an amount before a scale word or a currency word. + """ + tabulated = pynini.union(*[pynini.cross(word, f"{ip}.{fp}") for word, ip, fp, *_ in half_form_rows()]) + quarters = quarter_form_graph(short, "", ".", lambda fraction: fraction) + return pynini.union(tabulated, quarters).optimize() + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying spoken decimals, e.g. + பன்னிரண்டு புள்ளி ஐந்து -> decimal { integer_part: "12" fractional_part: "5" } + ஒன்று புள்ளி இரண்டு ஐந்து லட்சம் -> decimal { integer_part: "1" fractional_part: "25" quantity: "லட்சம்" } + ஒன்றரை -> decimal { integer_part: "1" fractional_part: "5" } + பத்தே கால் -> decimal { integer_part: "10" fractional_part: "25" } + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: CardinalFst): + super().__init__(name="decimal", kind="classify") + + # Fractional digits are spoken one to three at a time; a scale word after the fraction + # is a quantity, never more digits (ஐந்து புள்ளி ஐந்து லட்சம் -> 5.5 லட்சம்). + short = cardinal.words_to_digits @ pynini.closure(NEMO_DIGIT, 1, 3) + digit_by_digit = short + pynini.closure(delete_space + short) + point = pynutil.delete(POINT_WORD) + + integer_part = pynutil.insert("integer_part: \"") + cardinal.words_to_digits + pynutil.insert("\"") + fractional_part = pynutil.insert("fractional_part: \"") + digit_by_digit + pynutil.insert("\"") + + optional_sign = optional_sign_field() + + # A kept scale word after the fraction stays in the token, so the last fractional digit is + # never read as its multiplier (ஒன்று புள்ளி இரண்டு ஐந்து லட்சம் -> 1.25 லட்சம்). + quantity = pynini.closure( + pynutil.insert(" quantity: \"") + + pynutil.delete(" ") + + pynini.union(*kept_scale_words()) + + pynutil.insert("\"") + + pynutil.add_weight(pynini.accep(""), -0.2), + 0, + 1, + ) + graph = ( + optional_sign + + integer_part + + delete_space + + point + + delete_space + + insert_space + + fractional_part + + quantity + ) + + # Dotted chains round-trip: ஒன்று புள்ளி இரண்டு புள்ளி மூன்று -> 1.2.3. + chain_fraction = ( + pynutil.insert("fractional_part: \"") + + digit_by_digit + + pynini.closure(pynini.cross(f" {POINT_WORD} ", ".") + digit_by_digit, 1) + + pynutil.insert("\"") + ) + graph |= pynutil.add_weight( + optional_sign + integer_part + delete_space + point + delete_space + insert_space + chain_fraction, -0.1 + ) + + # Fused fractional words: ஒன்றரை -> 1.5, பத்தரை -> 10.5; the table, and the regular + # -ரை / -ே readings beyond it. Bare half/quarter nouns stay words. + graph |= pynini.union( + *[ + pynini.cross(word, f"integer_part: \"{ip}\" fractional_part: \"{fp}\"") + for word, ip, fp, *_ in half_form_rows() + ] + ) + # A fused fraction's integer part is at most three digits, so the fused readings compose a + # bounded slice rather than the whole number grammar. + graph |= decimal_fused(short.optimize()) + + self.fst = self.add_tokens(graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ta/taggers/fraction.py b/nemo_text_processing/inverse_text_normalization/ta/taggers/fraction.py new file mode 100644 index 000000000..776b0b8c3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/taggers/fraction.py @@ -0,0 +1,73 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ta.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.ta.taggers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.ta.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, delete_space, insert_space +from nemo_text_processing.text_normalization.ta.graph_utils import FRACTION_WORD + + +def denominator_to_number() -> 'pynini.FstLike': + """ + Undoes the locative -இல் on a spoken denominator (நான்கில் -> நான்கு): the tabulated forms of + ``data/fraction/denominator_locative.tsv``, the regular locative (-இல் replaces the final -உ, + a ம்-final scale word takes -த்தில்), and the hundreds compound -நூற்றில். + """ + tabulated = pynini.string_file(get_abs_path("data/fraction/denominator_locative.tsv")) + regular = NEMO_SIGMA + pynini.union(pynini.cross("ில்", "ு"), pynini.cross("த்தில்", "ம்"), pynini.cross("ியில்", "ி")) + hundreds = NEMO_SIGMA + pynini.cross("நூற்றில்", "நூறு") + return pynini.union(tabulated, regular, hundreds).optimize() + + +class FractionFst(GraphFst): + """ + Finite state transducer for classifying spoken fractions, e.g. + நான்கில் மூன்று -> fraction { denominator: "4" numerator: "3" } + ஐந்து கீழ் எழுபத்தேழு -> fraction { numerator: "5" denominator: "77" } + + The locative reading (denominator first) is what ASR output carries; the கீழ் reading is + what TN emits. + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: CardinalFst): + super().__init__(name="fraction", kind="classify") + + numerator = pynutil.insert("numerator: \"") + cardinal.words_to_digits + pynutil.insert("\"") + + # Undo the denominator's locative form, then read it as a number. The undoing is + # ambiguous until the number lexicon decides, so it is not determinized: that re-times the + # number's delayed outputs and explodes. + denominator_words = (denominator_to_number() @ cardinal.words_to_digits).optimize() + denominator = pynutil.insert("denominator: \"") + denominator_words + pynutil.insert("\"") + graph = denominator + delete_space + insert_space + numerator + + # "N கீழ் M" order: ஐந்து கீழ் எழுபத்தேழு -> 5/77. + graph |= ( + numerator + + delete_space + + pynutil.delete(FRACTION_WORD) + + delete_space + + insert_space + + pynutil.insert("denominator: \"") + + cardinal.words_to_digits + + pynutil.insert("\"") + ) + self.fst = self.add_tokens(graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ta/taggers/money.py b/nemo_text_processing/inverse_text_normalization/ta/taggers/money.py new file mode 100644 index 000000000..4624757c1 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/taggers/money.py @@ -0,0 +1,180 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, List + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ta.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.ta.taggers.cardinal import ( + CardinalFst, + kept_scale_words, + optional_sign_field, +) +from nemo_text_processing.inverse_text_normalization.ta.taggers.decimal import money_fused +from nemo_text_processing.inverse_text_normalization.ta.utils import get_abs_path, load_rows +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, NEMO_SIGMA, delete_space +from nemo_text_processing.text_normalization.ta.graph_utils import MONEY_SUFFIXES, POINT_WORD, RANGE_WORD +from nemo_text_processing.text_normalization.ta.utils import get_abs_path as tn_abs_path + + +def _minor_unit_rows(major_to_symbol: Dict[str, str]) -> List[List[str]]: + """ + Every minor-unit word TN can emit, paired with its major currency's symbol. + + The rows are derived from the TN ``data/money/major_minor_currencies.tsv`` the TN money + verbalizer reads, so the two directions cannot drift apart; ``data/money/minor_units.tsv`` + adds only what that pairing cannot give (a plural TN never emits, the everyday காசு). + """ + rows = [ + [minor, major_to_symbol[major]] + for major, minor, *_ in load_rows(tn_abs_path("data/money/major_minor_currencies.tsv"), 2) + if major in major_to_symbol + ] + seen = {tuple(row) for row in rows} + rows += [row[:2] for row in load_rows(get_abs_path("data/money/minor_units.tsv"), 2) if tuple(row[:2]) not in seen] + return rows + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying spoken money, e.g. + ஐம்பது ரூபாய் -> money { integer_part: "50" currency: "₹" } + ஐம்பது ரூபாய் ஐம்பது பைசா -> money { integer_part: "50" currency: "₹" fractional_part: "50" } + ஐந்து கோடி ரூபாய் -> money { integer_part: "5 கோடி" currency: "₹" } + ஐம்பது ரூபாய்க்கு -> money { integer_part: "50" currency: "₹" morphosyntactic_features: "க்கு" } + + Reads ``data/money/currency.tsv`` (spoken currency word -> symbol) and ``data/money/minor_units.tsv``. + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: CardinalFst): + super().__init__(name="money", kind="classify") + + major_rows = [row[:2] for row in load_rows(get_abs_path("data/money/currency.tsv"), 2)] + minor_rows = _minor_unit_rows(dict(major_rows)) + currency = pynini.string_map(major_rows) + minor = pynini.string_map(minor_rows) + # A minor unit belongs to one major currency: பைசா is rupees, சென்ட் is dollars. Grouping + # them by symbol keeps ஐந்து டாலர் ஐம்பது பைசா from reading as $5.50. + majors_by_symbol: Dict[str, List[str]] = {} + minors_by_symbol: Dict[str, List[str]] = {} + for word, symbol in major_rows: + majors_by_symbol.setdefault(symbol, []).append(word) + for word, symbol in minor_rows: + minors_by_symbol.setdefault(symbol, []).append(word) + + # A case suffix on the currency or minor-unit word is carried into the written form + # (₹50க்கு, ₹50.50க்கு). + optional_suffix = pynini.closure( + pynutil.insert(" morphosyntactic_features: \"") + pynini.union(*MONEY_SUFFIXES) + pynutil.insert("\""), + 0, + 1, + ) + currency_field = pynutil.insert(" currency: \"") + currency + pynutil.insert("\"") + optional_suffix + + amount_words = cardinal.words_to_digits_licensed + range_words = cardinal.words_to_digits + pynini.cross(" " + RANGE_WORD + " ", "-") + cardinal.words_to_digits + integer_part = ( + pynutil.insert("integer_part: \"") + + (amount_words | pynutil.add_weight(range_words, -0.5)) + + pynutil.insert("\"") + ) + # A range takes no minor unit, so the minor paths embed the amount only once each. + amount_part = pynutil.insert("integer_part: \"") + amount_words + pynutil.insert("\"") + # A lone fractional digit is a tens value in paise (ஐந்து பைசா -> .05). + two_digits = pynini.union(NEMO_DIGIT + NEMO_DIGIT, pynutil.insert("0") + NEMO_DIGIT) + fractional_part = pynutil.insert(" fractional_part: \"") + (amount_words @ two_digits) + pynutil.insert("\"") + + graph = integer_part + delete_space + currency_field + for symbol, minor_words in minors_by_symbol.items(): + if symbol not in majors_by_symbol: + continue + graph |= ( + amount_part + + delete_space + + pynutil.insert(f" currency: \"{symbol}\"") + + pynutil.delete(pynini.union(*majors_by_symbol[symbol])) + + delete_space + + fractional_part + + delete_space + + pynutil.delete(pynini.union(*minor_words)) + + optional_suffix + ) + + # Currency word first: ரூபாய் ஐம்பது -> ₹50. + graph |= ( + pynutil.insert("currency: \"") + + currency + + pynutil.insert("\"") + + delete_space + + pynutil.insert(" ") + + integer_part + + pynutil.insert(" preserve_order: true") + ) + + # Quantity-word money keeps the written idiom: ஐந்து கோடி ரூபாய் -> ₹5 கோடி, இரண்டு புள்ளி ஐந்து + # லட்சம் ரூபாய் -> ₹2.5 லட்சம். Expanded scale words (ஆயிரம்) are digits. Two scale words + # stack in the written idiom too: ஒரு லட்சம் கோடி ரூபாய் -> ₹1 லட்சம் கோடி. + kept = kept_scale_words() + quantity_written = pynini.union(*kept) + stacked = pynini.union(*kept) + " " + quantity_written + quantity_written = pynini.union(quantity_written, pynutil.add_weight(stacked, -0.1)) + short = cardinal.words_to_digits @ pynini.closure(NEMO_DIGIT, 1, 2) + frac_digits = short + pynini.closure(delete_space + short) + point = pynini.cross(pynini.accep(" ") + POINT_WORD + " ", ".") + # The amount before a kept scale word holds no kept scale word itself: ஐந்து கோடி ஐம்பது + # லட்சம் ரூபாய் is one number (₹55000000), not ₹50000050 லட்சம். + no_kept = pynini.difference(NEMO_SIGMA, NEMO_SIGMA + pynini.union(*kept) + NEMO_SIGMA) + amount_digits = pynini.compose(no_kept, amount_words) + pynini.closure(point + frac_digits, 0, 1) + # A fused half word is an amount too (ஒன்றரை லட்சம் ரூபாய் -> ₹1.5 லட்சம், ஒன்றரை ரூபாய் -> ₹1.50). + fused_amount = money_fused((cardinal.words_to_digits_licensed @ pynini.closure(NEMO_DIGIT, 1, 3)).optimize()) + amount_digits |= fused_amount + split_point = ( + pynini.closure(NEMO_DIGIT, 1) + + pynini.cross(".", "\" fractional_part: \"") + + (NEMO_DIGIT + NEMO_DIGIT | NEMO_DIGIT + pynutil.insert("0")) + ) + graph |= ( + pynutil.insert("integer_part: \"") + + (fused_amount @ split_point) + + pynutil.insert("\"") + + delete_space + + currency_field + ) + quantity_amount = ( + pynutil.insert("integer_part: \"") + + amount_digits + + pynini.accep(" ") + + quantity_written + + pynutil.insert("\"") + ) + graph |= pynutil.add_weight(quantity_amount + delete_space + currency_field, -1.0) + + # Minor-unit-only amounts: ஐம்பது பைசா -> ₹0.50, ஐம்பது சென்ட் -> $0.50. + graph |= ( + pynutil.insert("integer_part: \"0\"") + + fractional_part + + delete_space + + pynutil.insert(" currency: \"") + + minor + + pynutil.insert("\"") + + optional_suffix + ) + + # A spoken sign folds into the amount: மைனஸ் ஐந்நூறு ரூபாய் -> -₹500. + self.fst = self.add_tokens(optional_sign_field() + graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ta/taggers/ordinal.py b/nemo_text_processing/inverse_text_normalization/ta/taggers/ordinal.py new file mode 100644 index 000000000..560771c83 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/taggers/ordinal.py @@ -0,0 +1,64 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ta.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.ta.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR +from nemo_text_processing.text_normalization.ta.graph_utils import NEMO_TA_LETTER, sequential + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying spoken ordinals, e.g. + ஐந்தாவது -> ordinal { integer: "5" morphosyntactic_features: "வது" preserve_order: true } + பத்தாம் -> ordinal { integer: "10" morphosyntactic_features: "ஆம்" preserve_order: true } + ஐந்தாவதுக்கு -> ordinal { integer: "5" morphosyntactic_features: "வதுக்கு" preserve_order: true } + + Tamil reads the ordinal off its adjectival stem (ஐந்தா- -> ஐந்து -> 5) and carries the written + marker in its own field rather than inverting the TN ordinal graph: the inflected tail is + unbounded, so inverting it against the written marker leaves an ambiguous alignment the + composition cannot merge. + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: CardinalFst): + super().__init__(name="ordinal", kind="classify") + + # Undo the adjectival stem: the cardinal's final -உ becomes -ஆ and a ம்-final scale word + # becomes -மா (ஐந்து -> ஐந்தா, ஆயிரம் -> ஆயிரமா, நூறு -> நூற்றா). + to_cardinal = pynini.closure(NEMO_CHAR) + pynini.union( + pynini.cross("ா", "ு"), pynini.cross("மா", "ம்"), pynini.cross("ற்றா", "று") + ) + stem = sequential((to_cardinal @ cardinal.words_to_digits) | pynini.cross("முதலா", "1")) + + integer = pynutil.insert("integer: \"") + stem + pynutil.insert("\"") + + # -வது plus any inflected tail: ஐந்தாவது -> 5வது, ஐந்தாவதுக்கு -> 5வதுக்கு. + graph_vathu = ( + integer + + pynutil.insert(" morphosyntactic_features: \"") + + pynini.accep("வத") + + pynini.closure(NEMO_TA_LETTER, 1) + + pynutil.insert("\"") + ) + # The clipped ம் ordinal is written with the canonical ஆம் spelling (28ம் -> 28ஆம்). + graph_aam = integer + pynutil.insert(" morphosyntactic_features: \"ஆம்\"") + pynutil.delete("ம்") + + graph = (graph_vathu | graph_aam) + pynutil.insert(" preserve_order: true") + self.fst = self.add_tokens(graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ta/taggers/punctuation.py b/nemo_text_processing/inverse_text_normalization/ta/taggers/punctuation.py new file mode 100644 index 000000000..a0f33c5fe --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/taggers/punctuation.py @@ -0,0 +1,52 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ta.graph_utils import GraphFst +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE +from nemo_text_processing.text_normalization.ta.graph_utils import punctuation_code_points + +# ASCII marks that Unicode does not categorise as punctuation. +_ASCII_MARKS = "!#%&'()*+,-./:;<=>?@^_`{|}~\"" + + +class PunctuationFst(GraphFst): + """ + Finite state transducer for classifying punctuation, e.g. + a, -> tokens { name: "a" } tokens { name: "," } + + Markup such as or stays one token instead of splitting into marks. + """ + + def __init__(self): + super().__init__(name="punctuation", kind="classify") + + self.punct_marks = punctuation_code_points() + list(_ASCII_MARKS) + marks = pynini.union(*[pynini.escape(p) for p in self.punct_marks]) + punct = pynini.closure(marks, 1).optimize() + + tag_body = pynini.closure(NEMO_NOT_SPACE - pynini.union("<", ">"), 1) + emphasis = ( + pynini.accep("<") + + pynini.union(tag_body + pynini.closure(pynini.accep("/"), 0, 1), pynini.accep("/") + tag_body) + + pynini.accep(">") + ).optimize() + rest = pynini.difference(pynini.project(punct, "input"), pynini.project(emphasis, "input")) + punct = pynini.union(emphasis, pynini.compose(rest, punct)).optimize() + + self.graph = punct + self.graph_input = punct.copy().project("input").optimize() + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ta/taggers/telephone.py b/nemo_text_processing/inverse_text_normalization/ta/taggers/telephone.py new file mode 100644 index 000000000..0b04deead --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/taggers/telephone.py @@ -0,0 +1,77 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ta.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.ta.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, delete_space +from nemo_text_processing.text_normalization.ta.graph_utils import NEMO_TA_LETTER, PLUS_WORD, TO_ASCII_DIGITS +from nemo_text_processing.text_normalization.ta.utils import get_abs_path as tn_abs_path + +# Spoken zero variants beyond the telephone table's word. +ZERO_WORDS = ("பூஜ்ஜியம்",) + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for classifying spoken digit strings, e.g. + ஒன்பது ஒன்பது நான்கு ... பூஜ்யம் -> telephone { number_part: "9943206870" } + பிளஸ் தொண்ணூற்றொன்று ஒன்பது எட்டு ... -> telephone { country_code: "+91" number_part: "9876543210" } + பூஜ்யம் பூஜ்யம் ஏழு -> telephone { number_part: "007" } + + Three or more digit words in a row are a digit string (phone, PIN, OTP, 007). Reads the + TN ``data/telephone/number.tsv`` from the spoken side. + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: CardinalFst): + super().__init__(name="telephone", kind="classify") + + digit_words = pynini.invert(pynini.string_file(tn_abs_path("data/telephone/number.tsv"))).optimize() + digit = digit_words @ TO_ASCII_DIGITS + for word in ZERO_WORDS: + digit |= pynini.cross(word, "0") + digit = digit.optimize() + + # A case suffix on the last digit word is carried over (...பூஜ்யத்தில் -> ...0ல்), through + # the cardinal's own suffix reading. + suffixed_digit = cardinal.words_to_digits_suffixed @ (NEMO_DIGIT + pynini.closure(NEMO_TA_LETTER, 1)) + last = pynini.union(digit, suffixed_digit) + # After a country code the number is a 10-digit mobile or 11-digit landline. + number = digit + pynini.closure(delete_space + digit, 1) + delete_space + last + cc_number = digit + pynini.closure(delete_space + digit, 8, 9) + delete_space + last + + plus = pynini.cross(PLUS_WORD, "+") + # The plus word followed by one to three digit words, or by a spoken number. + code_digits = pynini.union( + digit + pynini.closure(delete_space + digit, 0, 2), + cardinal.words_to_digits @ pynini.closure(NEMO_DIGIT, 1, 3), + ) + country_code = pynutil.insert("country_code: \"") + plus + delete_space + code_digits + pynutil.insert("\"") + + number_part = pynutil.insert("number_part: \"") + number + pynutil.insert("\"") + cc_number_part = pynutil.insert("number_part: \"") + cc_number + pynutil.insert("\"") + graph = number_part | (country_code + pynutil.insert(" ") + delete_space + cc_number_part) + # A standalone two- or three-digit country code: பிளஸ் தொண்ணூற்றொன்று -> +91. + standalone = pynutil.insert("country_code: \"") + plus + delete_space + standalone += pynini.union( + digit + pynini.closure(delete_space + digit, 1, 2), + cardinal.words_to_digits @ pynini.closure(NEMO_DIGIT, 2, 3), + ) + pynutil.insert("\"") + graph |= pynutil.add_weight(standalone, 0.2) + self.fst = self.add_tokens(graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ta/taggers/time.py b/nemo_text_processing/inverse_text_normalization/ta/taggers/time.py new file mode 100644 index 000000000..d2f3e494d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/taggers/time.py @@ -0,0 +1,116 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ta.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.ta.taggers.cardinal import CardinalFst, half_form_rows +from nemo_text_processing.inverse_text_normalization.ta.taggers.decimal import FRACTION_MINUTES, quarter_form_graph +from nemo_text_processing.text_normalization.en.graph_utils import delete_space +from nemo_text_processing.text_normalization.en.utils import load_labels +from nemo_text_processing.text_normalization.ta.graph_utils import DAY_PARTS, TO_ASCII_DIGITS +from nemo_text_processing.text_normalization.ta.utils import get_abs_path as tn_abs_path + +CLOCK_MAX_HOUR = 23 + +# Bare "X மணி" is a duration (two hours); the hour-only form needs the dative மணிக்கு or a +# day-part word. அரை alone is half an hour, never half past twelve. +HOUR_NOUNS = ("மணிக்கு", "மணி") +MINUTE_NOUNS = ("நிமிடங்கள்", "நிமிடம்", "நிமிடத்திற்கு", "நிமிடத்தில்", "நிமிடத்துக்கு") +SECOND_NOUNS = ("வினாடிகள்", "வினாடி", "வினாடிக்கு", "வினாடியில்", "நொடி") +CLOCK_HOUR_NOUN = "மணிக்கு" +# The counting word for one minute or second (ஒரு நிமிடம் -> :01). +MINUTE_ONE = "ஒரு" + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying spoken times, e.g. + பத்து மணி முப்பது நிமிடம் -> time { hours: "10" minutes: "30" preserve_order: true } + பத்து மணிக்கு -> time { hours: "10" preserve_order: true } + பத்தரை மணிக்கு -> time { hours: "10" minutes: "30" preserve_order: true } + காலை பத்து மணி -> time { morphosyntactic_features: "காலை" hours: "10" preserve_order: true } + + A bare "X மணி" is a duration, so the hour-only form converts only with the dative மணிக்கு or a + fronted day-part word, which travels as ``morphosyntactic_features``. Reads the TN + ``data/time/{hours,minutes,seconds}.tsv`` tables from the spoken side. + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: CardinalFst): + super().__init__(name="time", kind="classify") + + def table_words(name: str) -> 'pynini.FstLike': + rows = [r for r in load_labels(tn_abs_path(f"data/time/{name}.tsv")) if len(r) >= 2] + return (pynini.invert(pynini.string_map([(k, v) for k, v, *_ in rows])) @ TO_ASCII_DIGITS).optimize() + + # Any spoken number up to 23 may head a time; ஒரு (the clock one) counts as 1 here. + hour_table = table_words("hours") + one_word = pynini.project(cardinal.words_to_digits @ pynini.accep("1"), "input") + hour_words = pynini.union( + cardinal.words_to_digits_licensed, cardinal.read(hour_table | pynini.cross(one_word, "1")) + ).optimize() + clock_hours = pynini.union(*[str(h) for h in range(CLOCK_MAX_HOUR + 1)]) + hour_words = (hour_words @ clock_hours).optimize() + minute_words = cardinal.read(table_words("minutes") | pynini.cross(MINUTE_ONE, "01")) + second_words = cardinal.read(table_words("seconds") | pynini.cross(MINUTE_ONE, "01")) + + hour_plain = pynutil.delete(pynini.union(*HOUR_NOUNS)) + minute_plain = pynutil.delete(pynini.union(*MINUTE_NOUNS)) + second_plain = pynutil.delete(pynini.union(*SECOND_NOUNS)) + clock_noun = delete_space + pynutil.delete(CLOCK_HOUR_NOUN) + + hours = pynutil.insert("hours: \"") + hour_words + pynutil.insert("\"") + minutes = pynutil.insert(" minutes: \"") + minute_words + pynutil.insert("\"") + seconds = pynutil.insert(" seconds: \"") + second_words + pynutil.insert("\"") + + graph_h = hours + clock_noun + graph_hm = hours + delete_space + hour_plain + delete_space + minutes + delete_space + minute_plain + graph_hms = graph_hm + delete_space + seconds + delete_space + second_plain + graph_hs = hours + delete_space + hour_plain + delete_space + seconds + delete_space + second_plain + # Hour and minute with no hour noun between them, as ASR often renders a clock time: + # பத்து முப்பது மணிக்கு -> 10:30. The dative is required, so a bare pair stays a number. + graph_hm_bare = hours + delete_space + minutes + clock_noun + + graph = graph_hms | graph_hm | graph_hs | graph_h | pynutil.add_weight(graph_hm_bare, 0.1) + + # Half- and quarter-hour idioms: பத்தரை மணிக்கு -> 10:30, பத்தே கால் மணிக்கு -> 10:15. Bare + # "Xரை மணி" is a duration (2.5 hours), so the clock reading needs the dative. + fused = pynini.union( + *[ + pynini.cross(word, f"hours: \"{ip}\" minutes: \"{FRACTION_MINUTES[fp]}\"") + for word, ip, fp, *_ in half_form_rows() + if fp in FRACTION_MINUTES and ip != "0" and int(ip) <= CLOCK_MAX_HOUR + ] + ) + fused |= quarter_form_graph( + hour_words, "hours: \"", "\"", lambda fraction: f" minutes: \"{FRACTION_MINUTES[fraction]}\"" + ) + graph |= fused + clock_noun + + # A fronted day-part word travels with the time and makes even a bare "X மணி" a clock + # time (காலை பத்து மணி -> காலை 10:00). + day_part = ( + pynutil.insert("morphosyntactic_features: \"") + + pynini.union(*DAY_PARTS) + + pynutil.insert("\" ") + + pynutil.delete(" ") + ) + bare_hour = (hours | fused) + delete_space + hour_plain + graph = pynini.closure(day_part, 0, 1) + graph | day_part + bare_hour + graph += pynutil.insert(" preserve_order: true") + self.fst = self.add_tokens(graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ta/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/ta/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..d21e37bb2 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/taggers/tokenize_and_classify.py @@ -0,0 +1,138 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ta.graph_utils import GraphFst, generator_main +from nemo_text_processing.inverse_text_normalization.ta.taggers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.ta.taggers.date import DateFst +from nemo_text_processing.inverse_text_normalization.ta.taggers.decimal import DecimalFst +from nemo_text_processing.inverse_text_normalization.ta.taggers.fraction import FractionFst +from nemo_text_processing.inverse_text_normalization.ta.taggers.money import MoneyFst +from nemo_text_processing.inverse_text_normalization.ta.taggers.ordinal import OrdinalFst +from nemo_text_processing.inverse_text_normalization.ta.taggers.punctuation import PunctuationFst +from nemo_text_processing.inverse_text_normalization.ta.taggers.telephone import TelephoneFst +from nemo_text_processing.inverse_text_normalization.ta.taggers.time import TimeFst +from nemo_text_processing.inverse_text_normalization.ta.taggers.whitelist import WhiteListFst +from nemo_text_processing.inverse_text_normalization.ta.taggers.word import WordFst +from nemo_text_processing.text_normalization.en.graph_utils import ( + INPUT_LOWER_CASED, + NEMO_SPACE, + NEMO_WHITE_SPACE, + delete_extra_space, + delete_space, +) +from nemo_text_processing.text_normalization.ta.taggers.cardinal import CardinalFst as TnCardinalFst + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + The spoken number forms are the Tamil TN cardinal's own grammar inverted, so the two + directions share one description of the number morphology. + + Args: + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + input_case: accepting either "lower_cased" or "cased" input. + """ + + def __init__( + self, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + input_case: str = INPUT_LOWER_CASED, + ): + super().__init__(name="tokenize_and_classify", kind="classify") + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + whitelist_file = os.path.basename(whitelist) if whitelist else "" + far_file = os.path.join(cache_dir, f"ta_itn_{input_case}_{whitelist_file}.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logging.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logging.info(f"Creating ClassifyFst grammars.") + cardinal = CardinalFst(TnCardinalFst()) + cardinal_graph = cardinal.fst + decimal_graph = DecimalFst(cardinal).fst + fraction_graph = FractionFst(cardinal).fst + ordinal_graph = OrdinalFst(cardinal).fst + date_graph = DateFst(cardinal).fst + time_graph = TimeFst(cardinal).fst + money_graph = MoneyFst(cardinal).fst + telephone_graph = TelephoneFst(cardinal).fst + whitelist_graph = WhiteListFst(input_file=whitelist).fst + punctuation = PunctuationFst() + punct_graph = punctuation.fst + word_graph = WordFst(punctuation).fst + + # A written number passes through (whitelist, below 1.0), then the classes from the + # most to the least specific reading of a spoken number. + classify = ( + pynutil.add_weight(whitelist_graph, 1.0) + | pynutil.add_weight(telephone_graph, 0.9) + | pynutil.add_weight(date_graph, 1.04) + | pynutil.add_weight(time_graph, 1.05) + | pynutil.add_weight(fraction_graph, 1.06) + | pynutil.add_weight(money_graph, 1.07) + | pynutil.add_weight(decimal_graph, 1.08) + | pynutil.add_weight(ordinal_graph, 1.09) + | pynutil.add_weight(cardinal_graph, 1.1) + ) + + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=2.1) + pynutil.insert(" }") + punct = pynini.closure( + pynini.union( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space), + (pynutil.insert(NEMO_SPACE) + punct), + ), + 1, + ) + + classify = pynini.union(classify, pynutil.add_weight(word_graph, 100)) + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(NEMO_SPACE)) + + token + + pynini.closure(pynutil.insert(NEMO_SPACE) + punct) + ) + + graph = token_plus_punct + pynini.closure( + pynini.union( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space), + (pynutil.insert(NEMO_SPACE) + punct + pynutil.insert(NEMO_SPACE)), + ) + + token_plus_punct + ) + + graph = delete_space + graph + delete_space + graph = pynini.union(graph, punct) + + self.fst = graph.optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) + logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/inverse_text_normalization/ta/taggers/whitelist.py b/nemo_text_processing/inverse_text_normalization/ta/taggers/whitelist.py new file mode 100644 index 000000000..b6c2475e4 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/taggers/whitelist.py @@ -0,0 +1,71 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ta.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.ta.taggers.cardinal import ambiguous_words, kept_scale_words +from nemo_text_processing.inverse_text_normalization.ta.utils import get_abs_path, load_rows +from nemo_text_processing.text_normalization.en.graph_utils import convert_space +from nemo_text_processing.text_normalization.ta.graph_utils import CURRENCY_SYMBOLS, NEMO_ALL_DIGIT, NEMO_TA_LETTER + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for classifying spans that must pass through ITN unchanged, e.g. + எல்லாம் ஒன்று -> tokens { name: "எல்லாம் ஒன்று" } + கால் வலிக்கிறது -> tokens { name: "கால் வலிக்கிறது" } + ₹5 கோடி -> tokens { name: "₹5 கோடி" } + + Three kinds of span: the phrases of ``data/whitelist/prose_phrases.tsv``, where a numeral + is a pronoun or an idiom; a ``standalone`` word of ``data/numbers/ambiguous.tsv`` (கால் + "leg", அரை "room") before another Tamil word, which is prose rather than a fraction; and an + already-written number (12.5%, 10-20, +91 9876543210, ₹5 கோடி, 2024ல்), which must not be + split into punctuation and digits or re-read. + + Args: + input_file: path to a file with whitelist replacements (each line: spoken\twritten), + added to the default spans + """ + + def __init__(self, input_file: str = None): + super().__init__(name="whitelist", kind="classify") + + phrases = [row[0] for row in load_rows(get_abs_path("data/whitelist/prose_phrases.tsv"), 1)] + prose = pynini.union(*phrases) + + standalone = [word for word, _ in ambiguous_words("standalone")] + followed = pynini.union(*standalone) + pynini.accep(" ") + pynini.closure(NEMO_TA_LETTER, 1) + + # A sign, a currency symbol, a glued case suffix (hyphenated or not) and a kept scale word + # all travel with the digits of a written number. + written = ( + pynini.closure(pynini.union("-", "+"), 0, 1) + + pynini.closure(pynini.union(*CURRENCY_SYMBOLS), 0, 1) + + pynini.closure(NEMO_ALL_DIGIT, 1) + + pynini.closure(pynini.union(*".:,/-") + pynini.closure(NEMO_ALL_DIGIT, 1)) + + pynini.closure("%", 0, 1) + + pynini.closure(pynini.closure("-", 0, 1) + pynini.closure(NEMO_TA_LETTER, 1), 0, 1) + + pynini.closure(" " + pynini.union(*kept_scale_words()), 0, 1) + ) + + # A written number outranks every reading; the prose spans only need to beat the number + # classes, which the tokenizer weights above 1.0. + graph = pynutil.add_weight(written, -0.2) | prose | followed + if input_file: + graph |= pynini.string_map([row[:2] for row in load_rows(input_file, 2)]) + + self.graph = convert_space(graph).optimize() + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ta/taggers/word.py b/nemo_text_processing/inverse_text_normalization/ta/taggers/word.py new file mode 100644 index 000000000..14c6e72f8 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/taggers/word.py @@ -0,0 +1,50 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ta.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.ta.taggers.punctuation import PunctuationFst +from nemo_text_processing.text_normalization.en.graph_utils import MIN_NEG_WEIGHT, NEMO_NOT_SPACE, convert_space +from nemo_text_processing.text_normalization.ta.graph_utils import NEMO_TA_BLOCK + +# Symbols a semiotic class owns, so the word class must not swallow them. +_CLASS_SYMBOLS = ["$", "€", "₩", "£", "¥", "#", "%"] + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying plain tokens, that do not belong to any special class. This can be considered as the default class. + e.g. வணக்கம் -> tokens { name: "வணக்கம்" } + + Args: + punctuation: PunctuationFst + """ + + def __init__(self, punctuation: PunctuationFst): + super().__init__(name="word", kind="classify") + + punct = punctuation.graph_input + default_graph = pynini.closure(pynini.difference(NEMO_NOT_SPACE, punct), 1) + symbols_to_exclude = (pynini.union(*_CLASS_SYMBOLS) | punct).optimize() + + graph = pynini.closure(pynini.difference(NEMO_TA_BLOCK, symbols_to_exclude), 1) + graph = pynutil.add_weight(graph, MIN_NEG_WEIGHT) | default_graph + + # No space is introduced around punctuation inside a word. + graph = pynini.closure(graph + pynini.closure(punct + graph, 0, 1)) + + self.graph = convert_space(graph) + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ta/utils.py b/nemo_text_processing/inverse_text_normalization/ta/utils.py new file mode 100644 index 000000000..af005995b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/utils.py @@ -0,0 +1,41 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from typing import List + +from nemo_text_processing.text_normalization.en.utils import load_labels + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +def load_rows(abs_path: str, min_fields: int) -> List[List[str]]: + """ + Loads a TSV table, keeping only the rows with at least ``min_fields`` columns. + + Args: + abs_path: absolute path of the table + min_fields: minimum number of columns a row must have + """ + return [row for row in load_labels(abs_path) if len(row) >= min_fields] diff --git a/nemo_text_processing/inverse_text_normalization/ta/verbalizers/__init__.py b/nemo_text_processing/inverse_text_normalization/ta/verbalizers/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/ta/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/ta/verbalizers/cardinal.py new file mode 100644 index 000000000..6af49255c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/verbalizers/cardinal.py @@ -0,0 +1,36 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ta.graph_utils import GraphFst +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, delete_space + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinals, e.g. + cardinal { negative: "true" integer: "120" } -> -120 + cardinal { positive: "true" integer: "5" } -> +5 + """ + + def __init__(self): + super().__init__(name="cardinal", kind="verbalize") + + sign = pynini.cross("negative: \"true\"", "-") | pynini.cross("positive: \"true\"", "+") + self.optional_sign = pynini.closure(sign + delete_space, 0, 1) + self.integer = pynutil.delete("integer: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + self.graph = self.optional_sign + self.integer + self.fst = self.delete_tokens(self.graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ta/verbalizers/date.py b/nemo_text_processing/inverse_text_normalization/ta/verbalizers/date.py new file mode 100644 index 000000000..868ca8db5 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/verbalizers/date.py @@ -0,0 +1,49 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ta.graph_utils import GraphFst +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + delete_preserve_order, + delete_space, + insert_space, +) + + +class DateFst(GraphFst): + """ + Finite state transducer for verbalizing dates, e.g. + date { day: "15" month: "ஜூன்" year: "2024" preserve_order: true } -> 15 ஜூன் 2024 + date { year: "2024" month: "ஜூன்" day: "15" preserve_order: true } -> 2024 ஜூன் 15 + """ + + def __init__(self): + super().__init__(name="date", kind="verbalize") + + def field(name: str) -> 'pynini.FstLike': + return pynutil.delete(f"{name}: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + day, month, year = field("day"), field("month"), field("year") + sep = delete_space + insert_space + graph = ( + day + sep + month + pynini.closure(sep + year, 0, 1) + | month + sep + year + | year + sep + month + sep + day + | month + sep + day + pynini.closure(sep + year, 0, 1) + ) + self.graph = graph + delete_preserve_order + self.fst = self.delete_tokens(self.graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ta/verbalizers/decimal.py b/nemo_text_processing/inverse_text_normalization/ta/verbalizers/decimal.py new file mode 100644 index 000000000..675677f18 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/verbalizers/decimal.py @@ -0,0 +1,45 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ta.graph_utils import GraphFst +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, delete_space, insert_space + + +class DecimalFst(GraphFst): + """ + Finite state transducer for verbalizing decimals, e.g. + decimal { integer_part: "12" fractional_part: "5" } -> 12.5 + decimal { integer_part: "5" fractional_part: "5" quantity: "லட்சம்" } -> 5.5 லட்சம் + """ + + def __init__(self): + super().__init__(name="decimal", kind="verbalize") + + sign = pynini.cross("negative: \"true\"", "-") | pynini.cross("positive: \"true\"", "+") + optional_sign = pynini.closure(sign + delete_space, 0, 1) + integer = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + fractional = pynutil.delete("fractional_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + quantity = pynutil.delete("quantity: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + self.graph = ( + optional_sign + + integer + + delete_space + + pynutil.insert(".") + + fractional + + pynini.closure(delete_space + insert_space + quantity, 0, 1) + ) + self.fst = self.delete_tokens(self.graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ta/verbalizers/fraction.py b/nemo_text_processing/inverse_text_normalization/ta/verbalizers/fraction.py new file mode 100644 index 000000000..6b71310b8 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/verbalizers/fraction.py @@ -0,0 +1,35 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ta.graph_utils import GraphFst +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, delete_space + + +class FractionFst(GraphFst): + """ + Finite state transducer for verbalizing fractions, e.g. + fraction { numerator: "3" denominator: "4" } -> 3/4 + fraction { denominator: "4" numerator: "3" } -> 3/4 + """ + + def __init__(self): + super().__init__(name="fraction", kind="verbalize") + + numerator = pynutil.delete("numerator: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + denominator = pynutil.delete("denominator: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + self.graph = numerator + delete_space + pynutil.insert("/") + denominator + self.fst = self.delete_tokens(self.graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ta/verbalizers/money.py b/nemo_text_processing/inverse_text_normalization/ta/verbalizers/money.py new file mode 100644 index 000000000..c5c07146c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/verbalizers/money.py @@ -0,0 +1,56 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ta.graph_utils import GraphFst +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + NEMO_NOT_QUOTE, + delete_preserve_order, + delete_space, +) + + +class MoneyFst(GraphFst): + """ + Finite state transducer for verbalizing money, e.g. + money { integer_part: "50" currency: "₹" fractional_part: "50" } -> ₹50.50 + money { integer_part: "50" currency: "₹" morphosyntactic_features: "க்கு" } -> ₹50க்கு + money { negative: "true" integer_part: "500" currency: "₹" } -> -₹500 + """ + + def __init__(self): + super().__init__(name="money", kind="verbalize") + + optional_sign = pynini.closure(pynini.cross("negative: \"true\"", "-") + delete_space, 0, 1) + currency = pynutil.delete("currency: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + integer = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + # A single spoken minor digit is tens of paise: ஐந்து பைசா is ₹0.05, not ₹0.5. + two_digits = pynini.union(NEMO_DIGIT + NEMO_DIGIT, pynutil.insert("0") + NEMO_DIGIT) + fraction = pynutil.delete("fractional_part: \"") + two_digits + pynutil.delete("\"") + suffix = ( + pynutil.delete("morphosyntactic_features: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) + self.graph = ( + optional_sign + + currency + + delete_space + + integer + + pynini.closure(delete_space + pynutil.insert(".") + fraction, 0, 1) + + pynini.closure(delete_space + suffix, 0, 1) + + delete_preserve_order + ) + self.fst = self.delete_tokens(self.graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ta/verbalizers/ordinal.py b/nemo_text_processing/inverse_text_normalization/ta/verbalizers/ordinal.py new file mode 100644 index 000000000..82334aabe --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/verbalizers/ordinal.py @@ -0,0 +1,37 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ta.graph_utils import GraphFst +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, delete_preserve_order, delete_space + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for verbalizing ordinals, e.g. + ordinal { integer: "5" morphosyntactic_features: "வது" preserve_order: true } -> 5வது + ordinal { integer: "10" morphosyntactic_features: "ஆம்" preserve_order: true } -> 10ஆம் + """ + + def __init__(self): + super().__init__(name="ordinal", kind="verbalize") + + integer = pynutil.delete("integer: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + marker = ( + pynutil.delete("morphosyntactic_features: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) + self.graph = integer + delete_space + marker + delete_preserve_order + self.fst = self.delete_tokens(self.graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ta/verbalizers/telephone.py b/nemo_text_processing/inverse_text_normalization/ta/verbalizers/telephone.py new file mode 100644 index 000000000..d688a503a --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/verbalizers/telephone.py @@ -0,0 +1,36 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ta.graph_utils import GraphFst +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, delete_space, insert_space + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for verbalizing telephone numbers, e.g. + telephone { number_part: "9943206870" } -> 9943206870 + telephone { country_code: "+91" number_part: "9876543210" } -> +91 9876543210 + telephone { country_code: "+91" } -> +91 + """ + + def __init__(self): + super().__init__(name="telephone", kind="verbalize") + + country_code = pynutil.delete("country_code: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + number_part = pynutil.delete("number_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + self.graph = (pynini.closure(country_code + delete_space + insert_space, 0, 1) + number_part) | country_code + self.fst = self.delete_tokens(self.graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ta/verbalizers/time.py b/nemo_text_processing/inverse_text_normalization/ta/verbalizers/time.py new file mode 100644 index 000000000..d0730a767 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/verbalizers/time.py @@ -0,0 +1,59 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ta.graph_utils import GraphFst +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + NEMO_NOT_QUOTE, + delete_preserve_order, + delete_space, + insert_space, +) + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing times, e.g. + time { hours: "10" minutes: "30" preserve_order: true } -> 10:30 + time { hours: "10" preserve_order: true } -> 10:00 + time { morphosyntactic_features: "காலை" hours: "10" preserve_order: true } -> காலை 10:00 + """ + + def __init__(self): + super().__init__(name="time", kind="verbalize") + + two_digits = pynini.union(NEMO_DIGIT + NEMO_DIGIT, pynutil.insert("0") + NEMO_DIGIT).optimize() + hours = pynutil.delete("hours: \"") + pynini.closure(NEMO_DIGIT, 1, 2) + pynutil.delete("\"") + minutes = pynutil.delete("minutes: \"") + two_digits + pynutil.delete("\"") + seconds = pynutil.delete("seconds: \"") + two_digits + pynutil.delete("\"") + # A fronted day-part word is written before the time. + day_part = pynini.closure( + pynutil.delete("morphosyntactic_features: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + + delete_space + + insert_space, + 0, + 1, + ) + + graph_h = hours + pynutil.insert(":00") + graph_hm = hours + delete_space + pynutil.insert(":") + minutes + graph_hms = graph_hm + delete_space + pynutil.insert(":") + seconds + graph_hs = hours + pynutil.insert(":00:") + delete_space + seconds + self.graph = day_part + (graph_hms | graph_hm | graph_hs | graph_h) + delete_preserve_order + self.fst = self.delete_tokens(self.graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ta/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/ta/verbalizers/verbalize.py new file mode 100644 index 000000000..ee801301d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/verbalizers/verbalize.py @@ -0,0 +1,48 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.ta.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.ta.verbalizers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.ta.verbalizers.date import DateFst +from nemo_text_processing.inverse_text_normalization.ta.verbalizers.decimal import DecimalFst +from nemo_text_processing.inverse_text_normalization.ta.verbalizers.fraction import FractionFst +from nemo_text_processing.inverse_text_normalization.ta.verbalizers.money import MoneyFst +from nemo_text_processing.inverse_text_normalization.ta.verbalizers.ordinal import OrdinalFst +from nemo_text_processing.inverse_text_normalization.ta.verbalizers.telephone import TelephoneFst +from nemo_text_processing.inverse_text_normalization.ta.verbalizers.time import TimeFst +from nemo_text_processing.inverse_text_normalization.ta.verbalizers.whitelist import WhiteListFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + """ + + def __init__(self): + super().__init__(name="verbalize", kind="verbalize") + + graph = ( + CardinalFst().fst + | DecimalFst().fst + | FractionFst().fst + | OrdinalFst().fst + | DateFst().fst + | TimeFst().fst + | MoneyFst().fst + | TelephoneFst().fst + | WhiteListFst().fst + ) + self.fst = graph diff --git a/nemo_text_processing/inverse_text_normalization/ta/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/ta/verbalizers/verbalize_final.py new file mode 100644 index 000000000..a53838afd --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/verbalizers/verbalize_final.py @@ -0,0 +1,45 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ta.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.ta.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.inverse_text_normalization.ta.verbalizers.word import WordFst +from nemo_text_processing.text_normalization.en.graph_utils import delete_extra_space, delete_space + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence, e.g. + tokens { cardinal { integer: "23" } } tokens { name: "பேர்" } tokens { name: "வந்தனர்" } -> 23 பேர் வந்தனர் + """ + + def __init__(self): + super().__init__(name="verbalize_final", kind="verbalize") + verbalize = VerbalizeFst().fst + word = WordFst().fst + types = verbalize | word + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + self.fst = graph diff --git a/nemo_text_processing/inverse_text_normalization/ta/verbalizers/whitelist.py b/nemo_text_processing/inverse_text_normalization/ta/verbalizers/whitelist.py new file mode 100644 index 000000000..a223a60bf --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/verbalizers/whitelist.py @@ -0,0 +1,40 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ta.graph_utils import GraphFst +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, delete_space + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for verbalizing whitelist spans, e.g. + tokens { name: "எல்லாம் ஒன்று" } -> எல்லாம் ஒன்று + """ + + def __init__(self): + super().__init__(name="whitelist", kind="verbalize") + + graph = ( + pynutil.delete("name:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_CHAR - " ", 1) + + pynutil.delete("\"") + ) + # Multi-word values travel with U+00A0 NO-BREAK SPACE; write them with plain spaces. + graph = graph @ pynini.cdrewrite(pynini.cross(" ", " "), "", "", NEMO_SIGMA) + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ta/verbalizers/word.py b/nemo_text_processing/inverse_text_normalization/ta/verbalizers/word.py new file mode 100644 index 000000000..410de9ab2 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ta/verbalizers/word.py @@ -0,0 +1,36 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ta.graph_utils import GraphFst +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, delete_space + + +class WordFst(GraphFst): + """ + Finite state transducer for verbalizing plain tokens, e.g. + tokens { name: "வணக்கம்" } -> வணக்கம் + """ + + def __init__(self): + super().__init__(name="word", kind="verbalize") + + # A value may itself be a U+0022 QUOTATION MARK token, so only the space is excluded. + chars = pynini.closure(NEMO_CHAR - " ", 1) + graph = pynutil.delete("name: \"") + chars + pynutil.delete("\"") + # Multi-word values travel with U+00A0 NO-BREAK SPACE; write them with plain spaces. + graph = graph @ pynini.cdrewrite(pynini.cross(" ", " "), "", "", NEMO_SIGMA) + self.fst = (delete_space + graph + delete_space).optimize() diff --git a/nemo_text_processing/text_normalization/normalize.py b/nemo_text_processing/text_normalization/normalize.py index d8ebf2f4d..36e59b2a3 100644 --- a/nemo_text_processing/text_normalization/normalize.py +++ b/nemo_text_processing/text_normalization/normalize.py @@ -191,6 +191,9 @@ def __init__( elif lang == 'ko': from nemo_text_processing.text_normalization.ko.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.text_normalization.ko.verbalizers.verbalize_final import VerbalizeFinalFst + elif lang == 'ta': + from nemo_text_processing.text_normalization.ta.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.text_normalization.ta.verbalizers.verbalize_final import VerbalizeFinalFst else: raise NotImplementedError(f"Language {lang} has not been supported yet.") @@ -737,7 +740,7 @@ def parse_args(): parser.add_argument( "--language", help="language", - choices=["en", "de", "es", "fr", "hu", "sv", "zh", "ar", "it", "hy", "ja", "hi", "ko", "vi", "pt"], + choices=["en", "de", "es", "fr", "hu", "sv", "zh", "ar", "it", "hy", "ja", "hi", "ko", "vi", "pt", "ta"], default="en", type=str, ) diff --git a/nemo_text_processing/text_normalization/run_evaluate.py b/nemo_text_processing/text_normalization/run_evaluate.py index 3a1964bbd..e3ffbbe56 100644 --- a/nemo_text_processing/text_normalization/run_evaluate.py +++ b/nemo_text_processing/text_normalization/run_evaluate.py @@ -35,7 +35,7 @@ def parse_args(): parser.add_argument( "--lang", help="language", - choices=['ar', 'de', 'en', 'es', 'fr', 'hu', 'it', 'ru', 'sv', 'zh', 'hy', 'hi', 'ko', 'vi', 'pt'], + choices=['ar', 'de', 'en', 'es', 'fr', 'hu', 'it', 'ru', 'sv', 'zh', 'hy', 'hi', 'ko', 'vi', 'pt', 'ta'], default="en", type=str, ) diff --git a/nemo_text_processing/text_normalization/ta/__init__.py b/nemo_text_processing/text_normalization/ta/__init__.py new file mode 100644 index 000000000..616482377 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.ta.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.text_normalization.ta.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.text_normalization.ta.verbalizers.verbalize_final import VerbalizeFinalFst diff --git a/nemo_text_processing/text_normalization/ta/data/README.md b/nemo_text_processing/text_normalization/ta/data/README.md new file mode 100644 index 000000000..f3a058c49 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/README.md @@ -0,0 +1,46 @@ +# Tamil data tables + +Two-column, tab-separated, NFC-normalized. `pynini.string_file` cannot carry a notes column, +so what each table holds and which grammars read it is recorded here. + +## Text normalization (this directory) + +| Table | Contents | Source / notes | +|---|---|---| +| `numbers/digit.tsv` | ௧-௯ → ஒன்று … ஒன்பது | Read by the cardinal tagger, from ASCII digits too | +| `numbers/zero.tsv` | ௦ → பூஜ்யம் | Read by the cardinal tagger | +| `numbers/teens_and_ties.tsv` | ௧௦-௯௯ → joined compound words | Read by the cardinal tagger | +| `numbers/hundred.tsv` | ௧௦௦ → நூறு | Read by the cardinal tagger | +| `numbers/hundreds_exact.tsv` | ௨௦௦-௯௦௦ → இருநூறு … தொள்ளாயிரம் | Read by the cardinal tagger | +| `numbers/hundreds_combined.tsv` | ௨-௮ → இருநூற்று … எண்ணூற்று | The joined sandhi stems (not bare prefixes such as முன்/நான், which would give wrong forms like "நான் நூற்று") | +| `numbers/quantity_words.tsv` | written scale word → spoken word → native\|english\|short | Read by the decimal, money and range taggers. `native` words are spoken as written, `english` (lakh, crore) and the glued `short` forms (L, cr, K, M, B) in Tamil | +| `date/{days,months,year_suffix}.tsv` | day/month numerals → words; era abbreviations | Read by the date tagger | +| `time/{hours,minutes,seconds}.tsv` | hours 0-24, minutes/seconds 1-59 → words | Minutes and seconds stop at 59 (10:60 is not a time). Hour 24 is admitted as 24:00 alone. Also read by the ITN time tagger from the spoken side | +| `money/currency.tsv` | symbol/code → currency word | Includes the ரூ./ரூ spellings | +| `money/major_minor_currencies.tsv` | major → minor unit word | Read by both directions: the TN money verbalizer emits these pairs and the ITN money tagger inverts them | +| `fraction/idiomatic.tsv` | numerator word, denominator word → everyday fraction word (ஒன்று இரண்டு → அரை) | 3 columns. The three pairs spoken as their own everyday words instead of the கீழ் reading | +| `measure/unit.tsv` | unit abbreviation → spoken unit | `st` (stone) is left out because it swallows English ordinals (1st); includes `மீ`, `லி`, `சத` and the dotless spellings | +| `telephone/number.tsv` | digit in either script → word | Also read by the serial, electronic and ITN telephone taggers | +| `whitelist/abbreviations.tsv` | abbreviation → expansion | Read by the whitelist tagger | +| `whitelist/symbol.tsv` | symbol → spoken word | `-` and `+` are left out (a lone hyphen or plus is punctuation, and a leading sign is a field of the number classes), as are `<` `>` (markup; spoken only between digits by the tokenizer) | +| `whitelist/percent_suffix.tsv` | `%` with a glued case suffix → the inflected percent word (%க்கு → சதவீதத்துக்கு) | Read by the tokenizer's spacing rewrites | +| `serial/letters.tsv` | A-Z → spoken English letter name | English letter names in Tamil script; read by the serial and electronic taggers | +| `electronic/symbols.tsv` | `. @ / - _ : ~ +` → spoken symbol inside an address | The words match `whitelist/symbol.tsv` where that table already spells one, so a symbol never reads two ways | +| `electronic/domains.tsv` | top-level domain → spoken form (com → காம், in → ஐ என்) | A bare domain is read as one only when it ends in a listed TLD | +| `roman/context.tsv` | cue word → written ordinal marker the numeral takes when it precedes the cue (வகுப்பு XII reads a cardinal, XII வகுப்பு an ordinal) | Latin cues (Class, Chapter) for mixed text | + +## Inverse text normalization (`inverse_text_normalization/ta/data`) + +| Table | Contents | Source / notes | +|---|---|---| +| `numbers/half_forms.tsv` | fused fractional words → integer/fraction digits (ஒன்றரை → 1.5) | 3 columns; the regular -ரை and -ே readings beyond the table are built in the decimal tagger | +| `numbers/ambiguous.tsv` | number words that are also ordinary words: word → condition → reading | `licensed` (ஒரு/ஓர், also the indefinite article) counts as a number only inside a money or clock reading; `standalone` (கால்/அரை/முக்கால்) only when no Tamil word follows | +| `numbers/scale_words.tsv` | scale word → trailing zeros → expand\|keep | `expand` multiplies the amount out (ஐந்து புள்ளி ஐந்து ஆயிரம் → 5500); `keep` leaves the written idiom (5.5 லட்சம்) | +| `whitelist/prose_phrases.tsv` | phrases where a numeral is a pronoun or idiom (ஒன்று சேர்) | Column 2 is the reason; protected verbatim by the ITN whitelist tagger | +| `money/currency.tsv` | spoken currency word → symbol | One row per output of the TN `money/currency.tsv` plus plurals. பவுண்டு is absent on purpose: that is the mass pound in `measure/unit.tsv`, the currency word is பவுண்ட் | +| `money/minor_units.tsv` | extra minor-unit word → symbol | Only the rows `major_minor_currencies.tsv` cannot supply (plurals, and ₹ காசு) | +| `fraction/denominator_locative.tsv` | locative -இல் form → cardinal word | The tabulated forms; the ITN fraction tagger also applies the regular locative, so any denominator round-trips | + +The ITN taggers read the TN `date/months.tsv`, `time/*.tsv`, `telephone/number.tsv` and +`money/major_minor_currencies.tsv` from the spoken side, so the two directions share one list +each and cannot drift apart. diff --git a/nemo_text_processing/text_normalization/ta/data/__init__.py b/nemo_text_processing/text_normalization/ta/data/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ta/data/date/__init__.py b/nemo_text_processing/text_normalization/ta/data/date/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/date/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ta/data/date/days.tsv b/nemo_text_processing/text_normalization/ta/data/date/days.tsv new file mode 100644 index 000000000..cb547b8ca --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/date/days.tsv @@ -0,0 +1,31 @@ +௦௧ ஒன்று +௦௨ இரண்டு +௦௩ மூன்று +௦௪ நான்கு +௦௫ ஐந்து +௦௬ ஆறு +௦௭ ஏழு +௦௮ எட்டு +௦௯ ஒன்பது +௧௦ பத்து +௧௧ பதினொன்று +௧௨ பன்னிரண்டு +௧௩ பதின்மூன்று +௧௪ பதினான்கு +௧௫ பதினைந்து +௧௬ பதினாறு +௧௭ பதினேழு +௧௮ பதினெட்டு +௧௯ பத்தொன்பது +௨௦ இருபது +௨௧ இருபத்தொன்று +௨௨ இருபத்திரண்டு +௨௩ இருபத்துமூன்று +௨௪ இருபத்துநான்கு +௨௫ இருபத்தைந்து +௨௬ இருபத்தாறு +௨௭ இருபத்தேழு +௨௮ இருபத்தெட்டு +௨௯ இருபத்தொன்பது +௩௦ முப்பது +௩௧ முப்பத்தொன்று diff --git a/nemo_text_processing/text_normalization/ta/data/date/months.tsv b/nemo_text_processing/text_normalization/ta/data/date/months.tsv new file mode 100644 index 000000000..8bfb3b631 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/date/months.tsv @@ -0,0 +1,12 @@ +௦௧ ஜனவரி +௦௨ பிப்ரவரி +௦௩ மார்ச் +௦௪ ஏப்ரல் +௦௫ மே +௦௬ ஜூன் +௦௭ ஜூலை +௦௮ ஆகஸ்ட் +௦௯ செப்டம்பர் +௧௦ அக்டோபர் +௧௧ நவம்பர் +௧௨ டிசம்பர் diff --git a/nemo_text_processing/text_normalization/ta/data/date/year_suffix.tsv b/nemo_text_processing/text_normalization/ta/data/date/year_suffix.tsv new file mode 100644 index 000000000..056cb1197 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/date/year_suffix.tsv @@ -0,0 +1,6 @@ +கி.பி. கிறிஸ்து பிறகு +கி.மு. கிறிஸ்து முன் +A.D. கிறிஸ்து பிறகு +B.C. கிறிஸ்து முன் +AD கிறிஸ்து பிறகு +BC கிறிஸ்து முன் diff --git a/nemo_text_processing/text_normalization/ta/data/electronic/__init__.py b/nemo_text_processing/text_normalization/ta/data/electronic/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/electronic/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ta/data/electronic/domains.tsv b/nemo_text_processing/text_normalization/ta/data/electronic/domains.tsv new file mode 100644 index 000000000..4ced8fc23 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/electronic/domains.tsv @@ -0,0 +1,18 @@ +com காம் +org ஆர்க் +net நெட் +edu எடு +gov கவ் +in ஐ என் +co கோ +ac ஏ சி +io ஐ ஓ +ai ஏ ஐ +uk யூ கே +us யூ எஸ் +info இன்ஃபோ +dev டெவ் +app ஆப் +me எம் ஈ +nic என் ஐ சி +res ஆர் ஈ எஸ் diff --git a/nemo_text_processing/text_normalization/ta/data/electronic/symbols.tsv b/nemo_text_processing/text_normalization/ta/data/electronic/symbols.tsv new file mode 100644 index 000000000..5567bbb59 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/electronic/symbols.tsv @@ -0,0 +1,8 @@ +. டாட் +@ எட் +/ வெட்டுக்கோடு +- டாஷ் +_ அடிக்கோடு +: கோலன் +~ டில்டு ++ பிளஸ் diff --git a/nemo_text_processing/text_normalization/ta/data/fraction/__init__.py b/nemo_text_processing/text_normalization/ta/data/fraction/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/fraction/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ta/data/fraction/idiomatic.tsv b/nemo_text_processing/text_normalization/ta/data/fraction/idiomatic.tsv new file mode 100644 index 000000000..9fe109219 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/fraction/idiomatic.tsv @@ -0,0 +1,3 @@ +ஒன்று இரண்டு அரை +ஒன்று நான்கு கால் +மூன்று நான்கு முக்கால் diff --git a/nemo_text_processing/text_normalization/ta/data/measure/__init__.py b/nemo_text_processing/text_normalization/ta/data/measure/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/measure/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ta/data/measure/unit.tsv b/nemo_text_processing/text_normalization/ta/data/measure/unit.tsv new file mode 100644 index 000000000..fc48f033b --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/measure/unit.tsv @@ -0,0 +1,155 @@ +°C டிகிரி செல்சியஸ் +°F டிகிரி பாரன்ஹீட் +K கெல்வின் +g கிராம் +kg கிலோகிராம் +mg மில்லிகிராம் +cg சென்டிகிராம் +dg டெசிகிராம் +Tg டெராகிராம் +Mg மெகாகிராம் +Gg கிகாகிராம் +hg ஹெக்டோகிராம் +dag டெக்காகிராம் +lb பவுண்டு +oz அவுன்ஸ் +t டன் +q குவிண்டால் +m மீட்டர் +cm சென்டிமீட்டர் +mm மில்லிமீட்டர் +km கிலோமீட்டர் +dm டெசிமீட்டர் +dam டெக்காமீட்டர் +nm நானோமீட்டர் +hm ஹெக்டோமீட்டர் +Mm மெகாமீட்டர் +mi மைல் +ft அடி +in அங்குலம் +yd யார்டு +µm மைக்ரோமீட்டர் +m² சதுர மீட்டர் +cm² சதுர சென்டிமீட்டர் +mm² சதுர மில்லிமீட்டர் +km² சதுர கிலோமீட்டர் +hm² சதுர ஹெக்டோமீட்டர் +dm² சதுர டெசிமீட்டர் +dam² சதுர டெக்காமீட்டர் +yd² சதுர யார்டு +ft² சதுர அடி +ac² சதுர ஏக்கர் +mi² சதுர மைல் +nm² சதுர நானோமீட்டர் +µm² சதுர மைக்ரோமீட்டர் +in² சதுர அங்குலம் +ha ஹெக்டேர் +ac ஏக்கர் +m³ கன மீட்டர் +cm³ கன சென்டிமீட்டர் +mm³ கன மில்லிமீட்டர் +dam³ கன டெக்காமீட்டர் +µm³ கன மைக்ரோமீட்டர் +l லிட்டர் +kl கிலோலிட்டர் +ml மில்லிலிட்டர் +dl டெசிலிட்டர் +hl ஹெக்டோலிட்டர் +cl சென்டிலிட்டர் +dal டெக்காலிட்டர் +L லிட்டர் +kL கிலோலிட்டர் +mL மில்லிலிட்டர் +dL டெசிலிட்டர் +hL ஹெக்டோலிட்டர் +cL சென்டிலிட்டர் +daL டெக்காலிட்டர் +GB கிகாபைட் +in³ கன அங்குலம் +ft³ கன அடி +yd³ கன யார்டு +mi³ கன மைல் +dm³ கன டெசிமீட்டர் +km³ கன கிலோமீட்டர் +nm³ கன நானோமீட்டர் +qt குவார்ட் +gal கேலன் +pt பைண்ட் +W வாட் +MW மெகாவாட் +KW கிலோவாட் +kW கிலோவாட் +b பிட் +Mb மெகாபிட் +B பைட் +KB கிலோபைட் +TB டெராபைட் +MB மெகாபைட் +PB பெட்டாபைட் +EB எக்சாபைட் +ZB ஜெட்டாபைட் +YB யோட்டாபைட் +C செல்சியஸ் +V வோல்ட் +Pa பாஸ்கல் +A ஆம்பியர் +J ஜூல் +s வினாடி +sec வினாடி +hr மணி +h மணி +min நிமிடம் +Ω ஓம் +MΩ மெகாஓம் +doz டஜன் +Hz ஹெர்ட்ஸ் +GHz கிகாஹெர்ட்ஸ் +KHz கிலோஹெர்ட்ஸ் +kHz கிலோஹெர்ட்ஸ் +N நியூட்டன் +dB டெசிபெல் +yr ஆண்டு +hp குதிரைத்திறன் +d நாள் +month மாதம் +months மாதங்கள் +ct காரட் +pH பிஎச் +km/h கிலோமீட்டர் ஒரு மணி நேரத்திற்கு +km/hr கிலோமீட்டர் ஒரு மணி நேரத்திற்கு +km/min கிலோமீட்டர் ஒரு நிமிடத்திற்கு +m/h மீட்டர் ஒரு மணி நேரத்திற்கு +m/hr மீட்டர் ஒரு மணி நேரத்திற்கு +mi/s மைல் ஒரு வினாடிக்கு +mi/h மைல் ஒரு மணி நேரத்திற்கு +mi/hr மைல் ஒரு மணி நேரத்திற்கு +mi/min மைல் ஒரு நிமிடத்திற்கு +x மடங்கு +X மடங்கு +* மடங்கு +% சதவீதம் +கி.மீ. கிலோமீட்டர் +கி.மீ கிலோமீட்டர் +கிமீ கிலோமீட்டர் +செ.மீ. சென்டிமீட்டர் +செ.மீ சென்டிமீட்டர் +செமீ சென்டிமீட்டர் +மி.மீ. மில்லிமீட்டர் +மி.மீ மில்லிமீட்டர் +மிமீ மில்லிமீட்டர் +மீ. மீட்டர் +கி.கி. கிலோகிராம் +கி.கி கிலோகிராம் +கிகி கிலோகிராம் +மி.லி. மில்லிலிட்டர் +மி.லி மில்லிலிட்டர் +மிலி மில்லிலிட்டர் +லி. லிட்டர் +cm2 சதுர சென்டிமீட்டர் +m2 சதுர மீட்டர் +km2 சதுர கிலோமீட்டர் +கி.மீ/மணி கிலோமீட்டர் ஒரு மணி நேரத்திற்கு +சத. சதவீதம் +மீ மீட்டர் +லி லிட்டர் +சத சதவீதம் diff --git a/nemo_text_processing/text_normalization/ta/data/money/__init__.py b/nemo_text_processing/text_normalization/ta/data/money/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/money/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ta/data/money/currency.tsv b/nemo_text_processing/text_normalization/ta/data/money/currency.tsv new file mode 100644 index 000000000..98b6b687a --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/money/currency.tsv @@ -0,0 +1,17 @@ +₹ ரூபாய் +RS ரூபாய் +Rs ரூபாய் +rs ரூபாய் +RS. ரூபாய் +Rs. ரூபாய் +rs. ரூபாய் +£ பவுண்ட் +₩ வான் +$ டாலர் +₺ லீரா +৳ டாக்கா +¥ யென் +₦ நைரா +€ யூரோ +ரூ. ரூபாய் +ரூ ரூபாய் diff --git a/nemo_text_processing/text_normalization/ta/data/money/major_minor_currencies.tsv b/nemo_text_processing/text_normalization/ta/data/money/major_minor_currencies.tsv new file mode 100644 index 000000000..0478c0924 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/money/major_minor_currencies.tsv @@ -0,0 +1,9 @@ +ரூபாய் பைசா +பவுண்ட் பென்ஸ் +வான் ஜியோன் +டாலர் சென்ட் +லீரா குருஸ் +டாக்கா பைசா +யென் சென் +நைரா கோபோ +யூரோ சென்ட் diff --git a/nemo_text_processing/text_normalization/ta/data/numbers/__init__.py b/nemo_text_processing/text_normalization/ta/data/numbers/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ta/data/numbers/digit.tsv b/nemo_text_processing/text_normalization/ta/data/numbers/digit.tsv new file mode 100644 index 000000000..1edfe8761 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +௧ ஒன்று +௨ இரண்டு +௩ மூன்று +௪ நான்கு +௫ ஐந்து +௬ ஆறு +௭ ஏழு +௮ எட்டு +௯ ஒன்பது diff --git a/nemo_text_processing/text_normalization/ta/data/numbers/hundred.tsv b/nemo_text_processing/text_normalization/ta/data/numbers/hundred.tsv new file mode 100644 index 000000000..7e3ef8526 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/numbers/hundred.tsv @@ -0,0 +1 @@ +௧௦௦ நூறு diff --git a/nemo_text_processing/text_normalization/ta/data/numbers/hundreds_combined.tsv b/nemo_text_processing/text_normalization/ta/data/numbers/hundreds_combined.tsv new file mode 100644 index 000000000..dc3007e17 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/numbers/hundreds_combined.tsv @@ -0,0 +1,7 @@ +௨ இருநூற்று +௩ முந்நூற்று +௪ நானூற்று +௫ ஐந்நூற்று +௬ அறுநூற்று +௭ எழுநூற்று +௮ எண்ணூற்று diff --git a/nemo_text_processing/text_normalization/ta/data/numbers/hundreds_exact.tsv b/nemo_text_processing/text_normalization/ta/data/numbers/hundreds_exact.tsv new file mode 100644 index 000000000..ae0f193c0 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/numbers/hundreds_exact.tsv @@ -0,0 +1,8 @@ +௨௦௦ இருநூறு +௩௦௦ முந்நூறு +௪௦௦ நானூறு +௫௦௦ ஐந்நூறு +௬௦௦ அறுநூறு +௭௦௦ எழுநூறு +௮௦௦ எண்ணூறு +௯௦௦ தொள்ளாயிரம் diff --git a/nemo_text_processing/text_normalization/ta/data/numbers/quantity_words.tsv b/nemo_text_processing/text_normalization/ta/data/numbers/quantity_words.tsv new file mode 100644 index 000000000..e28a0bfc8 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/numbers/quantity_words.tsv @@ -0,0 +1,23 @@ +கோடி கோடி native +இலட்சம் இலட்சம் native +லட்சம் லட்சம் native +ஆயிரம் ஆயிரம் native +மில்லியன் மில்லியன் native +பில்லியன் பில்லியன் native +டிரில்லியன் டிரில்லியன் native +thousand ஆயிரம் english +lakh இலட்சம் english +lakhs இலட்சம் english +crore கோடி english +crores கோடி english +million மில்லியன் english +billion பில்லியன் english +trillion டிரில்லியன் english +L இலட்சம் short +cr கோடி short +Cr கோடி short +CR கோடி short +K ஆயிரம் short +k ஆயிரம் short +M மில்லியன் short +B பில்லியன் short diff --git a/nemo_text_processing/text_normalization/ta/data/numbers/teens_and_ties.tsv b/nemo_text_processing/text_normalization/ta/data/numbers/teens_and_ties.tsv new file mode 100644 index 000000000..e75555aa6 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/numbers/teens_and_ties.tsv @@ -0,0 +1,90 @@ +௧௦ பத்து +௧௧ பதினொன்று +௧௨ பன்னிரண்டு +௧௩ பதின்மூன்று +௧௪ பதினான்கு +௧௫ பதினைந்து +௧௬ பதினாறு +௧௭ பதினேழு +௧௮ பதினெட்டு +௧௯ பத்தொன்பது +௨௦ இருபது +௨௧ இருபத்தொன்று +௨௨ இருபத்திரண்டு +௨௩ இருபத்துமூன்று +௨௪ இருபத்துநான்கு +௨௫ இருபத்தைந்து +௨௬ இருபத்தாறு +௨௭ இருபத்தேழு +௨௮ இருபத்தெட்டு +௨௯ இருபத்தொன்பது +௩௦ முப்பது +௩௧ முப்பத்தொன்று +௩௨ முப்பத்திரண்டு +௩௩ முப்பத்துமூன்று +௩௪ முப்பத்துநான்கு +௩௫ முப்பத்தைந்து +௩௬ முப்பத்தாறு +௩௭ முப்பத்தேழு +௩௮ முப்பத்தெட்டு +௩௯ முப்பத்தொன்பது +௪௦ நாற்பது +௪௧ நாற்பத்தொன்று +௪௨ நாற்பத்திரண்டு +௪௩ நாற்பத்துமூன்று +௪௪ நாற்பத்துநான்கு +௪௫ நாற்பத்தைந்து +௪௬ நாற்பத்தாறு +௪௭ நாற்பத்தேழு +௪௮ நாற்பத்தெட்டு +௪௯ நாற்பத்தொன்பது +௫௦ ஐம்பது +௫௧ ஐம்பத்தொன்று +௫௨ ஐம்பத்திரண்டு +௫௩ ஐம்பத்துமூன்று +௫௪ ஐம்பத்துநான்கு +௫௫ ஐம்பத்தைந்து +௫௬ ஐம்பத்தாறு +௫௭ ஐம்பத்தேழு +௫௮ ஐம்பத்தெட்டு +௫௯ ஐம்பத்தொன்பது +௬௦ அறுபது +௬௧ அறுபத்தொன்று +௬௨ அறுபத்திரண்டு +௬௩ அறுபத்துமூன்று +௬௪ அறுபத்துநான்கு +௬௫ அறுபத்தைந்து +௬௬ அறுபத்தாறு +௬௭ அறுபத்தேழு +௬௮ அறுபத்தெட்டு +௬௯ அறுபத்தொன்பது +௭௦ எழுபது +௭௧ எழுபத்தொன்று +௭௨ எழுபத்திரண்டு +௭௩ எழுபத்துமூன்று +௭௪ எழுபத்துநான்கு +௭௫ எழுபத்தைந்து +௭௬ எழுபத்தாறு +௭௭ எழுபத்தேழு +௭௮ எழுபத்தெட்டு +௭௯ எழுபத்தொன்பது +௮௦ எண்பது +௮௧ எண்பத்தொன்று +௮௨ எண்பத்திரண்டு +௮௩ எண்பத்துமூன்று +௮௪ எண்பத்துநான்கு +௮௫ எண்பத்தைந்து +௮௬ எண்பத்தாறு +௮௭ எண்பத்தேழு +௮௮ எண்பத்தெட்டு +௮௯ எண்பத்தொன்பது +௯௦ தொண்ணூறு +௯௧ தொண்ணூற்றொன்று +௯௨ தொண்ணூற்றிரண்டு +௯௩ தொண்ணூற்றுமூன்று +௯௪ தொண்ணூற்றுநான்கு +௯௫ தொண்ணூற்றைந்து +௯௬ தொண்ணூற்றாறு +௯௭ தொண்ணூற்றேழு +௯௮ தொண்ணூற்றெட்டு +௯௯ தொண்ணூற்றொன்பது diff --git a/nemo_text_processing/text_normalization/ta/data/numbers/zero.tsv b/nemo_text_processing/text_normalization/ta/data/numbers/zero.tsv new file mode 100644 index 000000000..1a6350c25 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/numbers/zero.tsv @@ -0,0 +1 @@ +௦ பூஜ்யம் diff --git a/nemo_text_processing/text_normalization/ta/data/roman/__init__.py b/nemo_text_processing/text_normalization/ta/data/roman/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/roman/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ta/data/roman/context.tsv b/nemo_text_processing/text_normalization/ta/data/roman/context.tsv new file mode 100644 index 000000000..70a2017a0 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/roman/context.tsv @@ -0,0 +1,29 @@ +வகுப்பு ஆம் +அத்தியாயம் ஆம் +பாகம் ஆம் +தொகுதி ஆம் +நூற்றாண்டு ஆம் +உலகப் போர் ஆம் +நிலை ஆம் +சுற்று ஆம் +பருவம் ஆம் +வகை ஆம் +பதிப்பு ஆம் +Class ஆம் +Chapter ஆம் +Part ஆம் +Volume ஆம் +Vol ஆம் +Section ஆம் +Grade ஆம் +Phase ஆம் +Stage ஆம் +Level ஆம் +Book ஆம் +Act ஆம் +Scene ஆம் +Round ஆம் +Season ஆம் +Type ஆம் +Mark ஆம் +Version ஆம் diff --git a/nemo_text_processing/text_normalization/ta/data/serial/__init__.py b/nemo_text_processing/text_normalization/ta/data/serial/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/serial/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ta/data/serial/letters.tsv b/nemo_text_processing/text_normalization/ta/data/serial/letters.tsv new file mode 100644 index 000000000..710461963 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/serial/letters.tsv @@ -0,0 +1,26 @@ +A ஏ +B பி +C சி +D டி +E ஈ +F எஃப் +G ஜி +H எச் +I ஐ +J ஜெ +K கே +L எல் +M எம் +N என் +O ஓ +P பி +Q கியூ +R ஆர் +S எஸ் +T டி +U யூ +V வி +W டபிள்யூ +X எக்ஸ் +Y வை +Z இசட் diff --git a/nemo_text_processing/text_normalization/ta/data/telephone/__init__.py b/nemo_text_processing/text_normalization/ta/data/telephone/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/telephone/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ta/data/telephone/number.tsv b/nemo_text_processing/text_normalization/ta/data/telephone/number.tsv new file mode 100644 index 000000000..862b195f4 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/telephone/number.tsv @@ -0,0 +1,20 @@ +0 பூஜ்யம் +1 ஒன்று +2 இரண்டு +3 மூன்று +4 நான்கு +5 ஐந்து +6 ஆறு +7 ஏழு +8 எட்டு +9 ஒன்பது +௦ பூஜ்யம் +௧ ஒன்று +௨ இரண்டு +௩ மூன்று +௪ நான்கு +௫ ஐந்து +௬ ஆறு +௭ ஏழு +௮ எட்டு +௯ ஒன்பது diff --git a/nemo_text_processing/text_normalization/ta/data/time/__init__.py b/nemo_text_processing/text_normalization/ta/data/time/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/time/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ta/data/time/hours.tsv b/nemo_text_processing/text_normalization/ta/data/time/hours.tsv new file mode 100644 index 000000000..5c9163b53 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/time/hours.tsv @@ -0,0 +1,25 @@ +௦ பூஜ்யம் +௧ ஒரு +௨ இரண்டு +௩ மூன்று +௪ நான்கு +௫ ஐந்து +௬ ஆறு +௭ ஏழு +௮ எட்டு +௯ ஒன்பது +௧௦ பத்து +௧௧ பதினொன்று +௧௨ பன்னிரண்டு +௧௩ பதின்மூன்று +௧௪ பதினான்கு +௧௫ பதினைந்து +௧௬ பதினாறு +௧௭ பதினேழு +௧௮ பதினெட்டு +௧௯ பத்தொன்பது +௨௦ இருபது +௨௧ இருபத்தொன்று +௨௨ இருபத்திரண்டு +௨௩ இருபத்துமூன்று +௨௪ இருபத்துநான்கு diff --git a/nemo_text_processing/text_normalization/ta/data/time/minutes.tsv b/nemo_text_processing/text_normalization/ta/data/time/minutes.tsv new file mode 100644 index 000000000..4591782be --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/time/minutes.tsv @@ -0,0 +1,59 @@ +௦௧ ஒன்று +௦௨ இரண்டு +௦௩ மூன்று +௦௪ நான்கு +௦௫ ஐந்து +௦௬ ஆறு +௦௭ ஏழு +௦௮ எட்டு +௦௯ ஒன்பது +௧௦ பத்து +௧௧ பதினொன்று +௧௨ பன்னிரண்டு +௧௩ பதின்மூன்று +௧௪ பதினான்கு +௧௫ பதினைந்து +௧௬ பதினாறு +௧௭ பதினேழு +௧௮ பதினெட்டு +௧௯ பத்தொன்பது +௨௦ இருபது +௨௧ இருபத்தொன்று +௨௨ இருபத்திரண்டு +௨௩ இருபத்துமூன்று +௨௪ இருபத்துநான்கு +௨௫ இருபத்தைந்து +௨௬ இருபத்தாறு +௨௭ இருபத்தேழு +௨௮ இருபத்தெட்டு +௨௯ இருபத்தொன்பது +௩௦ முப்பது +௩௧ முப்பத்தொன்று +௩௨ முப்பத்திரண்டு +௩௩ முப்பத்துமூன்று +௩௪ முப்பத்துநான்கு +௩௫ முப்பத்தைந்து +௩௬ முப்பத்தாறு +௩௭ முப்பத்தேழு +௩௮ முப்பத்தெட்டு +௩௯ முப்பத்தொன்பது +௪௦ நாற்பது +௪௧ நாற்பத்தொன்று +௪௨ நாற்பத்திரண்டு +௪௩ நாற்பத்துமூன்று +௪௪ நாற்பத்துநான்கு +௪௫ நாற்பத்தைந்து +௪௬ நாற்பத்தாறு +௪௭ நாற்பத்தேழு +௪௮ நாற்பத்தெட்டு +௪௯ நாற்பத்தொன்பது +௫௦ ஐம்பது +௫௧ ஐம்பத்தொன்று +௫௨ ஐம்பத்திரண்டு +௫௩ ஐம்பத்துமூன்று +௫௪ ஐம்பத்துநான்கு +௫௫ ஐம்பத்தைந்து +௫௬ ஐம்பத்தாறு +௫௭ ஐம்பத்தேழு +௫௮ ஐம்பத்தெட்டு +௫௯ ஐம்பத்தொன்பது diff --git a/nemo_text_processing/text_normalization/ta/data/time/seconds.tsv b/nemo_text_processing/text_normalization/ta/data/time/seconds.tsv new file mode 100644 index 000000000..4591782be --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/time/seconds.tsv @@ -0,0 +1,59 @@ +௦௧ ஒன்று +௦௨ இரண்டு +௦௩ மூன்று +௦௪ நான்கு +௦௫ ஐந்து +௦௬ ஆறு +௦௭ ஏழு +௦௮ எட்டு +௦௯ ஒன்பது +௧௦ பத்து +௧௧ பதினொன்று +௧௨ பன்னிரண்டு +௧௩ பதின்மூன்று +௧௪ பதினான்கு +௧௫ பதினைந்து +௧௬ பதினாறு +௧௭ பதினேழு +௧௮ பதினெட்டு +௧௯ பத்தொன்பது +௨௦ இருபது +௨௧ இருபத்தொன்று +௨௨ இருபத்திரண்டு +௨௩ இருபத்துமூன்று +௨௪ இருபத்துநான்கு +௨௫ இருபத்தைந்து +௨௬ இருபத்தாறு +௨௭ இருபத்தேழு +௨௮ இருபத்தெட்டு +௨௯ இருபத்தொன்பது +௩௦ முப்பது +௩௧ முப்பத்தொன்று +௩௨ முப்பத்திரண்டு +௩௩ முப்பத்துமூன்று +௩௪ முப்பத்துநான்கு +௩௫ முப்பத்தைந்து +௩௬ முப்பத்தாறு +௩௭ முப்பத்தேழு +௩௮ முப்பத்தெட்டு +௩௯ முப்பத்தொன்பது +௪௦ நாற்பது +௪௧ நாற்பத்தொன்று +௪௨ நாற்பத்திரண்டு +௪௩ நாற்பத்துமூன்று +௪௪ நாற்பத்துநான்கு +௪௫ நாற்பத்தைந்து +௪௬ நாற்பத்தாறு +௪௭ நாற்பத்தேழு +௪௮ நாற்பத்தெட்டு +௪௯ நாற்பத்தொன்பது +௫௦ ஐம்பது +௫௧ ஐம்பத்தொன்று +௫௨ ஐம்பத்திரண்டு +௫௩ ஐம்பத்துமூன்று +௫௪ ஐம்பத்துநான்கு +௫௫ ஐம்பத்தைந்து +௫௬ ஐம்பத்தாறு +௫௭ ஐம்பத்தேழு +௫௮ ஐம்பத்தெட்டு +௫௯ ஐம்பத்தொன்பது diff --git a/nemo_text_processing/text_normalization/ta/data/whitelist/__init__.py b/nemo_text_processing/text_normalization/ta/data/whitelist/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/whitelist/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ta/data/whitelist/abbreviations.tsv b/nemo_text_processing/text_normalization/ta/data/whitelist/abbreviations.tsv new file mode 100644 index 000000000..ce09c675c --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/whitelist/abbreviations.tsv @@ -0,0 +1,7 @@ +டாக். டாக்டர் +புரோ. பேராசிரியர் +இஞ். இன்ஜினியர் +லெ. லெப்டினன்ட் +வை. விஞ்ஞானி +கு. குமாரி +மா. மாஸ்டர் diff --git a/nemo_text_processing/text_normalization/ta/data/whitelist/percent_suffix.tsv b/nemo_text_processing/text_normalization/ta/data/whitelist/percent_suffix.tsv new file mode 100644 index 000000000..656adf30f --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/whitelist/percent_suffix.tsv @@ -0,0 +1,7 @@ +%க்கு சதவீதத்துக்கு +%க்கும் சதவீதத்துக்கும் +%ஆக சதவீதமாக +%ஆல் சதவீதத்தால் +%இல் சதவீதத்தில் +%ல் சதவீதத்தில் +%ஆவது சதவீதமாவது diff --git a/nemo_text_processing/text_normalization/ta/data/whitelist/symbol.tsv b/nemo_text_processing/text_normalization/ta/data/whitelist/symbol.tsv new file mode 100644 index 000000000..b97f93069 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/whitelist/symbol.tsv @@ -0,0 +1,146 @@ +& மற்றும் +# ஹாஷ் +@ எட் +§ பிரிவு +™ வர்த்தகச் சின்னம் +® பதிவுசெய்யப்பட்ட வர்த்தகச் சின்னம் +© பதிப்புரிமை +_ அடிக்கோடு +% சதவீதம் +* நட்சத்திரம் +/ வெட்டுக்கோடு += சமம் +^ கேரட் +| செங்குத்துக் கோடு +~ டில்டு +$ டாலர் +£ பவுண்ட் +€ யூரோ +₹ ரூபாய் +₩ வோன் +¥ யென் +° டிகிரி +º டிகிரி +κ கப்பா +ω ஓமேகா +α ஆல்ஃபா +ν நியூ +δ டெல்டா +Δ டெல்டா +Α ஆல்ஃபா +β பீடா +Β பீடா +χ கை +Χ கை +ε எப்சிலான் +Ε எப்சிலான் +φ பை +Φ பை +γ காமா +Γ காமா +η எதா +Η எதா +ι அயோடா +Ι அயோடா +κ கப்பா +Κ கப்பா +λ லாம்டா +Λ லாம்டா +μ மியூ +Μ மியூ +ν நியூ +Ν நியூ +ο ஒமிக்ரான் +Ο ஒமிக்ரான் +π பை +Π பை +θ தீடா +Θ தீடா +ρ ரோ +Ρ ரோ +σ சிக்மா +Σ சிக்மா +τ தாவ் +Τ தாவ் +υ அப்சிலான் +Υ அப்சிலான் +ω ஓமேகா +Ω ஓமேகா +ξ கிசை +Ξ கிசை +ψ சாய் +Ψ சாய் +ζ செதா +ζ செதா +Ζ செதா +± பிளஸ் மைனஸ் +∓ மைனஸ் பிளஸ் +√ வர்க்கமூலம் +∛ கனமூலம் +∜ நான்காம் மூலம் +∞ முடிவிலி +≠ சமமற்றது +≈ ஏறக்குறைய +≤ குறைவு அல்லது சமம் +≥ அதிகம் அல்லது சமம் +≡ சர்வசமம் +≢ சர்வசமம் அல்ல +≅ ஏறக்குறைய சமம் +∼ ஒத்தது +≃ ஏறக்குறைய சமம் +≪ மிகக் குறைவு +≫ மிக அதிகம் +∑ கூடுதல் +∏ பெருக்கற்பலன் +∫ தொகையீடு +∬ இரட்டைத் தொகையீடு +∭ மும்மைத் தொகையீடு +∮ கோட்டுத் தொகையீடு +∈ உறுப்பு +∉ உறுப்பு அல்ல +∋ உள்ளடக்கியது +∌ உள்ளடக்கவில்லை +∪ சேர்ப்பு +∩ வெட்டு +∖ கண வித்தியாசம் +⊆ உட்கணம் +⊂ தகு உட்கணம் +⊇ மேற்கணம் +⊃ தகு மேற்கணம் +⊈ உட்கணம் அல்ல +⊉ மேற்கணம் அல்ல +∅ வெற்று கணம் +∀ அனைத்திற்கும் +∃ உள்ளது +∄ இல்லை +¬ அல்ல +∧ மற்றும் +∨ அல்லது +⊕ நேரடி கூட்டல் +⊗ நேரடி பெருக்கல் +⊥ செங்குத்து +∥ இணை +∠ கோணம் +∟ செங்கோணம் +∂ பகுதி வகைக்கெழு +∇ நாப்ளா +∝ விகிதாசாரம் +∴ எனவே +∵ ஏனெனில் +∶ விகிதம் +ℝ மெய்யெண்கள் +ℂ சிக்கலெண்கள் +ℕ இயல் எண்கள் +ℤ முழுக்கள் +ℚ விகிதமுறு எண்கள் +⟹ குறிக்கிறது +⟸ பின்னோக்கி குறிக்கிறது +⟺ இருமறுக் குறிப்பு +⟷ சமான +→ வலது அம்பு +← இடது அம்பு +↔ இருமுனை அம்பு +↑ மேல் அம்பு +↓ கீழ் அம்பு +× பெருக்கல் +÷ வகுத்தல் diff --git a/nemo_text_processing/text_normalization/ta/graph_utils.py b/nemo_text_processing/text_normalization/ta/graph_utils.py new file mode 100644 index 000000000..112f169fe --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/graph_utils.py @@ -0,0 +1,259 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import sys +from functools import lru_cache +from pathlib import Path +from typing import Dict, List +from unicodedata import category + +import pynini +from pynini import Far +from pynini.export import export +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, NEMO_SIGMA, delete_space + +# Tamil digits occupy U+0BE6 TAMIL DIGIT ZERO to U+0BEF TAMIL DIGIT NINE. +TA_DIGITS = "".join(chr(0x0BE6 + i) for i in range(10)) +NEMO_TA_ZERO = TA_DIGITS[0] +NEMO_TA_DIGIT = pynini.union(*TA_DIGITS).optimize() +NEMO_TA_NON_ZERO = pynini.union(*TA_DIGITS[1:]).optimize() +# Combined Tamil and ASCII digits for graphs that read both scripts. +NEMO_ALL_DIGIT = pynini.union(NEMO_TA_DIGIT, NEMO_DIGIT).optimize() +NEMO_ALL_ZERO = pynini.union(NEMO_TA_ZERO, "0").optimize() +NEMO_ALL_NON_ZERO = pynini.union(NEMO_TA_NON_ZERO, pynini.difference(NEMO_DIGIT, "0")).optimize() + +# The Tamil block U+0B80-U+0BFF; the letters are the block minus its digits. +NEMO_TA_BLOCK = pynini.union(*[chr(i) for i in range(0x0B80, 0x0C00)]).optimize() +NEMO_TA_LETTER = pynini.difference(NEMO_TA_BLOCK, NEMO_TA_DIGIT).optimize() + +# Digit bridges between the two scripts, one digit at a time and over a whole run. +TA_TO_ASCII_DIGIT = pynini.string_map([(n, str(i)) for i, n in enumerate(TA_DIGITS)]).optimize() +ASCII_TO_TA_DIGIT = pynini.invert(TA_TO_ASCII_DIGIT).optimize() +# A run of digits in either script to ASCII, and a run of ASCII digits to Tamil. +TO_ASCII_DIGITS = pynini.closure(pynini.union(TA_TO_ASCII_DIGIT, NEMO_DIGIT)).optimize() +TO_TA_DIGITS = pynini.closure(ASCII_TO_TA_DIGIT).optimize() + +MINUS_WORD = "மைனஸ்" +PLUS_WORD = "பிளஸ்" +# Spoken between the bounds of a range (10-20 -> பத்து முதல் இருபது). +RANGE_WORD = "முதல்" +POINT_WORD = "புள்ளி" +# Read between the parts of a non-idiomatic fraction: 5/77 -> ஐந்து கீழ் எழுபத்தேழு. +FRACTION_WORD = "கீழ்" + +# Fractional-hour words used by the time and fraction grammars. +TA_KAAL = "கால்" +TA_ARAI = "அரை" +TA_MUKKAL = "முக்கால்" + +# Day-part words TN fronts before a clock time (காலை பத்து மணி) and ITN reads back. +DAY_PARTS = ("காலை", "அதிகாலை", "மதியம்", "நண்பகல்", "மாலை", "இரவு", "முற்பகல்", "பிற்பகல்") +DAY_PART_ABBREVIATIONS = { + "மு.ப.": "முற்பகல்", + "மு.ப": "முற்பகல்", + "பி.ப.": "பிற்பகல்", + "பி.ப": "பிற்பகல்", +} +AM_WORD = "முற்பகல்" +PM_WORD = "பிற்பகல்" + +# Vulgar fraction signs as spoken numerator and denominator words; the fraction verbalizer +# speaks the pair as its everyday word (ஒன்று/இரண்டு -> அரை). +VULGAR_PAIRS = {"½": ("ஒன்று", "இரண்டு"), "¼": ("ஒன்று", "நான்கு"), "¾": ("மூன்று", "நான்கு")} + +# Case suffixes written glued to a money amount (₹150க்கு), attached to the currency word +# by the money verbalizer; a glued ல் is spelled இல் in the field, the form its sandhi takes. +MONEY_SUFFIXES = ("க்கு", "க்கும்", "க்குள்", "ஆக", "ஆல்", "இல்") + +# Every case or ordinal suffix that may be written glued to a digit (2024ல், 100க்கு, +# 5வது); any other Tamil word glued to a digit is split off by the tokenizer. +GLUED_SUFFIXES = ( + "ல்", + "இல்", + "க்கு", + "க்கும்", + "க்குள்", + "கள்", + "களில்", + "உம்", + "ும்", + "ஆக", + "ஆல்", + "ால்", + "ஓடு", + "உடன்", + "ஐ", + "ன்", + "இன்", + "லிருந்து", + "இலிருந்து", + "த்தில்", + "த்துக்கு", + "தான்", + "ஆம்", + "ம்", + "ஆவது", + "வது", + "ஆவதாக", + "வதாக", +) + +# Currency symbols the money grammars read; also what may precede a re-fed written amount. +CURRENCY_SYMBOLS = "₹$£€¥₩₺৳₦" + +MIN_NEG_WEIGHT = -0.0001 +MIN_POS_WEIGHT = 0.0001 + + +@lru_cache(maxsize=None) +def punctuation_code_points() -> List[str]: + """ + Every Unicode punctuation code point, computed once per process on first use. + + The scan is ~1.1 M category lookups, so it is deferred: a process that only loads a + compiled grammar from the FAR cache never pays for it. + """ + return [chr(i) for i in range(sys.maxunicode + 1) if category(chr(i)).startswith("P")] + + +def rank(weight: float) -> 'pynini.FstLike': + """ + A weight-carrying epsilon for the tail of a union branch: at the head the same weight + would keep the branch's prefix from merging with its neighbours'. + + Args: + weight: weight of the epsilon arc + """ + return pynutil.insert("", weight) + + +def unweighted(fst: 'pynini.FstLike') -> 'pynini.FstLike': + """ + Drops every arc weight, leaving only the consuming grammar's own weights to rank paths. + + Args: + fst: input fst + """ + return pynini.arcmap(fst.optimize(), map_type="rmweight").optimize() + + +def sequential(fst: 'pynini.FstLike') -> 'pynini.FstLike': + """ + Input-deterministic form of an acyclic transducer, for grammars that read spoken words. + + An inverted TN grammar emits its digits before consuming any input (the TN side deleted + them), so composing a string with it explores the whole digit skeleton at every word + start, in every tagger that embeds it. Determinizing on the input delays each output + until the input that decides it has been read, so composition explores one path per + input prefix. The language, outputs and weights are unchanged. + + Args: + fst: an acyclic transducer; several outputs for one input are kept as alternatives + + Raises: + ValueError: if ``fst`` is cyclic, because determinization may then not terminate + """ + acyclic = pynini.ACYCLIC + if fst.properties(acyclic, True) != acyclic: + raise ValueError("sequential() needs an acyclic transducer.") + return pynini.determinize(fst, det_type="nonfunctional").optimize() + + +def generator_main(file_name: str, graphs: Dict[str, 'pynini.FstLike']): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logging.info(f'Created {file_name}') + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + '/grammars/' + kind + '/' + name + '.far') + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> 'pynini.FstLike': + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> 'pynini.FstLike': + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> 'pynini.FstLike': + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross(" ", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/text_normalization/ta/taggers/__init__.py b/nemo_text_processing/text_normalization/ta/taggers/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ta/taggers/cardinal.py b/nemo_text_processing/text_normalization/ta/taggers/cardinal.py new file mode 100644 index 000000000..ef3b1555f --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/taggers/cardinal.py @@ -0,0 +1,324 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, insert_space +from nemo_text_processing.text_normalization.ta.graph_utils import ( + ASCII_TO_TA_DIGIT, + GLUED_SUFFIXES, + NEMO_ALL_DIGIT, + NEMO_ALL_NON_ZERO, + NEMO_TA_LETTER, + TA_DIGITS, + GraphFst, + unweighted, +) +from nemo_text_processing.text_normalization.ta.utils import get_abs_path + +# The joined spoken forms of 150-189 (நூற்றைம்பது) that ITN must read; TN itself emits the +# linking form (நூற்று ஐம்பது). +_JOINED_TENS = { + 5: ("நூற்றைம்பது", "நூற்றைம்பத்து"), + 6: ("நூற்றறுபது", "நூற்றறுபத்து"), + 7: ("நூற்றெழுபது", "நூற்றெழுபத்து"), + 8: ("நூற்றெண்பது", "நூற்றெண்பத்து"), +} + +# Tails of the number words ending in -ஒன்று and the stem each takes when ஆயிரம் fuses onto +# it (இருபத்தொன்று ஆயிரம் -> இருபத்தோராயிரம்). +ONE_TAILS = (("ற்றொன்று", "ற்றோரா"), ("தொன்று", "தோரா"), ("ஒன்று", "ஓரா")) + + +def _ending(u: str, m: str) -> 'pynini.FstLike': + """ + Rewrite of a final U+0BC1 TAMIL VOWEL SIGN U to ``u`` and of a final ம் to ``m``. + """ + return pynini.union(pynini.cross("ு", u), pynini.cross("ம்", m)) + + +_DATIVE = pynini.union("க்கு", "க்குள்", "க்கும்") +_OPTIONAL_I = pynutil.delete(pynini.closure("இ", 0, 1)) + +# Written case suffix and the rewrite of the number word's ending it calls for (None leaves +# the word as it is). +CASE_SUFFIXES = ( + (pynutil.delete(pynini.union("ல்", "இல்")), _ending("ில்", "த்தில்")), + (pynutil.delete("த்தில்"), pynini.cross("ம்", "த்தில்")), + (_DATIVE, pynini.difference(NEMO_CHAR, "்")), + (_DATIVE, pynini.cross("ம்", "த்து")), + (pynini.union("கள்", "களில்"), None), + (pynini.cross("உம்", "ம்"), _ending("ு", "மு")), + (pynutil.delete("ஆ") + pynini.accep("க"), _ending("ா", "மா")), + (pynutil.delete("ஆ") + pynini.accep("ல்"), _ending("ா", "த்தா")), + (pynutil.delete("ஓ") + pynini.accep("டு"), _ending("ோ", "த்தோ")), + (pynutil.delete("உ") + pynini.accep("டன்"), _ending("ு", "த்து")), + (pynutil.delete("ஐ"), _ending("ை", "த்தை")), + (_OPTIONAL_I + pynini.accep("ன்"), _ending("ி", "த்தி")), + (_OPTIONAL_I + pynini.accep("லிருந்து"), _ending("ி", "த்தி")), + (pynini.accep("தான்"), None), +) +# Suffixes written with a bare vowel sign (5ால், 100ும்); TN reads them, ITN writes the +# independent-vowel spelling above instead. +SIGN_SPELLED_SUFFIXES = ( + (pynini.accep("ால்"), _ending("", "த்த")), + (pynini.accep("ும்"), _ending("", "ம")), +) + +# The adjectival stem replaces the cardinal's final -உ (or -ம்) with -ஆ: ஐந்து -> ஐந்தா, +# ஆயிரம் -> ஆயிரமா; then a written ordinal marker follows. +ORDINAL_STEM = NEMO_SIGMA + pynini.union(pynini.cross("ு", "ா"), pynini.cross("ம்", "மா")) +# ஆம் and the clipped ம் spell the same ordinal (28ஆம், 28ம்); any inflected tail after வத- +# is carried over (3ஆவதாக -> மூன்றாவதாக, 5வதுக்கு -> ஐந்தாவதுக்கு). +ORDINAL_MARKERS = pynini.union( + pynutil.delete(pynini.union("வது", "ஆவது")) + pynutil.insert("வது"), + pynutil.delete(pynini.union("ஆம்", "ம்")) + pynutil.insert("ம்"), + pynutil.delete(pynini.closure("ஆ", 0, 1)) + pynini.accep("வத") + pynini.closure(NEMO_TA_LETTER, 1), +).optimize() + + +def _dual_script(table: 'pynini.FstLike') -> 'pynini.FstLike': + """ + A table keyed by Tamil digits, also readable from ASCII digits. + """ + return pynini.union(table, pynini.closure(ASCII_TO_TA_DIGIT) @ table).optimize() + + +def _digit(index: int) -> 'pynini.FstLike': + """ + The digit ``index`` in either script. + """ + return pynini.union(str(index), TA_DIGITS[index]) + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals, e.g. + -௨௩ -> cardinal { negative: "true" integer: "இருபத்துமூன்று" } + 2024ல் -> cardinal { integer: "இரண்டாயிரத்து இருபத்துநான்கில்" } + 007 -> cardinal { integer: "பூஜ்யம் பூஜ்யம் ஏழு" } + + Numbers up to the crore range are read as words; longer digit runs and leading-zero runs + read digit by digit. The Indian (12,34,567) and international (1,234,567) grouping commas + are accepted, both closing with a 3-digit group; 1,5 and 15,06 are not groupings. + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + + digit = _dual_script(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) + zero = _dual_script(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) + teens_ties = _dual_script(pynini.string_file(get_abs_path("data/numbers/teens_and_ties.tsv"))) + hundred = _dual_script(pynini.string_file(get_abs_path("data/numbers/hundred.tsv"))) + hundreds_exact = _dual_script(pynini.string_file(get_abs_path("data/numbers/hundreds_exact.tsv"))) + hundreds_combined = _dual_script(pynini.string_file(get_abs_path("data/numbers/hundreds_combined.tsv"))) + self.digit = digit + self.zero = zero + # The spoken zero, which the decimal and money grammars insert on their own. + self.zero_word = pynini.shortestpath(zero.copy().project("output")).string() + + teens_and_ties = pynutil.add_weight(teens_ties, -0.1) + zero_delete = pynutil.add_weight(pynutil.delete(_digit(0)), -0.1) + + def zeros(count: int) -> 'pynini.FstLike': + return pynini.closure(zero_delete, count, count) if count else pynini.accep("") + + def scale(head, word: str, n_zeros: int, subs) -> 'pynini.FstLike': + """ + Exact multiple (all trailing zeros) or the head plus a remainder. + """ + suffix = pynutil.insert(word) + graph = head + zeros(n_zeros) + suffix + for count, sub in subs: + graph |= head + suffix + zeros(count) + insert_space + sub + return graph.optimize() + + # 100-199: நூறு, else the linking நூற்று; 150-189 join the hundred onto the tens. + hundred_link = pynutil.insert(" நூற்று") + insert_space + hundreds = ( + hundred + | pynutil.delete(_digit(1) + _digit(0)) + hundred_link + digit + | pynutil.delete(_digit(1)) + hundred_link + teens_ties + ) + # TN keeps the linking form (150 -> நூற்று ஐம்பது); the joined forms are spoken variants + # that ITN must read, so they stay in the graph a step behind. + for k, (exact, stem) in _JOINED_TENS.items(): + joined = pynini.cross(_digit(1) + _digit(k) + _digit(0), exact) + joined |= pynutil.delete(_digit(1) + _digit(k)) + pynutil.insert(stem) + insert_space + digit + hundreds |= pynutil.add_weight(joined, 0.01) + # 200-999: the exact hundreds, else the joined stem; 900 links as தொள்ளாயிரத்து. + link = hundreds_combined | pynutil.delete(_digit(9)) + pynutil.insert(" தொள்ளாயிரத்து") + hundreds |= hundreds_exact + hundreds |= link + pynutil.delete(_digit(0)) + insert_space + digit + hundreds |= link + insert_space + teens_ties + hundreds = hundreds.optimize() + + below_thousand = [(2, digit), (1, teens_ties), (0, hundreds)] + thousands = scale(digit, " ஆயிரம்", 3, below_thousand) + ten_thousands = scale(teens_and_ties, " ஆயிரம்", 3, below_thousand) + below_lakh = [(4, digit), (3, teens_ties), (2, hundreds), (1, thousands), (0, ten_thousands)] + lakhs = scale(digit, " இலட்சம்", 5, below_lakh) + ten_lakhs = scale(teens_and_ties, " இலட்சம்", 5, below_lakh) + below_crore = [(6, digit), (5, teens_ties), (4, hundreds), (3, thousands)] + below_crore += [(2, ten_thousands), (1, lakhs), (0, ten_lakhs)] + crores = scale(digit, " கோடி", 7, below_crore) + ten_crores = scale(teens_and_ties, " கோடி", 7, below_crore) + + # A leading zero is read out: 05 -> பூஜ்யம் ஐந்து. + digit_word = (digit | zero).optimize() + leading_zero = pynutil.add_weight(zero + insert_space + digit_word, 0.5) + number = pynini.union( + digit, + zero, + teens_and_ties, + hundreds, + thousands, + ten_thousands, + lakhs, + ten_lakhs, + crores, + ten_crores, + leading_zero, + ).optimize() + + # Spacing is normalized inside the graph itself (the hundreds insert a leading space), + # so inversion for ITN sees exactly the strings TN emits. + squeeze = pynini.cdrewrite(pynini.cross(pynini.closure(" ", 2), " "), "", "", NEMO_SIGMA) + strip_leading = pynini.cdrewrite(pynutil.delete(pynini.closure(" ", 1)), "[BOS]", "", NEMO_SIGMA) + self.raw_graph = (number @ squeeze @ strip_leading).optimize() + + # Sandhi: after a stem ending ற்று, a ப/த-initial word doubles its consonant and joins, + # e.g. நூற்று பத்து -> நூற்றுப்பத்து (110). + sandhi = pynini.cdrewrite( + pynini.union(pynini.cross(" ப", "ப்ப"), pynini.cross(" த", "த்த")), "ற்று", "", NEMO_SIGMA + ) + # Scale-word style: exactly one thousand is bare ஆயிரம்; a counting prefix before a + # scale word is ஒரு, not ஒன்று (ஒரு இலட்சம், ஒரு கோடி). + exact_end = pynini.union("[EOS]", " கோடி") + drop_one_exact = pynini.cdrewrite(pynini.cross("ஒன்று ஆயிரம்", "ஆயிரம்"), "[BOS]", exact_end, NEMO_SIGMA) + drop_one_rest = pynini.cdrewrite(pynini.cross("ஒன்று ஆயிரம்", "ஆயிரத்து"), "[BOS]", " ", NEMO_SIGMA) + oru_scales = pynini.cdrewrite( + pynini.cross("ஒன்று ", "ஒரு "), "[BOS]", pynini.union("இலட்சம்", "கோடி"), NEMO_SIGMA + ) + + # Thousands fuse with the number word in front of them: இரண்டு ஆயிரம் -> இரண்டாயிரம், and + # with a remainder இரண்டாயிரத்து (2024 -> இரண்டாயிரத்து இருபத்துநான்கு). Every multiplier + # ends in U+0BC1 TAMIL VOWEL SIGN U, which the fusion replaces, except the -ஒன்று words, + # whose tails take ஓர்; those go first so the generic rule never sees them. A compound + # multiplier fuses on its last component, so no left context is imposed. + def fuse(tail: str, right) -> 'pynini.FstLike': + one_words = pynini.string_map([(f"{word} ஆயிரம்", f"{stem}{tail}") for word, stem in ONE_TAILS]) + rest = pynini.cross("ு ஆயிரம்", f"ா{tail}") + return pynini.cdrewrite(one_words, "", right, NEMO_SIGMA) @ pynini.cdrewrite(rest, "", right, NEMO_SIGMA) + + # A scale word takes its oblique linking form when more of the number follows and its + # nominative form when the number ends there: 200000 -> இரண்டு இலட்சம், but + # 250000 -> இரண்டு இலட்சத்து ஐம்பதாயிரம். Thousands already do this above. + oblique_scales = pynini.cdrewrite( + pynini.union(pynini.cross("இலட்சம்", "இலட்சத்து"), pynini.cross("கோடி", "கோடியே")), "", " ", NEMO_SIGMA + ) + # Kept as three stages: ITN must keep accepting the plainer spoken variants + # (ஒரு இலட்சம் ஐம்பது ஆயிரம்) that TN itself no longer emits. + self.style_scales = (sandhi @ drop_one_exact @ drop_one_rest @ oru_scales).optimize() + self.style_fused = (self.style_scales @ fuse("யிரம்", exact_end) @ fuse("யிரத்து", " ")).optimize() + # The bare digit reading, which ITN inverts; the grouping commas below are TN input only. + self.number_graph = (self.raw_graph @ self.style_fused @ oblique_scales).optimize() + + # Grouping commas are deleted before the digits are read. A grouping opens with a + # non-zero digit, so 00,000 is two zero runs and a comma. + delete_comma = pynutil.delete(",") + two, three = NEMO_ALL_DIGIT**2, NEMO_ALL_DIGIT**3 + head = NEMO_ALL_NON_ZERO + pynini.closure(NEMO_ALL_DIGIT, 0, 1) + indian = head + pynini.closure(delete_comma + two) + delete_comma + three + international = head + pynini.closure(NEMO_ALL_DIGIT, 0, 1) + pynini.closure(delete_comma + three, 1) + grouped = pynini.union(indian, international).optimize() + self.final_graph = pynini.union(self.number_graph, grouped @ self.number_graph).optimize() + + # Digit-by-digit fallback for shapes the number grammar rejects, e.g. leading-zero runs + # (007) and digit strings beyond the crore range. A valid grouping beyond that range + # (12,34,56,78,901) reads digit by digit as one token, so it is penalised only enough to + # lose to any real number reading, not to a split at its commas. + digit_by_digit = (digit_word + pynini.closure(insert_space + digit_word, 1)).optimize() + self.digit_by_digit = digit_by_digit + grouped_digit_by_digit = pynutil.add_weight(grouped @ digit_by_digit, -18.0) + + # Case-suffixed numbers, e.g. 2024ல் -> ...இருபத்துநான்கில். + self.suffixed_graph = self.attach_case_suffix(self.final_graph) + + # A sign is a field, so the verbalizer renders it and ITN can invert it. + optional_sign = pynini.closure( + pynutil.insert("negative: ") + pynini.cross("-", "\"true\" ") + | pynutil.insert("positive: ") + pynini.cross("+", "\"true\" "), + 0, + 1, + ) + integer = ( + self.final_graph + | pynutil.add_weight(self.suffixed_graph, 0.1) + | pynutil.add_weight(self.digit_by_digit | grouped_digit_by_digit, 20.0) + ) + graph = optional_sign + pynutil.insert("integer: \"") + integer + pynutil.insert("\"") + self.fst = self.add_tokens(graph).optimize() + + # Every case or ordinal suffix that may stay glued to a digit; the tokenizer splits any + # other Tamil word off a digit. + ordinal_tail = pynini.union("வத", "ஆவத") + pynini.closure(NEMO_TA_LETTER, 1) + self.known_suffixes = pynini.union(*GLUED_SUFFIXES, ordinal_tail).optimize() + + def attach_case_suffix(self, graph: 'pynini.FstLike', include_vowel: bool = True) -> 'pynini.FstLike': + """ + Accepts a written case suffix after ``graph`` and attaches it to the last spoken word. + + Args: + graph: a digits-to-words transducer + include_vowel: if False, leave out the suffixes that are a bare vowel sign + """ + rows = CASE_SUFFIXES + (SIGN_SPELLED_SUFFIXES if include_vowel else ()) + return pynini.union( + *[(graph if ending is None else graph @ (NEMO_SIGMA + ending)) + written for written, ending in rows] + ).optimize() + + def ordinal_graph(self, graph: 'pynini.FstLike') -> 'pynini.FstLike': + """ + Reads ``graph`` followed by the written ordinal marker and an optional inflected tail. + """ + return ((graph @ ORDINAL_STEM) + ORDINAL_MARKERS).optimize() + + def readable_years(self) -> 'pynini.FstLike': + """ + The number readings ITN inverts, unweighted: the zero-deletion and teens bonuses are + TN's own preferences and would otherwise decide ITN token boundaries. + """ + return unweighted(self.number_graph) + + def itn_input_graph(self) -> 'pynini.FstLike': + """ + Every spoken form ITN inverts: the styled number and the plainer variants TN itself no + longer emits, minus the leading-zero pair, which ITN must read as the digit run 0 1 + rather than 01. Unweighted, because the zero-deletion and teens bonuses are TN's own + preferences and would otherwise decide ITN token boundaries. + """ + not_leading_zero = pynini.difference(NEMO_SIGMA, pynini.accep("பூஜ்யம் ") + NEMO_SIGMA) + variants = pynini.union( + self.raw_graph, + self.raw_graph @ self.style_scales, + self.raw_graph @ self.style_fused, + self.number_graph, + ) + return unweighted(variants @ not_leading_zero) diff --git a/nemo_text_processing/text_normalization/ta/taggers/date.py b/nemo_text_processing/text_normalization/ta/taggers/date.py new file mode 100644 index 000000000..421d65f6a --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/taggers/date.py @@ -0,0 +1,138 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_DIGIT, GraphFst, insert_space +from nemo_text_processing.text_normalization.ta.graph_utils import ( + ASCII_TO_TA_DIGIT, + NEMO_TA_DIGIT, + NEMO_TA_ZERO, + TO_TA_DIGITS, +) +from nemo_text_processing.text_normalization.ta.utils import get_abs_path + + +class DateFst(GraphFst): + """ + Finite state transducer for classifying dates, e.g. + 15-06-2024 -> date { day: "பதினைந்து" month: "ஜூன்" year: "இரண்டாயிரத்து இருபத்துநான்கு" } + 2024-06-15 -> date { year: "இரண்டாயிரத்து இருபத்துநான்கு" month: "ஜூன்" day: "பதினைந்து" } + 15-06-2024ல் -> date { day: "பதினைந்து" month: "ஜூன்" year: "இரண்டாயிரத்து இருபத்துநான்கில்" } + கி.பி. 2024 -> date { era: "கிறிஸ்து பிறகு" year: "இரண்டாயிரத்து இருபத்துநான்கு" } + + Reads ``data/date/days.tsv``, ``data/date/months.tsv`` and ``data/date/year_suffix.tsv``. + A numeric date needs all three components with a 4-digit year and one separator + throughout, so 15-06-24 and 10-20 are not dates. + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="date", kind="classify", deterministic=deterministic) + + days = pynini.string_file(get_abs_path("data/date/days.tsv")) + months = pynini.string_file(get_abs_path("data/date/months.tsv")) + year_suffix = pynini.string_file(get_abs_path("data/date/year_suffix.tsv")) + + # Two-digit day/month in either script; a single digit is zero-padded. + pad_zero = pynutil.insert(NEMO_TA_ZERO) + two_digit_input = pynini.union( + NEMO_TA_DIGIT + NEMO_TA_DIGIT, + pad_zero + NEMO_TA_DIGIT, + pynini.compose(NEMO_DIGIT + NEMO_DIGIT, TO_TA_DIGITS), + pad_zero + pynini.compose(NEMO_DIGIT, ASCII_TO_TA_DIGIT), + ).optimize() + days_graph = pynini.compose(two_digit_input, days).optimize() + months_graph = pynini.compose(two_digit_input, months).optimize() + + # Four-digit years. + year_graph = pynini.union( + pynini.compose(NEMO_TA_DIGIT**4, cardinal.final_graph), + pynini.compose(NEMO_DIGIT**4, cardinal.final_graph), + ).optimize() + + delete_separator = pynutil.delete(pynini.union("-", "/", ".")) + + # One date uses one separator throughout. That is enforced by filtering the input below + # rather than by building each ordering once per separator, which would triple the + # tagger; without it 15-06.2024 and 2024/06-15 also tag as dates. + not_separator = pynini.difference(NEMO_CHAR, pynini.union("-", "/", ".")) + one_separator = pynini.union( + *[ + pynini.closure(not_separator) + + separator + + pynini.closure(not_separator) + + separator + + pynini.closure(not_separator) + for separator in ("-", "/", ".") + ] + ).optimize() + + day_component = pynutil.insert("day: \"") + days_graph + pynutil.insert("\"") + month_component = pynutil.insert("month: \"") + months_graph + pynutil.insert("\"") + # A case suffix or ordinal marker on the date lands on the year (2024ல், 2024க்கு, 2024ஆம்). + year_component = ( + pynutil.insert("year: \"") + + (year_graph | cardinal.attach_case_suffix(year_graph) | cardinal.ordinal_graph(year_graph)) + + pynutil.insert("\"") + ) + + graph_dd_mm_yyyy = ( + day_component + + insert_space + + delete_separator + + month_component + + insert_space + + delete_separator + + year_component + ) + graph_mm_dd_yyyy = ( + month_component + + insert_space + + delete_separator + + day_component + + insert_space + + delete_separator + + year_component + + pynutil.insert(" preserve_order: true") + ) + graph_yyyy_mm_dd = ( + year_component + + insert_space + + delete_separator + + month_component + + insert_space + + delete_separator + + day_component + ) + + era_graph = pynutil.insert("era: \"") + year_suffix + pynutil.insert("\"") + # The year after an era word is a date's year. + era_graph |= era_graph + pynini.accep(" ") + pynutil.insert("year: \"") + year_graph + pynutil.insert("\"") + + numeric_dates = pynini.compose( + one_separator, + pynutil.add_weight(graph_dd_mm_yyyy, -0.001) + | pynutil.add_weight(graph_yyyy_mm_dd, -0.001) + | graph_mm_dd_yyyy, + ) + final_graph = numeric_dates | pynutil.add_weight(era_graph, -0.001) + + self.final_graph = final_graph.optimize() + self.fst = self.add_tokens(self.final_graph) diff --git a/nemo_text_processing/text_normalization/ta/taggers/decimal.py b/nemo_text_processing/text_normalization/ta/taggers/decimal.py new file mode 100644 index 000000000..16a56709f --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/taggers/decimal.py @@ -0,0 +1,116 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Tuple + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, insert_space +from nemo_text_processing.text_normalization.en.utils import load_labels +from nemo_text_processing.text_normalization.ta.graph_utils import POINT_WORD, rank +from nemo_text_processing.text_normalization.ta.utils import get_abs_path + + +def quantity_words() -> Tuple['pynini.FstLike', 'pynini.FstLike', 'pynini.FstLike']: + """ + The written scale ("quantity") words a number may carry, from ``data/numbers/quantity_words.tsv`` + (written, spoken, kind), grouped by how they attach to the number. + + Returns: + spaced: native and English words that follow the number after a space (கோடி, lakh) + short: shorthands that may be glued to the number (L, cr, K, M) + native: native words only, for a second stacked scale word (₹1 லட்சம் கோடி) + """ + by_kind = {"native": [], "english": [], "short": []} + for written, spoken, kind, *_ in load_labels(get_abs_path("data/numbers/quantity_words.tsv")): + by_kind[kind].append((written, spoken)) + native = pynini.string_map(by_kind["native"]).optimize() + spaced = pynini.union(native, pynini.string_map(by_kind["english"])).optimize() + short = pynini.string_map(by_kind["short"]).optimize() + return spaced, short, native + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimals, e.g. + -12.5006 கோடி -> decimal { negative: "true" integer_part: "பன்னிரண்டு" fractional_part: "ஐந்து பூஜ்யம் பூஜ்யம் ஆறு" quantity: "கோடி" } + +5.5 -> decimal { positive: "true" integer_part: "ஐந்து" fractional_part: "ஐந்து" } + 1 கோடி -> decimal { integer_part: "ஒன்று" quantity: "கோடி" } + .5 -> decimal { integer_part: "பூஜ்யம்" fractional_part: "ஐந்து" } + 1.2.3 -> decimal { integer_part: "ஒன்று" fractional_part: "இரண்டு புள்ளி மூன்று" } + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + + graph_digit = cardinal.digit | cardinal.zero + cardinal_graph = cardinal.final_graph + + # Digits in either script read one at a time (the fractional reading). + self.graph = (graph_digit + pynini.closure(insert_space + graph_digit)).optimize() + + point = pynutil.delete(".") + + optional_sign = pynini.closure( + ( + pynutil.insert("negative: ") + pynini.cross("-", "\"true\"") + | pynutil.insert("positive: ") + pynini.cross("+", "\"true\"") + ) + + insert_space, + 0, + 1, + ) + + self.graph_fractional = ( + pynutil.insert("fractional_part: \"") + + (self.graph | cardinal.attach_case_suffix(self.graph) + rank(0.1)) + + pynutil.insert("\"") + ) + self.graph_integer = pynutil.insert("integer_part: \"") + cardinal_graph + pynutil.insert("\"") + + final_graph_wo_sign = self.graph_integer + point + insert_space + self.graph_fractional + + # Bare-dot decimals: .5 reads as . + bare_dot = ( + pynutil.insert(f"integer_part: \"{cardinal.zero_word}\"") + point + insert_space + self.graph_fractional + ) + # Dotted chains (versions, IPs): every segment after the first reads digit-by-digit with + # the point word between them. + dotted_chain = ( + self.graph_integer + + point + + insert_space + + pynutil.insert("fractional_part: \"") + + self.graph + + pynini.closure(pynini.cross(".", f" {POINT_WORD} ") + self.graph, 1) + + pynutil.insert("\"") + ) + final_graph_wo_sign |= pynutil.add_weight(bare_dot, 0.1) + final_graph_wo_sign |= pynutil.add_weight(dotted_chain, 0.5) + + # A cardinal or decimal followed by a quantity word (5 லட்சம், 1.5 கோடி, 2 lakh). + spaced, _, _ = quantity_words() + quantity = pynutil.delete(" ") + insert_space + pynutil.insert("quantity: \"") + spaced + pynutil.insert("\"") + with_quantity = self.graph_integer + quantity + with_quantity |= final_graph_wo_sign + quantity + + self.final_graph_wo_negative = final_graph_wo_sign | with_quantity + final_graph = optional_sign + self.final_graph_wo_negative + self.fst = self.add_tokens(final_graph).optimize() diff --git a/nemo_text_processing/text_normalization/ta/taggers/electronic.py b/nemo_text_processing/text_normalization/ta/taggers/electronic.py new file mode 100644 index 000000000..7e4727fa0 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/taggers/electronic.py @@ -0,0 +1,113 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + GraphFst, + convert_space, + insert_space, +) +from nemo_text_processing.text_normalization.en.utils import load_labels +from nemo_text_processing.text_normalization.ta.taggers.serial import digit_words, letter_names +from nemo_text_processing.text_normalization.ta.utils import get_abs_path + +# Characters a URL path may hold; the symbols the whitelist speaks (# % &) are split off by the +# tokenizer before the text reaches the tagger, so a path stops at them. +PATH_SYMBOLS = "./-_~:+" + +# Every piece of an address costs the same, so the cheapest reading is the one with the fewest +# pieces: a run of letters is read whole rather than split into shorter runs. +_PIECE_WEIGHT = 1.0 + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for classifying electronic addresses, e.g. + kumar@gmail.com -> tokens { name: "kumar எட் gmail டாட் காம்" } + www.example.com/page2 -> tokens { name: "டபிள்யூ டபிள்யூ டபிள்யூ டாட் example டாட் காம் வெட்டுக்கோடு page இரண்டு" } + 192.168.1.1 -> tokens { name: "ஒன்று ஒன்பது இரண்டு டாட் ..." } + @handle -> tokens { name: "எட் handle" } + + A Latin word is left for the voice to read; a lone letter is spelled, a digit is read on its + own, and the symbols come from ``data/electronic/symbols.tsv``. A top-level domain reads from + ``data/electronic/domains.tsv`` and is what makes a bare domain one (example.com); http:// + and https:// are not spoken. + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="electronic", kind="classify", deterministic=deterministic) + + letter = letter_names() + digit = digit_words() + symbols = pynini.string_file(get_abs_path("data/electronic/symbols.tsv")) + tld = pynutil.add_weight( + pynini.string_map([(k, v) for k, v, *_ in load_labels(get_abs_path("data/electronic/domains.tsv"))]), + -0.01, + ) + tld_shape = pynini.project(tld, "input") + dot = pynini.accep(".") @ symbols + + piece = pynini.union(pynini.closure(NEMO_ALPHA, 2), letter, digit, pynini.union(*PATH_SYMBOLS) @ symbols) + piece = pynutil.add_weight(piece, _PIECE_WEIGHT) + run = piece + pynini.closure(insert_space + piece) + + # Labels joined by dots, closing with a known top-level domain, possibly after a + # second-level one that is also in the table (co.in). + label_shape = pynini.closure(pynini.union(NEMO_ALPHA, NEMO_DIGIT, "-"), 1) + domain_shape = pynini.closure(label_shape + ".", 1) + tld_shape + pynini.closure("." + tld_shape, 0, 1) + domain = domain_shape @ ( + pynini.closure((label_shape @ run) + insert_space + dot + insert_space, 1) + + tld + + pynini.closure(insert_space + dot + insert_space + tld, 0, 1) + ) + + at = pynini.accep("@") @ symbols + local = pynini.closure(pynini.union(NEMO_ALPHA, NEMO_DIGIT, *".-_+"), 1) @ run + email = local + insert_space + at + insert_space + domain + + # "www" is spelled rather than read as a word, and outranks the plain domain reading. + w = pynini.shortestpath(pynini.accep("W") @ letter).string() + www = ( + pynutil.add_weight( + pynutil.delete("www") + pynutil.insert(f"{w} {w} {w}") + insert_space + dot, -_PIECE_WEIGHT + ) + + insert_space + ) + protocol = pynutil.delete(pynini.union("http://", "https://")) + path = pynini.closure(pynini.union(NEMO_ALPHA, NEMO_DIGIT, *PATH_SYMBOLS), 1) @ run + slash = pynini.accep("/") @ symbols + optional_path = pynini.closure(insert_space + slash + pynini.closure(insert_space + path, 0, 1), 0, 1) + url = pynini.union(protocol + pynini.closure(www, 0, 1) + domain, www + domain, domain) + optional_path + + octet = pynini.closure(NEMO_DIGIT, 1, 3) @ (digit + pynini.closure(insert_space + digit)) + ip = octet + pynini.closure(insert_space + dot + insert_space + octet, 3, 3) + + handle = at + insert_space + (pynini.closure(pynini.union(NEMO_ALPHA, NEMO_DIGIT, "_"), 1) @ run) + + graph = pynini.union( + pynutil.add_weight(email, 0.1), + pynutil.add_weight(url, 0.2), + pynutil.add_weight(ip, 0.1), + pynutil.add_weight(handle, 0.3), + ) + self.graph = convert_space(graph).optimize() + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/nemo_text_processing/text_normalization/ta/taggers/fraction.py b/nemo_text_processing/text_normalization/ta/taggers/fraction.py new file mode 100644 index 000000000..4a70c57a5 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/taggers/fraction.py @@ -0,0 +1,77 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst +from nemo_text_processing.text_normalization.ta.graph_utils import NEMO_ALL_DIGIT, NEMO_ALL_ZERO, VULGAR_PAIRS + + +class FractionFst(GraphFst): + """ + Finite state transducer for classifying fractions, e.g. + 3/4 -> fraction { numerator: "மூன்று" denominator: "நான்கு" } + 2 3/4 -> fraction { integer_part: "இரண்டு" numerator: "மூன்று" denominator: "நான்கு" } + ½ -> fraction { numerator: "ஒன்று" denominator: "இரண்டு" } + + A vulgar sign is tagged as its numerator and denominator words, so the verbalizer speaks + it as the everyday word (அரை) exactly as it does 1/2. + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="fraction", kind="classify", deterministic=deterministic) + + cardinal_graph = cardinal.final_graph + any_digit = NEMO_ALL_DIGIT + + # A zero or zero-led denominator (1/0, 15/06) is not a fraction. + non_zero_led = pynini.difference( + pynini.closure(any_digit, 1), NEMO_ALL_ZERO + pynini.closure(any_digit) + ).optimize() + denominator_graph = pynini.compose(non_zero_led, cardinal_graph).optimize() + + integer = pynutil.insert("integer_part: \"") + cardinal_graph + pynutil.insert("\"") + # A zero-led numerator (06/24) is a date fragment, not a fraction. + numerator_input = pynini.difference( + pynini.closure(any_digit, 1), NEMO_ALL_ZERO + pynini.closure(any_digit, 1) + ).optimize() + numerator = ( + pynutil.insert("numerator: \"") + + pynini.compose(numerator_input, cardinal_graph) + + (pynini.cross("/", "\" ") | pynini.cross(" / ", "\" ")) + ) + denominator = pynutil.insert("denominator: \"") + denominator_graph + pynutil.insert("\"") + + graph = pynini.closure(integer + pynini.accep(" "), 0, 1) + numerator + denominator + optional_negative = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + graph = optional_negative + graph + + # Vulgar signs, alone or after an integer (2¾, 12 ½). + optional_space = pynutil.delete(pynini.closure(" ", 0, 1)) + pairs = pynini.union( + *[ + pynutil.delete(sign) + pynutil.insert(f"numerator: \"{num}\" denominator: \"{den}\"") + for sign, (num, den) in VULGAR_PAIRS.items() + ] + ) + graph |= optional_negative + (pynini.closure(integer + optional_space + pynutil.insert(" "), 0, 1) + pairs) + + self.graph = graph + self.fst = self.add_tokens(self.graph).optimize() diff --git a/nemo_text_processing/text_normalization/ta/taggers/measure.py b/nemo_text_processing/text_normalization/ta/taggers/measure.py new file mode 100644 index 000000000..9bcc1c751 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/taggers/measure.py @@ -0,0 +1,91 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import ( + TO_LOWER, + GraphFst, + convert_space, + delete_zero_or_one_space, + insert_space, +) +from nemo_text_processing.text_normalization.en.utils import load_labels +from nemo_text_processing.text_normalization.ta.graph_utils import RANGE_WORD +from nemo_text_processing.text_normalization.ta.utils import get_abs_path + +# Single letters that are far more often part of an identifier (47A, 5G) than a unit. +ID_PRONE = frozenset("ABCGJKNVWXbdhqsx*") + + +class MeasureFst(GraphFst): + """ + Finite state transducer for classifying measures, e.g. + 5 கி.மீ. -> measure { cardinal { integer: "ஐந்து" } units: "கிலோமீட்டர்" preserve_order: true } + 12.5kg -> measure { decimal { integer_part: "பன்னிரண்டு" fractional_part: "ஐந்து" } units: "கிலோகிராம்" preserve_order: true } + -40°C -> measure { negative: "true" cardinal { integer: "நாற்பது" } units: "டிகிரி செல்சியஸ்" preserve_order: true } + 5-10 kg -> measure { cardinal { integer: "ஐந்து முதல் பத்து" } units: "கிலோகிராம்" preserve_order: true } + + Reads ``data/measure/unit.tsv``; a single-letter unit needs a space before it, so a glued + identifier such as 47A is left to the serial class. + + Args: + cardinal: CardinalFst + decimal: DecimalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="measure", kind="classify", deterministic=deterministic) + + rows = [r for r in load_labels(get_abs_path("data/measure/unit.tsv")) if len(r) >= 2] + multi = pynini.string_map([(k, v) for k, v, *_ in rows if len(k) > 1 or k not in ID_PRONE]).optimize() + single = pynini.string_map([(k, v) for k, v, *_ in rows if len(k) == 1]).optimize() + + # Accept uppercase spellings of Latin units (5KG). + lowercase = pynini.closure(TO_LOWER | pynini.union(*"abcdefghijklmnopqrstuvwxyz°²./"), 2) + multi |= pynini.compose(lowercase, multi).optimize() + + unit_multi = convert_space(multi).optimize() + unit_single = convert_space(single).optimize() + unit_part = (delete_zero_or_one_space + unit_multi) | (pynutil.delete(" ") + unit_single) + + # 5-10 kg reads as a range amount. + amount = pynini.union( + cardinal.final_graph, cardinal.final_graph + pynini.cross("-", f" {RANGE_WORD} ") + cardinal.final_graph + ).optimize() + graph_cardinal = pynutil.insert("cardinal { integer: \"") + amount + pynutil.insert("\" }") + graph_decimal = ( + pynutil.insert("decimal { ") + + decimal.graph_integer + + pynutil.delete(".") + + insert_space + + pynutil.insert("fractional_part: \"") + + decimal.graph + + pynutil.insert("\" }") + ) + + optional_negative = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + + graph = ( + optional_negative + + (graph_cardinal | graph_decimal) + + pynutil.insert(" units: \"") + + unit_part + + pynutil.insert("\"") + + pynutil.insert(" preserve_order: true") + ) + self.fst = self.add_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/ta/taggers/money.py b/nemo_text_processing/text_normalization/ta/taggers/money.py new file mode 100644 index 000000000..99291321a --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/taggers/money.py @@ -0,0 +1,201 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, insert_space +from nemo_text_processing.text_normalization.en.utils import load_labels +from nemo_text_processing.text_normalization.ta.graph_utils import ( + CURRENCY_SYMBOLS, + MONEY_SUFFIXES, + NEMO_ALL_DIGIT, + NEMO_TA_DIGIT, + NEMO_TA_NON_ZERO, + NEMO_TA_ZERO, + POINT_WORD, + RANGE_WORD, +) +from nemo_text_processing.text_normalization.ta.taggers.decimal import quantity_words +from nemo_text_processing.text_normalization.ta.utils import get_abs_path + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying money, e.g. + ₹50 -> money { currency_maj: "ரூபாய்" integer_part: "ஐம்பது" } + ₹50.50 -> money { currency_maj: "ரூபாய்" integer_part: "ஐம்பது" fractional_part: "ஐம்பது" currency_min: "centiles" } + ₹5 கோடி -> money { currency_maj: "ரூபாய்" integer_part: "ஐந்து கோடி" } + ₹150க்கு -> money { currency_maj: "ரூபாய்" integer_part: "நூற்று ஐம்பது" morphosyntactic_features: "க்கு" } + + The ``centiles`` placeholder is resolved by the verbalizer to the minor currency word, and a + case suffix written on the amount travels as ``morphosyntactic_features`` for the verbalizer + to attach to the currency word. Reads ``data/money/currency.tsv`` (symbol or code -> word) + and ``data/numbers/quantity_words.tsv``. + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="classify", deterministic=deterministic) + + currency_rows = [r for r in load_labels(get_abs_path("data/money/currency.tsv")) if len(r) >= 2] + currency_graph = pynini.string_map([(k, v) for k, v, *_ in currency_rows]).optimize() + rupee_word = dict((k, v) for k, v, *_ in currency_rows)["₹"] + spaced, short, native = quantity_words() + + cardinal_graph = cardinal.final_graph + + optional_graph_negative = pynini.closure( + pynutil.insert("negative: ") + pynini.cross("-", "\"true\"") + insert_space, 0, 1 + ) + currency_major = pynutil.insert("currency_maj: \"") + currency_graph + pynutil.insert("\"") + optional_space = pynini.closure(pynini.accep(" "), 0, 1) + # Every branch that reads the symbol first shares this head, and with it one copy of + # the amount. The weight that ranks a branch therefore rides on its tail: on the head it + # would make the shared prefixes differ and the copies would not merge. + currency_prefix = optional_graph_negative + currency_major + optional_space + insert_space + open_amount = pynutil.insert("integer_part: \"") + close_amount = pynutil.insert("\"") + + range_word = f" {RANGE_WORD} " + range_amount = cardinal_graph + pynini.cross("-", range_word) + cardinal_graph + integer = open_amount + cardinal_graph + pynutil.add_weight(close_amount, -0.1) + integer_range = open_amount + range_amount + pynutil.add_weight(close_amount, -0.05) + + # ₹50.5 means 50 paise: a lone fractional digit is scaled by ten before lookup. + one_digit_padded = pynini.union(NEMO_DIGIT + pynutil.insert("0"), NEMO_TA_DIGIT + pynutil.insert(NEMO_TA_ZERO)) + # .05 is five paise: a leading zero in the minor unit is dropped. + zero_lead = pynini.union(pynutil.delete("0") + NEMO_DIGIT, pynutil.delete(NEMO_TA_ZERO) + NEMO_TA_DIGIT) + two_digits = pynini.union(pynini.difference(NEMO_DIGIT, "0") + NEMO_DIGIT, NEMO_TA_NON_ZERO + NEMO_TA_DIGIT) + fraction_digits = pynini.union(one_digit_padded, zero_lead, two_digits).optimize() + fraction = pynutil.insert("fractional_part: \"") + (fraction_digits @ cardinal_graph) + pynutil.insert("\"") + currency_minor = pynutil.insert("currency_min: \"centiles\"") + minor_amount = optional_space + pynini.cross(".", " ") + fraction + insert_space + currency_minor + + optional_slash_dash = pynini.closure( + pynutil.add_weight(pynini.closure(pynini.accep(" "), 0, 1) + pynutil.delete("/-"), -0.1), 0, 1 + ) + # A trailing .00 minor part is silent (₹1,999.00 -> ...ரூபாய்). + delete_zero_frac = pynutil.delete( + pynini.union(".00", "." + NEMO_TA_ZERO + NEMO_TA_ZERO, ".0", "." + NEMO_TA_ZERO) + ) + + # ₹5 கோடி style: the amount carries a scale word and the currency reads after it. English + # scale words and the shorthands L/cr/K/M/B are spoken natively (₹2 lakh, ₹15L, $50M); + # two scale words may stack (₹1 லட்சம் கோடி). + quantity_word = ( + pynini.accep(" ") + spaced | pynutil.delete(pynini.closure(" ", 0, 1)) + insert_space + short + ) + pynini.closure(pynini.accep(" ") + native, 0, 1) + single_frac_digit = NEMO_ALL_DIGIT @ cardinal_graph + point_word = f" {POINT_WORD} " + amount_with_point = cardinal_graph + pynini.closure( + pynini.cross(".", point_word) + (cardinal.digit_by_digit | single_frac_digit), 0, 1 + ) + # ₹5-10 கோடி reads as a range amount. + amount_with_point |= amount_with_point + pynini.cross("-", range_word) + amount_with_point + + # ₹50, ₹50.50 and ₹1,999.00 all read the same integer amount. + after_integer = ( + optional_slash_dash + | minor_amount + optional_slash_dash + | pynutil.add_weight(delete_zero_frac + optional_slash_dash, -0.1) + ) + after_amount = pynutil.add_weight(quantity_word + close_amount + optional_slash_dash, -0.2) + # ₹150க்கு: a case suffix on the amount is carried as a field and attached to the currency + # word by the verbalizer; it may also follow a scale word. The verbalizer joins the suffix + # onto ரூபாய் with sandhi, which wants the independent-vowel spelling of a glued ல். + written_suffix = pynini.union(*MONEY_SUFFIXES) | pynini.cross("ல்", "இல்") + case_suffix = pynutil.insert(" morphosyntactic_features: \"") + written_suffix + pynutil.insert("\"") + after_amount |= pynutil.add_weight(pynini.closure(quantity_word, 0, 1) + close_amount + case_suffix, -0.1) + + # ₹50.123: three or more minor digits are not paise; read as a decimal amount. + long_fraction = pynini.compose(pynini.closure(NEMO_ALL_DIGIT, 3), cardinal.digit_by_digit) + graph_long_fraction = ( + currency_prefix + + open_amount + + cardinal_graph + + pynini.cross(".", point_word) + + long_fraction + + pynutil.add_weight(close_amount, 0.2) + ) + + # 50/- with no symbol is rupees. + graph_slash_rupee = ( + pynutil.insert(f"currency_maj: \"{rupee_word}\"") + + insert_space + + integer + + optional_space + + pynutil.add_weight(pynutil.delete("/-"), -0.1) + ) + + # ₹.50 reads as paise only (symbol currencies only: Rs./ரூ. own the dot). + symbol_currency = pynini.compose(pynini.union(*CURRENCY_SYMBOLS), currency_graph) + currency_symbol_major = pynutil.insert("currency_maj: \"") + symbol_currency + pynutil.insert("\"") + graph_bare_paise = ( + currency_symbol_major + + optional_space + + insert_space + + pynutil.insert(f"integer_part: \"{cardinal.zero_word}\"") + + pynini.cross(".", " ") + + fraction + + insert_space + + pynutil.add_weight(currency_minor, -0.1) + ) + + # ₹-500: the sign may follow the symbol. + negative_after_currency = ( + currency_major + + optional_space + + pynutil.insert(" negative: ") + + pynini.cross("-", "\"true\"") + + optional_space + + insert_space + + integer + + pynutil.add_weight(optional_slash_dash, 0.1) + ) + + # The amount may also stand before the currency word (50 ரூபாய், 50.50 ரூபாய்). + graph_major_only_suffix = ( + optional_graph_negative + integer + insert_space + optional_space + currency_major + optional_slash_dash + ) + graph_major_and_minor_suffix = ( + optional_graph_negative + + integer + + optional_space + + pynini.cross(".", " ") + + fraction + + optional_space + + insert_space + + currency_minor + + insert_space + + currency_major + + optional_slash_dash + ) + + graph_currencies = ( + currency_prefix + integer + after_integer + | currency_prefix + integer_range + optional_slash_dash + | currency_prefix + open_amount + amount_with_point + after_amount + | graph_long_fraction + | graph_slash_rupee + | graph_bare_paise + | negative_after_currency + | pynutil.add_weight(graph_major_only_suffix | graph_major_and_minor_suffix, 0.5) + ) + + self.fst = self.add_tokens(graph_currencies.optimize()) diff --git a/nemo_text_processing/text_normalization/ta/taggers/ordinal.py b/nemo_text_processing/text_normalization/ta/taggers/ordinal.py new file mode 100644 index 000000000..713da02c7 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/taggers/ordinal.py @@ -0,0 +1,54 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst +from nemo_text_processing.text_normalization.ta.graph_utils import NEMO_ALL_ZERO +from nemo_text_processing.text_normalization.ta.taggers.cardinal import ORDINAL_MARKERS + + +def first_ordinal() -> 'pynini.FstLike': + """ + முதல் is the idiomatic stem for first (1வது -> முதலாவது, 1ஆம் -> முதலாம்). + """ + return pynini.union(pynini.cross("௧", "முதலா"), pynini.cross("1", "முதலா")) + ORDINAL_MARKERS + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying ordinals, e.g. + 5வது -> ordinal { integer: "ஐந்தாவது" } + 1வது -> ordinal { integer: "முதலாவது" } + 28ம் -> ordinal { integer: "இருபத்தெட்டாம்" } + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="ordinal", kind="classify", deterministic=deterministic) + + graph = cardinal.ordinal_graph(cardinal.final_graph) + graph = pynini.union(graph, pynutil.add_weight(first_ordinal(), -0.1)) + # A leading zero on an ordinal is not spoken: 01ஆம் is முதலாம் and 007ஆம் ஏழாம், never + # பூஜ்யம் ஒன்றாம். Stripping outranks the cardinal's leading-zero reading; 0வது stays. + stripped = pynutil.delete(pynini.closure(NEMO_ALL_ZERO, 1)) + graph + graph = pynini.union(graph, pynutil.add_weight(stripped, -1.0)) + + final_graph = pynutil.insert("integer: \"") + graph + pynutil.insert("\"") + self.fst = self.add_tokens(final_graph).optimize() diff --git a/nemo_text_processing/text_normalization/ta/taggers/punctuation.py b/nemo_text_processing/text_normalization/ta/taggers/punctuation.py new file mode 100644 index 000000000..c4fbe2ef9 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/taggers/punctuation.py @@ -0,0 +1,59 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst +from nemo_text_processing.text_normalization.ta.graph_utils import punctuation_code_points + +# ASCII marks that Unicode does not categorise as punctuation. +_ASCII_MARKS = "!#%&'()*+,-./:;<=>?@^_`{|}~\"" + + +class PunctuationFst(GraphFst): + """ + Finite state transducer for classifying punctuation, e.g. + a, -> tokens { name: "a" } tokens { name: "," } + + Markup such as or stays one token instead of splitting into marks. + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="punctuation", kind="classify", deterministic=deterministic) + + self.punct_marks = punctuation_code_points() + list(_ASCII_MARKS) + marks = pynini.union(*[pynini.escape(p) for p in self.punct_marks]) + punct = pynini.closure(marks, 1).optimize() + + tag_body = pynini.closure(NEMO_NOT_SPACE - pynini.union("<", ">"), 1) + emphasis = ( + pynini.accep("<") + + pynini.union(tag_body + pynini.closure(pynini.accep("/"), 0, 1), pynini.accep("/") + tag_body) + + pynini.accep(">") + ).optimize() + # The markup reading takes its inputs out of the run of marks by acceptor difference: a + # priority union over sigma gives the same relation but is far slower to optimize. + rest = pynini.difference(pynini.project(punct, "input"), pynini.project(emphasis, "input")) + punct = pynini.union(emphasis, pynini.compose(rest, punct)).optimize() + + self.graph = punct + # pynini.Fst.project mutates in place, so the word tagger takes this pre-projected copy + # rather than projecting the graph the tokenizer also holds. + self.graph_input = punct.copy().project("input").optimize() + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/nemo_text_processing/text_normalization/ta/taggers/range.py b/nemo_text_processing/text_normalization/ta/taggers/range.py new file mode 100644 index 000000000..c42a4bc7c --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/taggers/range.py @@ -0,0 +1,52 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space +from nemo_text_processing.text_normalization.ta.graph_utils import RANGE_WORD, rank +from nemo_text_processing.text_normalization.ta.taggers.decimal import quantity_words + + +class RangeFst(GraphFst): + """ + Finite state transducer for classifying numeric ranges, e.g. + 10-20 -> tokens { name: "பத்து முதல் இருபது" } + 10-20ல் -> tokens { name: "பத்து முதல் இருபதில்" } + 5-10 லட்சம் -> tokens { name: "ஐந்து முதல் பத்து லட்சம்" } + + A glued case suffix on the upper bound and a scale word after it belong to the range. + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="range", kind="classify", deterministic=deterministic) + + spaced, _, _ = quantity_words() + # A scale word after the upper bound belongs to the range, not to a decimal quantity. + quantity = pynini.accep(" ") + spaced + graph = ( + cardinal.final_graph + + pynutil.delete(pynini.closure(" ", 0, 1) + "-" + pynini.closure(" ", 0, 1)) + + pynutil.insert(f" {RANGE_WORD} ") + + (cardinal.final_graph | cardinal.suffixed_graph + rank(0.1)) + + pynini.closure(quantity, 0, 1) + ) + self.graph = convert_space(graph).optimize() + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/nemo_text_processing/text_normalization/ta/taggers/roman.py b/nemo_text_processing/text_normalization/ta/taggers/roman.py new file mode 100644 index 000000000..9f5efc863 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/taggers/roman.py @@ -0,0 +1,92 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_ALPHA, GraphFst, convert_space, insert_space +from nemo_text_processing.text_normalization.en.utils import load_labels +from nemo_text_processing.text_normalization.ta.graph_utils import NEMO_TA_LETTER +from nemo_text_processing.text_normalization.ta.taggers.ordinal import first_ordinal +from nemo_text_processing.text_normalization.ta.utils import get_abs_path + +MAX_NUMERAL = 39 + +_ROMAN_VALUES = ((10, "X"), (9, "IX"), (5, "V"), (4, "IV"), (1, "I")) + + +def to_roman(n: int) -> str: + """ + The Roman numeral for ``n`` (1-39). + """ + letters = [] + for value, symbol in _ROMAN_VALUES: + while n >= value: + letters.append(symbol) + n -= value + return "".join(letters) + + +def roman_to_digits() -> 'pynini.FstLike': + """ + Transducer from a Roman numeral I-XXXIX to its ASCII digits. + """ + return pynini.string_map([(to_roman(n), str(n)) for n in range(1, MAX_NUMERAL + 1)]).optimize() + + +class RomanFst(GraphFst): + """ + Finite state transducer for classifying Roman numerals in context, e.g. + வகுப்பு XII -> roman { key_cardinal: "வகுப்பு" integer: "பன்னிரண்டு" preserve_order: true } + XII வகுப்பு -> roman { integer: "பன்னிரண்டாம்" key_cardinal: "வகுப்பு" preserve_order: true } + ராஜராஜன்-II -> roman { key_cardinal: "ராஜராஜன்" integer: "இரண்டு" preserve_order: true } + XIIஆம் -> roman { integer: "பன்னிரண்டாம்" } + + A Roman numeral is read as a number only where context says so: a cue word before or after + it, a name joined to it with a hyphen, or a written ordinal marker glued to it. A bare I, + V, X or MIX is a word. The cue words come from ``data/roman/context.tsv`` (cue word -> + the written ordinal marker the numeral takes when it precedes the cue). + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="roman", kind="classify", deterministic=deterministic) + + to_digits = roman_to_digits() + cardinal_words = pynini.compose(to_digits, cardinal.final_graph).optimize() + ordinal_reader = pynini.union( + cardinal.ordinal_graph(cardinal.final_graph), pynutil.add_weight(first_ordinal(), -0.1) + ).optimize() + + def field(name: str, value: 'pynini.FstLike') -> 'pynini.FstLike': + return pynutil.insert(f"{name}: \"") + value + pynutil.insert("\"") + + separator = pynutil.delete(pynini.union(" ", "-")) + insert_space + cued = [] + for cue, marker, *_ in load_labels(get_abs_path("data/roman/context.tsv")): + key = field("key_cardinal", convert_space(pynini.accep(cue))) + cued.append(key + separator + field("integer", cardinal_words)) + ordinal = pynini.compose(to_digits + pynutil.insert(marker), ordinal_reader) + cued.append(field("integer", ordinal) + separator + key) + name = field("key_cardinal", pynini.closure(pynini.union(NEMO_TA_LETTER, NEMO_ALPHA), 1)) + cued.append(name + pynutil.delete("-") + insert_space + field("integer", cardinal_words)) + in_context = pynini.union(*cued) + pynutil.insert(" preserve_order: true") + + glued = pynini.compose(to_digits + pynini.closure(NEMO_TA_LETTER, 1), ordinal_reader) + graph = pynini.union(in_context, field("integer", glued)) + self.fst = self.add_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/ta/taggers/serial.py b/nemo_text_processing/text_normalization/ta/taggers/serial.py new file mode 100644 index 000000000..6af50d789 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/taggers/serial.py @@ -0,0 +1,187 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_UPPER, + TO_UPPER, + GraphFst, + convert_space, + insert_space, +) +from nemo_text_processing.text_normalization.en.utils import load_labels +from nemo_text_processing.text_normalization.ta.graph_utils import NEMO_ALL_DIGIT, NEMO_ALL_NON_ZERO +from nemo_text_processing.text_normalization.ta.utils import get_abs_path + +# Upper-case runs this long or shorter are spelled letter by letter (5G, PAN, KA 01 AB 1234); a +# longer run is an English word or acronym the voice reads on its own (COVID-19). +MAX_SPELLED_RUN = 4 + + +def letter_names() -> 'pynini.FstLike': + """ + One Latin letter, either case, to its spoken name from ``data/serial/letters.tsv``. + """ + table = pynini.string_file(get_abs_path("data/serial/letters.tsv")) + return pynini.union(table, TO_UPPER @ table).optimize() + + +def digit_words() -> 'pynini.FstLike': + """ + One digit in either script to its word from ``data/telephone/number.tsv``. + """ + return pynini.string_file(get_abs_path("data/telephone/number.tsv")).optimize() + + +def unit_letters() -> List[str]: + """ + The single upper-case abbreviations in ``data/measure/unit.tsv`` (C, K, W, A, V ...). + """ + rows = load_labels(get_abs_path("data/measure/unit.tsv")) + return [row[0] for row in rows if len(row) >= 2 and len(row[0]) == 1 and row[0].isupper()] + + +def spelled(letter: 'pynini.FstLike', lower: int, upper: int) -> 'pynini.FstLike': + """ + ``lower`` to ``upper`` letters read one name at a time, space-separated. + """ + return letter + pynini.closure(insert_space + letter, lower - 1, upper - 1) + + +class SerialFst(GraphFst): + """ + Finite state transducer for classifying alphanumeric codes, e.g. + 5G -> tokens { name: "ஐந்து ஜி" } + COVID-19 -> tokens { name: "COVID பத்தொன்பது" } + ABCDE1234F -> tokens { name: "ABCDE ஆயிரத்து இருநூற்று முப்பத்துநான்கு எஃப்" } + KA 01 AB 1234 -> tokens { name: "கே ஏ பூஜ்யம் ஒன்று ஏ பி ஒன்று இரண்டு மூன்று நான்கு" } + + A code mixes Latin letters and digits, glued or joined by ``-`` or ``/``, and holds at least + one upper-case letter; a digit group of one to four digits without a leading zero reads as a + cardinal, any other group digit by digit. A hyphen chain of three or more digit groups that + is neither a date nor a telephone number (1-800-555) is a code too. Dimensions (5x3) and a + digit run glued to a unit abbreviation (170C) are left alone. + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="serial", kind="classify", deterministic=deterministic) + + any_digit = NEMO_ALL_DIGIT + letter = letter_names() + digit = digit_words() + + digit_by_digit = digit + pynini.closure(insert_space + digit) + short_shape = NEMO_ALL_NON_ZERO + pynini.closure(any_digit, 0, 3) + number = pynini.union( + short_shape @ cardinal.final_graph, + pynini.difference(pynini.closure(any_digit, 1), short_shape) @ digit_by_digit, + ) + + letters = pynini.union( + spelled(letter, 1, MAX_SPELLED_RUN), + pynini.difference(pynini.closure(NEMO_ALPHA, 1), pynini.closure(NEMO_UPPER, 1, MAX_SPELLED_RUN)), + ) + + # Letters and digits alternate inside a glued piece, so a digit run is never split. + glued = pynini.union( + letters + + pynini.closure(insert_space + number + insert_space + letters) + + pynini.closure(insert_space + number, 0, 1), + number + + pynini.closure(insert_space + letters + insert_space + number) + + pynini.closure(insert_space + letters, 0, 1), + ) + separator = pynutil.delete(pynini.union("-", "/")) + insert_space + # Optimized before it is composed with the shape filters below: the alternation of + # letters and numbers leaves tens of thousands of states that determinize to a few + # thousand, and composing the unoptimized form explodes. + code = (glued + pynini.closure(separator + glued)).optimize() + + # An Indian vehicle plate: state, district, series and a four-digit number, with or + # without spaces or hyphens (KA 01 AB 1234, TN-09-AB-1234, MH12DE1433). Its groups always + # read digit by digit, so the shape is taken out of the general code reading below, + # which would otherwise read a four-digit group as a cardinal. + one_or_two = pynini.closure(any_digit, 1, 2) + gap = pynini.closure(pynini.union(" ", "-"), 0, 1) + plate_shape = ( + NEMO_UPPER**2 + + gap + + one_or_two + + gap + + pynini.closure(NEMO_UPPER, 1, 3) + + gap + + pynini.closure(any_digit, 4, 4) + ).optimize() + + code_chars = pynini.closure(pynini.union(NEMO_ALPHA, any_digit, "-", "/")) + digits = pynini.closure(any_digit, 1) + # A digit run closing in a unit abbreviation belongs to the measure class, which reads it + # only when a space separates the two (170 C); glued, it is left as written. + measure_shape = digits + pynini.union(*unit_letters()) + shape = pynini.difference( + pynini.intersect(code_chars + any_digit + code_chars, code_chars + NEMO_UPPER + code_chars), + pynini.union(digits + pynini.union("x", "X") + digits, plate_shape, measure_shape), + ).optimize() + mixed = (shape @ code).optimize() + + # 1-800-555, 1-2-3: three or more groups of up to four digits, fewer than ten digits in + # all (a telephone number has ten) and not a date. + group = pynini.closure(any_digit, 1, 4) + date = pynini.union( + one_or_two + "-" + one_or_two + "-" + pynini.closure(any_digit, 2, 4), + pynini.closure(any_digit, 4, 4) + "-" + one_or_two + "-" + one_or_two, + ) + at_most_nine = pynini.closure(pynini.closure("-") + any_digit, 0, 9) + pynini.closure("-") + chain_shape = pynini.difference( + pynini.intersect(group + pynini.closure("-" + group, 2), at_most_nine), date + ).optimize() + chain = ( + chain_shape @ (number + pynini.closure(pynutil.delete("-") + insert_space + number, 2)).optimize() + ).optimize() + + plate_sep = pynini.closure(pynutil.delete(pynini.union(" ", "-")), 0, 1) + insert_space + plate = ( + plate_shape + @ ( + spelled(letter, 2, 2) + + plate_sep + + (one_or_two @ digit_by_digit) + + plate_sep + + spelled(letter, 1, 3) + + plate_sep + + (pynini.closure(any_digit, 4, 4) @ digit_by_digit) + ).optimize() + ).optimize() + + # A short upper-case cue before a code is spelled with it: PNR 4X7K9M, PAN ABCDE1234F. + cued = spelled(letter, 2, MAX_SPELLED_RUN) + pynutil.delete(" ") + insert_space + mixed + + graph = pynini.union( + pynutil.add_weight(mixed, 0.2), + pynutil.add_weight(chain, 0.3), + pynutil.add_weight(plate, 0.1), + pynutil.add_weight(cued, 0.1), + ) + self.graph = convert_space(graph).optimize() + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/nemo_text_processing/text_normalization/ta/taggers/telephone.py b/nemo_text_processing/text_normalization/ta/taggers/telephone.py new file mode 100644 index 000000000..81c7d33f7 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/taggers/telephone.py @@ -0,0 +1,158 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_space, insert_space +from nemo_text_processing.text_normalization.ta.graph_utils import NEMO_ALL_DIGIT, NEMO_ALL_ZERO, PLUS_WORD +from nemo_text_processing.text_normalization.ta.utils import get_abs_path + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for classifying telephone numbers, e.g. + 9943206870 -> telephone { number_part: "ஒன்பது ஒன்பது நான்கு மூன்று இரண்டு பூஜ்யம் ஆறு எட்டு ஏழு பூஜ்யம்" } + +91 9876543210 -> telephone { country_code: "பிளஸ் ஒன்பது ஒன்று" number_part: "..." } + 044-28230000 -> telephone { number_part: "பூஜ்யம் நான்கு நான்கு இரண்டு எட்டு ..." } + +91 -> telephone { country_code: "பிளஸ் தொண்ணூற்றொன்று" } + + Reads ``data/telephone/number.tsv`` (digit in either script -> word). Indian mobile, + landline and toll-free shapes are read digit by digit; a case suffix on the number lands on + the last digit word. + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="telephone", kind="classify", deterministic=deterministic) + + single_digit_to_word = pynini.string_file(get_abs_path("data/telephone/number.tsv")).optimize() + mobile_first_digit = pynini.union(*"6789", *"௬௭௮௯") + zero_digit = NEMO_ALL_ZERO + one_word = pynini.union("1", "௧") @ single_digit_to_word + + digit_word = single_digit_to_word + insert_space + last_digit_word = single_digit_to_word + delete_sep = pynutil.delete(pynini.union("-", " ")) + optional_sep = pynini.closure(delete_sep, 0, 1) + + # A case suffix on the number lands on the last digit word (9876543210க்கு -> ...பூஜ்யத்துக்கு). + last_digit_suffixed = cardinal.attach_case_suffix(last_digit_word) + + def shapes(last: 'pynini.FstLike'): + # 10-digit mobile starting 6-9, optionally after a trunk 0 (09876543210); a 5-5 split + # with space or dash is common. + trunk = pynini.closure((zero_digit @ single_digit_to_word) + insert_space, 0, 1) + mobile = ( + trunk + + (mobile_first_digit @ single_digit_to_word) + + insert_space + + pynini.closure(digit_word, 3, 3) + + digit_word + + optional_sep + + pynini.closure(digit_word, 4, 4) + + last + ) + + # Landline: STD code starting 0 (2-4 digits, optionally in parentheses), a dash or + # space, then a 6-8 digit subscriber number optionally split once. + std_digits = (zero_digit @ single_digit_to_word) + insert_space + pynini.closure(digit_word, 1, 3) + std_code = std_digits | (pynutil.delete("(") + std_digits + pynutil.delete(")")) + # A hyphen split inside the subscriber is only the 4-4 shape (2823-0000), so a date + # like 01-04-2024 never reads as a landline. + subscriber = ( + pynini.closure(digit_word, 2, 4) + + pynini.closure(pynutil.delete(" "), 0, 1) + + pynini.closure(digit_word, 2, 3) + + last + ) + subscriber |= ( + pynini.closure(digit_word, 4, 4) + pynutil.delete("-") + pynini.closure(digit_word, 3, 3) + last + ) + landline = std_code + optional_sep + subscriber + + # Toll-free: 1800-XXX-XXXX / 1-800-XXX-XXXX. + toll_free = ( + one_word + + insert_space + + optional_sep + + pynini.closure(digit_word, 3, 3) + + delete_sep + + pynini.closure(digit_word, 3, 3) + + delete_sep + + pynini.closure(digit_word, 3, 3) + + last + ) + # Toll-free 1800-11-4000 / 1800 11 4000: 1800 + 2-3 digits + 3-4 digits. + toll_free |= ( + one_word + + insert_space + + pynini.closure(digit_word, 3, 3) + + delete_sep + + pynini.closure(digit_word, 2, 3) + + delete_sep + + pynini.closure(digit_word, 2, 3) + + last + ) + + # After a country code the STD code drops its leading zero: +91-44-28230000, + # optionally in parentheses: +91 (44) 2823 0000. + std_digits_no_zero = pynini.closure(digit_word, 2, 4) + std_no_zero = ( + (std_digits_no_zero | pynutil.delete("(") + std_digits_no_zero + pynutil.delete(")")) + + delete_sep + + subscriber + ) + return pynini.union(mobile, landline, toll_free), std_no_zero + + plain, cc_landline = shapes(last_digit_word) + suffixed, cc_landline_suffixed = shapes(last_digit_suffixed) + + country_code = ( + pynutil.insert("country_code: \"") + + pynini.cross("+", PLUS_WORD) + + insert_space + + pynini.closure(digit_word, 0, 2) + + last_digit_word + + pynutil.insert("\" ") + + pynini.closure(delete_space | pynutil.delete("-"), 0, 1) + ) + + def number_part(inner: 'pynini.FstLike') -> 'pynini.FstLike': + return pynutil.insert("number_part: \"") + inner + pynutil.insert("\"") + + graph = pynini.union( + pynutil.add_weight(country_code + number_part(plain | cc_landline), 0.1), + pynutil.add_weight(number_part(plain), 0.1), + pynutil.add_weight(country_code + number_part(suffixed | cc_landline_suffixed), 0.2), + pynutil.add_weight(number_part(suffixed), 0.2), + ) + + # A + before 11-13 glued digits with no separator (+919876543210) reads digit by digit; + # a shorter run (+91, +5) is a signed cardinal. + standalone_cc = ( + pynutil.insert("country_code: \"") + + pynini.cross("+", PLUS_WORD) + + insert_space + + pynini.compose(pynini.closure(NEMO_ALL_DIGIT, 11, 13), cardinal.digit_by_digit) + + pynutil.insert("\"") + ) + graph |= pynutil.add_weight(standalone_cc, 0.3) + + self.final = graph.optimize() + self.fst = self.add_tokens(self.final) diff --git a/nemo_text_processing/text_normalization/ta/taggers/time.py b/nemo_text_processing/text_normalization/ta/taggers/time.py new file mode 100644 index 000000000..216008530 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/taggers/time.py @@ -0,0 +1,171 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_CHAR, + NEMO_DIGIT, + NEMO_SPACE, + GraphFst, + insert_space, +) +from nemo_text_processing.text_normalization.ta.graph_utils import ( + AM_WORD, + DAY_PART_ABBREVIATIONS, + DAY_PARTS, + NEMO_TA_DIGIT, + NEMO_TA_NON_ZERO, + NEMO_TA_ZERO, + PM_WORD, + TO_TA_DIGITS, +) +from nemo_text_processing.text_normalization.ta.utils import get_abs_path + +# The verbalizer speaks மணி itself, so a written மணி/மணிக்கு and a bare case suffix on the +# digits (3:30க்கு, 10:30 இல்) are absorbed rather than carried as a field. +HOUR_NOUNS = ("மணிக்கு", "மணி") +ABSORBED_SUFFIXES = ("க்கு", "ல்", "இல்") +# Hour 24 is only meaningful as 24:00. +EXACT_ONLY_HOURS = ("இருபத்துநான்கு",) + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying time, e.g. + 12:30:30 -> time { hours: "பன்னிரண்டு" minutes: "முப்பது" seconds: "முப்பது" } + 1:40 -> time { hours: "ஒரு" minutes: "நாற்பது" } + 10:00க்கு -> time { hours: "பத்து" } + 10:30 AM -> time { hours: "பத்து" minutes: "முப்பது" meridiem: "முற்பகல்" } + காலை 10.30 -> time { hours: "பத்து" minutes: "முப்பது" meridiem: "காலை" } + + Reads ``data/time/hours.tsv``, ``data/time/minutes.tsv`` and ``data/time/seconds.tsv`` (Tamil + digits to words; hours 0-24, minutes and seconds 01-59). A press-style dotted time (10.30) + is only a time with a clock context: a trailing hour noun, or a day-part word before or + after it. + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="time", kind="classify", deterministic=deterministic) + + hours_graph = pynini.string_file(get_abs_path("data/time/hours.tsv")) + minutes_graph = pynini.string_file(get_abs_path("data/time/minutes.tsv")) + seconds_graph = pynini.string_file(get_abs_path("data/time/seconds.tsv")) + + delete_colon = pynutil.delete(":") + + delete_leading_zero_native = ( + (NEMO_TA_NON_ZERO + NEMO_TA_DIGIT) | (pynutil.delete(NEMO_TA_ZERO) + NEMO_TA_DIGIT) | NEMO_TA_DIGIT + ).optimize() + delete_leading_zero_ascii = ( + (pynini.difference(NEMO_DIGIT, "0") + NEMO_DIGIT) | (pynutil.delete("0") + NEMO_DIGIT) | NEMO_DIGIT + ).optimize() + + hour_input = ( + pynini.compose(delete_leading_zero_native, hours_graph) + | pynini.compose(delete_leading_zero_ascii, TO_TA_DIGITS @ hours_graph) + ).optimize() + minute_input = ( + pynini.compose(pynini.closure(NEMO_TA_DIGIT, 1), minutes_graph) + | pynini.compose(pynini.closure(NEMO_DIGIT, 1), TO_TA_DIGITS @ minutes_graph) + ).optimize() + second_input = ( + pynini.compose(pynini.closure(NEMO_TA_DIGIT, 1), seconds_graph) + | pynini.compose(pynini.closure(NEMO_DIGIT, 1), TO_TA_DIGITS @ seconds_graph) + ).optimize() + + hour_any = hour_input + hour_input = hour_input @ pynini.difference(pynini.closure(NEMO_CHAR), pynini.union(*EXACT_ONLY_HOURS)) + self.hours = pynutil.insert("hours: \"") + hour_input + pynutil.insert("\" ") + hours_any = pynutil.insert("hours: \"") + hour_any + pynutil.insert("\" ") + self.minutes = pynutil.insert("minutes: \"") + minute_input + pynutil.insert("\" ") + self.seconds = pynutil.insert("seconds: \"") + second_input + pynutil.insert("\" ") + + # A trailing written hour noun, or a case suffix the verbalizer does not attach, is + # consumed silently, glued to the digits or spaced (10:30க்கு, 10:30 இல், 7:00 மணி). + space = pynini.closure(NEMO_SPACE, 0, 1) + hour_word_tail = space + pynutil.delete(pynini.union(*HOUR_NOUNS)) + absorbed_tail = hour_word_tail | space + pynutil.delete(pynini.union(*ABSORBED_SUFFIXES)) + optional_tail = pynini.closure(absorbed_tail, 0, 1).optimize() + + graph_hms = ( + self.hours + + delete_colon + + insert_space + + self.minutes + + delete_colon + + insert_space + + self.seconds + + optional_tail + ) + double_zero = pynini.union("00", NEMO_TA_ZERO + NEMO_TA_ZERO) + delete_zero_seconds = pynini.closure(pynutil.delete(":" + double_zero), 0, 1) + graph_hm = self.hours + delete_colon + insert_space + self.minutes + delete_zero_seconds + optional_tail + delete_zero_minutes = delete_colon + pynutil.delete(double_zero) + graph_h = hours_any + delete_zero_minutes + delete_zero_seconds + optional_tail + # 10:00:30 keeps only the seconds. + graph_h_s = self.hours + delete_zero_minutes + delete_colon + insert_space + self.seconds + optional_tail + + # Trailing AM/PM becomes a meridiem field the verbalizer fronts. + meridiem_word = pynini.cross(pynini.union("AM", "am", "A.M.", "a.m."), AM_WORD) | pynini.cross( + pynini.union("PM", "pm", "P.M.", "p.m."), PM_WORD + ) + required_meridiem = ( + pynutil.delete(pynini.closure(" ", 0, 1)) + + pynutil.insert("meridiem: \"") + + meridiem_word + + pynutil.insert("\" ") + ) + meridiem = pynini.closure(required_meridiem, 0, 1) + + final_graph = ( + graph_hms + | pynutil.add_weight(graph_hm, 1.0) + | pynutil.add_weight(graph_h_s, 1.0) + | pynutil.add_weight(graph_h, 0.8) + ) + meridiem + + # A bare hour with AM/PM is a clock time: 7 AM, 7pm. + final_graph |= pynutil.add_weight(self.hours + required_meridiem, 0.9) + + # Press-style dotted time (10.30) is only a time with a clock context. + two_digit_minutes = pynini.compose( + pynini.union(NEMO_TA_DIGIT + NEMO_TA_DIGIT, NEMO_DIGIT + NEMO_DIGIT), minute_input + ) + dotted = ( + self.hours + + pynutil.delete(".") + + insert_space + + pynutil.insert("minutes: \"") + + two_digit_minutes + + pynutil.insert("\" ") + ) + # 6.00 reads as the bare hour. + dotted |= self.hours + pynutil.delete("." + double_zero) + dotted_tail = pynini.closure(hour_word_tail, 0, 1) + contexts = [(w, w) for w in DAY_PARTS] + list(DAY_PART_ABBREVIATIONS.items()) + dotted_graphs = [dotted + hour_word_tail, dotted + dotted_tail + required_meridiem] + for written, spoken in contexts: + meridiem_field = pynutil.insert(f"meridiem: \"{spoken}\" ") + dotted_graphs.append(pynutil.delete(written) + pynutil.delete(" ") + dotted + dotted_tail + meridiem_field) + dotted_graphs.append(dotted + space + pynutil.delete(written) + meridiem_field) + # A cued dotted time must outrank a measure reading of the same span (10.30 மணி). + final_graph |= pynutil.add_weight(pynini.union(*dotted_graphs), -2.5) + + self.fst = self.add_tokens(final_graph).optimize() diff --git a/nemo_text_processing/text_normalization/ta/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/ta/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..91ae6cda8 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/taggers/tokenize_and_classify.py @@ -0,0 +1,316 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_CHAR, + NEMO_NOT_SPACE, + NEMO_SIGMA, + NEMO_SPACE, + NEMO_WHITE_SPACE, + GraphFst, + delete_extra_space, + delete_space, +) +from nemo_text_processing.text_normalization.en.utils import load_labels +from nemo_text_processing.text_normalization.ta.graph_utils import ( + CURRENCY_SYMBOLS, + MINUS_WORD, + NEMO_ALL_DIGIT, + NEMO_TA_LETTER, + RANGE_WORD, + generator_main, +) +from nemo_text_processing.text_normalization.ta.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.ta.taggers.date import DateFst +from nemo_text_processing.text_normalization.ta.taggers.decimal import DecimalFst +from nemo_text_processing.text_normalization.ta.taggers.electronic import ElectronicFst +from nemo_text_processing.text_normalization.ta.taggers.fraction import FractionFst +from nemo_text_processing.text_normalization.ta.taggers.measure import MeasureFst +from nemo_text_processing.text_normalization.ta.taggers.money import MoneyFst +from nemo_text_processing.text_normalization.ta.taggers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.ta.taggers.punctuation import PunctuationFst +from nemo_text_processing.text_normalization.ta.taggers.range import RangeFst +from nemo_text_processing.text_normalization.ta.taggers.roman import RomanFst +from nemo_text_processing.text_normalization.ta.taggers.serial import SerialFst +from nemo_text_processing.text_normalization.ta.taggers.telephone import TelephoneFst +from nemo_text_processing.text_normalization.ta.taggers.time import TimeFst +from nemo_text_processing.text_normalization.ta.taggers.whitelist import WhiteListFst +from nemo_text_processing.text_normalization.ta.taggers.word import WordFst +from nemo_text_processing.text_normalization.ta.utils import get_abs_path + +# Symbols the whitelist speaks; each is split into its own token wherever it stands. +SPOKEN_SYMBOLS = "#*&^%|~" +# Spoken < and > between two digits, and + between two digits. +LESS_THAN = "விடக் குறைவு" +GREATER_THAN = "விட அதிகம்" +INFIX_PLUS = "கூட்டல்" + +# Zero-width and directional format characters with no linguistic role (ZWJ/ZWNJ are kept). +_FORMAT_CHARS = "​⁠­‎‏؜‪‫‬‭‮⁦⁧⁨⁩" +# Dash lookalikes read like an ASCII hyphen; exotic spaces like a space. +_DASHES = "‐‑‒–—―" +_SPACES = "   " + + +def _pre_process(known_suffixes: 'pynini.FstLike') -> 'pynini.FstLike': + """ + Spacing rewrites composed in front of the sentence grammar: they split symbols and glued + words off digits and decide what a hyphen or a minus sign means, so that no class grammar + has to embed those shapes itself. + + Args: + known_suffixes: every case or ordinal suffix that may stay glued to a digit; any other + Tamil word glued to a digit is split off + """ + letter = NEMO_TA_LETTER + any_digit = NEMO_ALL_DIGIT + spaces = pynini.closure(" ") + edge = pynini.union("[BOS]", " ") + + # A zero-width space or word joiner between two digits is a boundary, not glue; the other + # format characters are dropped and dash lookalikes read like a hyphen. + joiner = pynini.union("​", "⁠") + split_joiner = pynini.cdrewrite(pynini.cross(pynini.closure(joiner, 1), " "), any_digit, any_digit, NEMO_SIGMA) + clean = ( + split_joiner + @ pynini.cdrewrite(pynutil.delete(pynini.union(*_FORMAT_CHARS)), "", "", NEMO_SIGMA) + @ pynini.cdrewrite(pynini.cross(pynini.union(*_DASHES), "-"), "", "", NEMO_SIGMA) + @ pynini.cdrewrite(pynini.cross(pynini.union(*_SPACES), " "), "", "", NEMO_SIGMA) + ) + + # U+2212 MINUS SIGN between digits is subtraction; elsewhere it is a plain minus. + minus = f" {MINUS_WORD} " + true_minus = pynini.cdrewrite( + pynini.cross("−", minus), any_digit + spaces, spaces + any_digit, NEMO_SIGMA + ) @ pynini.cdrewrite(pynini.cross("−", "-"), "", "", NEMO_SIGMA) + + # %க்கு reads as a dative percent word; other case suffixes on % likewise + # (data/whitelist/percent_suffix.tsv). Any other Tamil word glued to % is a separate word. + trailing_punct = pynini.union(*[pynini.escape(c) for c in "()\"'{}[].,!?%"]) + percent_rows = [row for row in load_labels(get_abs_path("data/whitelist/percent_suffix.tsv")) if len(row) >= 2] + percent_suffix = pynini.cdrewrite( + pynini.union(*[pynini.cross(written, " " + spoken) for written, spoken, *_ in percent_rows]), + any_digit, + pynini.union(" ", "[EOS]", trailing_punct), + NEMO_SIGMA, + ) + percent_word = pynini.cdrewrite(pynutil.insert(" "), "%", letter, NEMO_SIGMA) + + # A hyphen inside an equation is a minus, not a range: 5-3=2, 10 - 5 = 5. + subtraction_minus = pynini.cdrewrite( + pynini.cross("-", minus), + any_digit + spaces, + spaces + pynini.closure(pynini.union(any_digit, "-", " "), 1) + "=", + NEMO_SIGMA, + ) + # <, > and + are markup or a sign except between two digits, where they are operators. + comparison = pynini.cdrewrite( + pynini.union( + pynini.cross("<", f" {LESS_THAN} "), + pynini.cross(">", f" {GREATER_THAN} "), + pynini.cross("+", f" {INFIX_PLUS} "), + ), + any_digit + spaces, + spaces + any_digit, + NEMO_SIGMA, + ) + + # Split the symbols the whitelist speaks off digits and words: 5×3=15 -> 5 × 3 = 15, 5% -> 5 %. + operator = pynini.union("×", "÷", "%", "=") + space_after_digit = pynini.cdrewrite(pynutil.insert(" "), any_digit, operator, NEMO_SIGMA) + space_before_digit = pynini.cdrewrite(pynutil.insert(" "), operator, any_digit, NEMO_SIGMA) + spoken_symbol = pynini.union(*SPOKEN_SYMBOLS) + split_symbol = pynini.cdrewrite(pynutil.insert(" "), NEMO_NOT_SPACE, spoken_symbol, NEMO_SIGMA) @ pynini.cdrewrite( + pynutil.insert(" "), spoken_symbol, NEMO_NOT_SPACE, NEMO_SIGMA + ) + # @ and _ are spoken too, but stay glued between ASCII letters or digits so an e-mail + # address or an identifier (user@example.com, a_b) passes through whole. + edge_symbol = pynini.union("@", "_") + not_identifier = pynini.difference(NEMO_NOT_SPACE, pynini.union(NEMO_ALPHA, edge_symbol)) + split_edge_symbol = pynini.cdrewrite( + pynutil.insert(" "), not_identifier, edge_symbol, NEMO_SIGMA + ) @ pynini.cdrewrite(pynutil.insert(" "), edge_symbol, not_identifier, NEMO_SIGMA) + # A hyphen between two amounts is a range: ₹5 - ₹10, ₹5-₹10. + currency = pynini.union(*CURRENCY_SYMBOLS) + money_range = pynini.cdrewrite( + pynini.cross("-", f" {RANGE_WORD} "), any_digit + spaces, spaces + currency, NEMO_SIGMA + ) + + # A hyphen between a digit and a case/ordinal suffix belongs to the suffix (3-வது, 2024-ல், + # 100-க்கு); any other hyphen joining a digit to a Tamil word is a separator + # (5-அவர்கள் -> 5 அவர்கள், 15-ஜூன்-2024 -> 15 ஜூன் 2024). + drop_ordinal_hyphen = pynini.cdrewrite(pynutil.delete("-"), any_digit, known_suffixes, NEMO_SIGMA) + joiner_hyphen_to_space = pynini.cdrewrite( + pynini.cross("-", " "), any_digit, letter, NEMO_SIGMA + ) @ pynini.cdrewrite(pynini.cross("-", " "), letter, any_digit, NEMO_SIGMA) + # Case and ordinal suffixes may stay glued to a digit; anything else glued to a digit is a + # separate word (5கிலோ -> 5 கிலோ). + word = pynini.closure(letter, 1) + unknown_word = pynini.difference(word, known_suffixes).optimize() + boundary = pynini.union(" ", "[EOS]", pynini.difference(NEMO_CHAR, letter)) + split_digit_word = pynini.cdrewrite(pynutil.insert(" "), any_digit, unknown_word + boundary, NEMO_SIGMA) + # A Tamil letter glued to a digit on its left is a separate word too (ஜி20, 5.மணி). + letter_digit = pynini.cdrewrite(pynutil.insert(" "), letter, any_digit, NEMO_SIGMA) + dot_letter = pynini.cdrewrite(pynutil.insert(" "), any_digit + ".", letter, NEMO_SIGMA) + + return ( + clean + @ true_minus + @ drop_ordinal_hyphen + @ percent_suffix + @ percent_word + @ subtraction_minus + @ comparison + @ space_after_digit + @ space_before_digit + @ split_symbol + @ split_edge_symbol + @ money_range + @ joiner_hyphen_to_space + @ letter_digit + @ dot_letter + @ split_digit_word + ).optimize() + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence including punctuation. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + input_case: str, + deterministic: bool = True, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + whitelist_file = os.path.basename(whitelist) if whitelist else "" + far_file = os.path.join( + cache_dir, + f"ta_tn_{deterministic}_deterministic_{input_case}_{whitelist_file}_tokenize.far", + ) + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logging.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logging.info(f"Creating ClassifyFst grammars.") + + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + + decimal = DecimalFst(cardinal=cardinal, deterministic=deterministic) + decimal_graph = decimal.fst + + fraction_graph = FractionFst(cardinal=cardinal, deterministic=deterministic).fst + date_graph = DateFst(cardinal=cardinal, deterministic=deterministic).fst + time_graph = TimeFst(deterministic=deterministic).fst + ordinal_graph = OrdinalFst(cardinal=cardinal, deterministic=deterministic).fst + measure_graph = MeasureFst(cardinal=cardinal, decimal=decimal, deterministic=deterministic).fst + money_graph = MoneyFst(cardinal=cardinal, deterministic=deterministic).fst + telephone_graph = TelephoneFst(cardinal=cardinal, deterministic=deterministic).fst + range_graph = RangeFst(cardinal=cardinal, deterministic=deterministic).fst + roman_graph = RomanFst(cardinal=cardinal, deterministic=deterministic).fst + serial_graph = SerialFst(cardinal=cardinal, deterministic=deterministic).fst + electronic_graph = ElectronicFst(deterministic=deterministic).fst + + whitelist_graph = WhiteListFst( + input_case=input_case, deterministic=deterministic, input_file=whitelist + ).fst + + punctuation = PunctuationFst(deterministic=deterministic) + punct_graph = punctuation.fst + + word = WordFst(punctuation=punctuation, deterministic=deterministic) + word_graph = word.fst + + # The number classes are ranked so that a span every one of them can read goes to + # the most specific: a telephone shape before a cardinal, a date before a range, a + # range (10-20) before two cardinals and a hyphen. Codes and addresses only exist + # where no number class reads the span. + classify = ( + pynutil.add_weight(whitelist_graph, 1.01) + | pynutil.add_weight(telephone_graph, 0.5) + | pynutil.add_weight(measure_graph, 1.03) + | pynutil.add_weight(date_graph, 1.04) + | pynutil.add_weight(time_graph, 1.05) + | pynutil.add_weight(fraction_graph, 1.06) + | pynutil.add_weight(decimal_graph, 1.08) + | pynutil.add_weight(range_graph, 1.09) + | pynutil.add_weight(cardinal_graph, 1.1) + | pynutil.add_weight(money_graph, 1.1) + | pynutil.add_weight(ordinal_graph, 1.1) + | pynutil.add_weight(roman_graph, 1.1) + | pynutil.add_weight(electronic_graph, 1.1) + | pynutil.add_weight(serial_graph, 1.2) + ) + + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=2.1) + pynutil.insert(" }") + punct = pynini.closure( + pynini.union( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space), + (pynutil.insert(NEMO_SPACE) + punct), + ), + 1, + ) + + classify = pynini.union(classify, pynutil.add_weight(word_graph, 100)) + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(NEMO_SPACE)) + + token + + pynini.closure(pynutil.insert(NEMO_SPACE) + punct) + ) + + graph = token_plus_punct + pynini.closure( + pynini.union( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space), + (pynutil.insert(NEMO_SPACE) + punct + pynutil.insert(NEMO_SPACE)), + ) + + token_plus_punct + ) + + graph = delete_space + graph + delete_space + graph = pynini.union(graph, punct).optimize() + + # The spacing rewrites run over the text first, so every grammar above reads + # clean digit runs and spaced symbols. + self.fst = pynini.compose(_pre_process(cardinal.known_suffixes), graph).optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) + logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/ta/taggers/whitelist.py b/nemo_text_processing/text_normalization/ta/taggers/whitelist.py new file mode 100644 index 000000000..d198295eb --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/taggers/whitelist.py @@ -0,0 +1,65 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import ( + INPUT_LOWER_CASED, + NEMO_SIGMA, + GraphFst, + convert_space, +) +from nemo_text_processing.text_normalization.en.utils import load_labels +from nemo_text_processing.text_normalization.ta.utils import get_abs_path, table_fst + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for classifying whitelist entries, e.g. + டாக். -> tokens { name: "டாக்டர்" } + % -> tokens { name: "சதவீதம்" } + + Reads ``data/whitelist/abbreviations.tsv`` and ``data/whitelist/symbol.tsv``; a symbol + is spoken wherever it stands, except the slash, which the fraction and measure classes own. + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + input_file: path to a file with whitelist replacements, added to the default tables + """ + + def __init__(self, input_case: str, deterministic: bool = True, input_file: str = None): + super().__init__(name="whitelist", kind="classify", deterministic=deterministic) + + def _get_whitelist_graph(file: str) -> 'pynini.FstLike': + whitelist = [row for row in load_labels(file) if len(row) >= 2] + if input_case == INPUT_LOWER_CASED: + whitelist = [[x.lower(), y] for x, y, *_ in whitelist] + else: + whitelist = [[x, y] for x, y, *_ in whitelist] + return pynini.string_map(whitelist) + + graph = _get_whitelist_graph(get_abs_path("data/whitelist/abbreviations.tsv")) + graph |= pynini.compose( + pynini.difference(NEMO_SIGMA, pynini.accep("/")).optimize(), + table_fst(get_abs_path("data/whitelist/symbol.tsv")), + ).optimize() + + if input_file: + graph |= _get_whitelist_graph(input_file) + + self.graph = convert_space(graph).optimize() + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/nemo_text_processing/text_normalization/ta/taggers/word.py b/nemo_text_processing/text_normalization/ta/taggers/word.py new file mode 100644 index 000000000..388499f4d --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/taggers/word.py @@ -0,0 +1,64 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import ( + MIN_NEG_WEIGHT, + NEMO_ALPHA, + NEMO_NOT_SPACE, + GraphFst, + convert_space, +) +from nemo_text_processing.text_normalization.ta.graph_utils import NEMO_TA_BLOCK +from nemo_text_processing.text_normalization.ta.taggers.punctuation import PunctuationFst + +# Symbols a semiotic class owns, so the word class must not swallow them. +_CLASS_SYMBOLS = ["$", "€", "₩", "£", "¥", "#", "%"] + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying words, e.g. + தமிழ் -> tokens { name: "தமிழ்" } + + A run of Tamil characters is preferred over the fallback that accepts any non-space + characters, and a URL stays one token instead of splitting into punctuation marks. + + Args: + punctuation: PunctuationFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, punctuation: PunctuationFst, deterministic: bool = True): + super().__init__(name="word", kind="classify", deterministic=deterministic) + + punct = punctuation.graph_input + default_graph = pynini.closure(pynini.difference(NEMO_NOT_SPACE, punct), 1) + symbols_to_exclude = (pynini.union(*_CLASS_SYMBOLS) | punct).optimize() + + graph = pynini.closure(pynini.difference(NEMO_TA_BLOCK, symbols_to_exclude), 1) + graph = pynutil.add_weight(graph, MIN_NEG_WEIGHT) | default_graph + + url_body = pynini.closure(pynini.difference(NEMO_NOT_SPACE, pynini.accep("\"")), 1) + url = (pynini.closure(NEMO_ALPHA, 1) + "://" + url_body) | ("www." + url_body) + graph = pynutil.add_weight(url, MIN_NEG_WEIGHT) | graph + + # No space is introduced around punctuation inside a word. + graph = pynini.closure(graph + pynini.closure(punct + graph, 0, 1)) + + self.graph = convert_space(graph) + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/nemo_text_processing/text_normalization/ta/utils.py b/nemo_text_processing/text_normalization/ta/utils.py new file mode 100644 index 000000000..4263b3887 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/utils.py @@ -0,0 +1,48 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini + +from nemo_text_processing.text_normalization.en.utils import load_labels + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +def table_fst(abs_path: str, key: int = 0, value: int = 1) -> 'pynini.FstLike': + """ + Compiles two columns of a TSV table into an optimized string map. + + Unlike ``pynini.string_file`` this tolerates a third column that is not a weight, so it is + the loader for the tables that carry a kind or a note in their last column. + + Args: + abs_path: absolute path of the table + key: index of the input column + value: index of the output column + """ + width = max(key, value) + 1 + rows = [row for row in load_labels(abs_path) if len(row) >= width] + return pynini.string_map([(row[key], row[value]) for row in rows]).optimize() diff --git a/nemo_text_processing/text_normalization/ta/verbalizers/__init__.py b/nemo_text_processing/text_normalization/ta/verbalizers/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ta/verbalizers/cardinal.py b/nemo_text_processing/text_normalization/ta/verbalizers/cardinal.py new file mode 100644 index 000000000..a0f03e8f5 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/verbalizers/cardinal.py @@ -0,0 +1,42 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space +from nemo_text_processing.text_normalization.ta.graph_utils import MINUS_WORD, PLUS_WORD + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinals, e.g. + cardinal { negative: "true" integer: "இருபத்துமூன்று" } -> மைனஸ் இருபத்துமூன்று + cardinal { positive: "true" integer: "ஐந்து" } -> பிளஸ் ஐந்து + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + + sign = pynini.cross("negative: \"true\"", f"{MINUS_WORD} ") | pynini.cross( + "positive: \"true\"", f"{PLUS_WORD} " + ) + self.optional_sign = pynini.closure(sign + delete_space, 0, 1) + self.integer = delete_space + pynutil.delete("\"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + self.numbers = self.optional_sign + pynutil.delete("integer:") + self.integer + self.fst = self.delete_tokens(self.numbers).optimize() diff --git a/nemo_text_processing/text_normalization/ta/verbalizers/date.py b/nemo_text_processing/text_normalization/ta/verbalizers/date.py new file mode 100644 index 000000000..382ea9f31 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/verbalizers/date.py @@ -0,0 +1,60 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SPACE, + GraphFst, + delete_preserve_order, + delete_space, +) + + +def _field(name: str) -> 'pynini.FstLike': + """ + Consumes ``name: "value"``, emitting the value. + """ + return pynutil.delete(f"{name}: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + +class DateFst(GraphFst): + """ + Finite state transducer for verbalizing dates, e.g. + date { day: "பதினைந்து" month: "ஜூன்" year: "இரண்டாயிரத்து இருபத்துநான்கு" } -> பதினைந்து ஜூன் இரண்டாயிரத்து இருபத்துநான்கு + date { year: "இரண்டாயிரத்து இருபத்துநான்கு" month: "ஜூன்" day: "பதினைந்து" } -> இரண்டாயிரத்து இருபத்துநான்கு ஜூன் பதினைந்து + date { era: "கிறிஸ்து பிறகு" year: "இரண்டாயிரத்து இருபத்துநான்கு" } -> கிறிஸ்து பிறகு இரண்டாயிரத்து இருபத்துநான்கு + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="date", kind="verbalize", deterministic=deterministic) + + day, month, year, era = _field("day"), _field("month"), _field("year"), _field("era") + graph = ( + day + NEMO_SPACE + month + | month + NEMO_SPACE + day + | day + NEMO_SPACE + month + NEMO_SPACE + year + | month + NEMO_SPACE + day + NEMO_SPACE + year + | year + NEMO_SPACE + month + NEMO_SPACE + day + | era + | era + NEMO_SPACE + year + ) + self.graph = graph + delete_space + delete_preserve_order + self.fst = self.delete_tokens(self.graph).optimize() diff --git a/nemo_text_processing/text_normalization/ta/verbalizers/decimal.py b/nemo_text_processing/text_normalization/ta/verbalizers/decimal.py new file mode 100644 index 000000000..5e747c3d5 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/verbalizers/decimal.py @@ -0,0 +1,73 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, insert_space +from nemo_text_processing.text_normalization.ta.graph_utils import MINUS_WORD, PLUS_WORD, POINT_WORD + + +class DecimalFst(GraphFst): + """ + Finite state transducer for verbalizing decimals, e.g. + decimal { negative: "true" integer_part: "பன்னிரண்டு" fractional_part: "ஐந்து பூஜ்யம் பூஜ்யம் ஆறு" quantity: "கோடி" } -> மைனஸ் பன்னிரண்டு புள்ளி ஐந்து பூஜ்யம் பூஜ்யம் ஆறு கோடி + decimal { integer_part: "ஒன்று" quantity: "லட்சம்" } -> ஒரு லட்சம் + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="decimal", kind="verbalize", deterministic=deterministic) + + delete_space = pynutil.delete(" ") + self.optional_sign = pynini.closure( + ( + pynini.cross("negative: \"true\"", f"{MINUS_WORD} ") + | pynini.cross("positive: \"true\"", f"{PLUS_WORD} ") + ) + + delete_space, + 0, + 1, + ) + self.integer = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + self.fractional_default = ( + pynutil.delete("fractional_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) + self.fractional = pynutil.insert(f" {POINT_WORD} ") + self.fractional_default + + self.quantity = ( + delete_space + + insert_space + + pynutil.delete("quantity: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + self.optional_quantity = pynini.closure(self.quantity, 0, 1) + + # A counting ஒன்று before a scale word reads as ஒரு (ஒரு லட்சம்). + one_as_oru = pynini.cross("ஒன்று", "ஒரு") | pynini.difference( + pynini.closure(NEMO_NOT_QUOTE, 1), pynini.accep("ஒன்று") + ) + integer_before_quantity = pynutil.delete("integer_part: \"") + one_as_oru + pynutil.delete("\"") + + graph = self.optional_sign + ( + integer_before_quantity + self.quantity + | self.integer + delete_space + self.fractional + self.optional_quantity + ) + + self.numbers = graph + self.fst = self.delete_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/ta/verbalizers/fraction.py b/nemo_text_processing/text_normalization/ta/verbalizers/fraction.py new file mode 100644 index 000000000..a03ece05b --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/verbalizers/fraction.py @@ -0,0 +1,99 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + GraphFst, + delete_space, + insert_space, +) +from nemo_text_processing.text_normalization.en.utils import load_labels +from nemo_text_processing.text_normalization.ta.graph_utils import FRACTION_WORD, MINUS_WORD, TA_ARAI +from nemo_text_processing.text_normalization.ta.utils import get_abs_path + + +class FractionFst(GraphFst): + """ + Finite state transducer for verbalizing fractions, e.g. + fraction { numerator: "ஒன்று" denominator: "இரண்டு" } -> அரை + fraction { numerator: "ஐந்து" denominator: "எழுபத்தேழு" } -> ஐந்து கீழ் எழுபத்தேழு + fraction { integer_part: "இரண்டு" numerator: "மூன்று" denominator: "நான்கு" } -> இரண்டே முக்கால் + + 1/2, 1/4 and 3/4 have their own everyday words (``data/fraction/idiomatic.tsv``) and are + spoken as those; any other pair is read with கீழ். A mixed number fuses with an everyday + fraction (2 3/4 -> இரண்டே முக்கால், 1 1/2 -> ஒன்றரை) and otherwise joins with மற்றும். + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="fraction", kind="verbalize", deterministic=deterministic) + + denominator = pynutil.delete("denominator: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + numerator = pynutil.delete("numerator: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + integer = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + special_rows = [r for r in load_labels(get_abs_path("data/fraction/idiomatic.tsv")) if len(r) >= 3] + + # Both readings consume the same field order so the weight decides between them, because + # the engine picks a field permutation before the verbalizer sees the token. + special = pynini.union( + *[ + pynutil.delete("numerator: \"") + + pynutil.delete(num) + + pynutil.delete("\"") + + delete_space + + pynutil.delete("denominator: \"") + + pynini.cross(den, word) + + pynutil.delete("\"") + for num, den, word, *_ in special_rows + ] + ).optimize() + with_keezh = numerator + delete_space + insert_space + pynutil.insert(FRACTION_WORD + " ") + denominator + + # The half joins as -ரை; the quarters take the -ஏ link and stay a separate word. An + # integer that does not end in -உ (ஆயிரம்) has no fused form and falls back to மற்றும். + mixed = [] + for num, den, word, *_ in special_rows: + link, tail = ("ரை", "") if word == TA_ARAI else ("ே", " " + word) + mixed.append( + pynutil.delete("integer_part: \"") + + (pynini.closure(NEMO_NOT_QUOTE, 1) @ (NEMO_SIGMA + pynini.cross("ு", link))) + + pynutil.delete("\"") + + delete_space + + pynutil.delete("numerator: \"") + + pynutil.delete(num) + + pynutil.delete("\"") + + delete_space + + pynutil.delete("denominator: \"") + + pynini.cross(den, tail) + + pynutil.delete("\"") + ) + + bare = pynutil.add_weight(special, -1.0) | with_keezh + graph = ( + bare + | pynutil.add_weight(pynini.union(*mixed), -2.0) + | integer + delete_space + pynutil.insert(" மற்றும் ") + bare + ) + + optional_sign = pynini.closure(pynini.cross("negative: \"true\"", f"{MINUS_WORD} ") + delete_space, 0, 1) + self.graph = optional_sign + graph + self.fst = self.delete_tokens(self.graph).optimize() diff --git a/nemo_text_processing/text_normalization/ta/verbalizers/measure.py b/nemo_text_processing/text_normalization/ta/verbalizers/measure.py new file mode 100644 index 000000000..135451a52 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/verbalizers/measure.py @@ -0,0 +1,77 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_preserve_order, + delete_space, + insert_space, +) +from nemo_text_processing.text_normalization.ta.graph_utils import MINUS_WORD + + +class MeasureFst(GraphFst): + """ + Finite state transducer for verbalizing measures, e.g. + measure { cardinal { integer: "ஐந்து" } units: "கிலோமீட்டர்" preserve_order: true } -> ஐந்து கிலோமீட்டர் + measure { decimal { integer_part: "பன்னிரண்டு" fractional_part: "ஐந்து" } units: "கிலோகிராம்" preserve_order: true } -> பன்னிரண்டு புள்ளி ஐந்து கிலோகிராம் + measure { cardinal { integer: "ஒன்று" } units: "கிலோகிராம்" preserve_order: true } -> ஒரு கிலோகிராம் + + Args: + decimal: DecimalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="measure", kind="verbalize", deterministic=deterministic) + + optional_sign = pynini.closure(pynini.cross("negative: \"true\"", f"{MINUS_WORD} ") + delete_space, 0, 1) + # A whole-field ஒன்று before the unit noun reads as ஒரு. + one_as_oru = pynini.cross("ஒன்று", "ஒரு") | pynini.difference( + pynini.closure(NEMO_NOT_QUOTE, 1), pynini.accep("ஒன்று") + ) + graph_cardinal = ( + pynutil.delete("cardinal {") + + delete_space + + pynutil.delete("integer: \"") + + one_as_oru + + pynutil.delete("\"") + + delete_space + + pynutil.delete("}") + ) + graph_decimal = ( + pynutil.delete("decimal {") + + delete_space + + decimal.integer + + delete_space + + decimal.fractional + + delete_space + + pynutil.delete("}") + ) + units = pynutil.delete("units: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + graph = ( + optional_sign + + (graph_cardinal | graph_decimal) + + delete_space + + insert_space + + units + + delete_preserve_order + ) + self.fst = self.delete_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/ta/verbalizers/money.py b/nemo_text_processing/text_normalization/ta/verbalizers/money.py new file mode 100644 index 000000000..a737f68db --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/verbalizers/money.py @@ -0,0 +1,115 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, NEMO_SPACE, GraphFst +from nemo_text_processing.text_normalization.en.utils import load_labels +from nemo_text_processing.text_normalization.ta.graph_utils import MINUS_WORD, POINT_WORD +from nemo_text_processing.text_normalization.ta.utils import get_abs_path + + +class MoneyFst(GraphFst): + """ + Finite state transducer for verbalizing money, e.g. + money { integer_part: "பன்னிரண்டு" currency_maj: "ரூபாய்" } -> பன்னிரண்டு ரூபாய் + money { integer_part: "பன்னிரண்டு" currency_maj: "ரூபாய்" fractional_part: "ஐம்பது" currency_min: "centiles" } -> பன்னிரண்டு ரூபாய் ஐம்பது பைசா + money { currency_maj: "ரூபாய்" integer_part: "பூஜ்யம்" fractional_part: "ஐம்பது" currency_min: "centiles" } -> ஐம்பது பைசா + money { integer_part: "ஐம்பது" currency_maj: "ரூபாய்" morphosyntactic_features: "ஆக" } -> ஐம்பது ரூபாயாக + + The ``centiles`` placeholder is resolved from ``data/money/major_minor_currencies.tsv``; a + case suffix in ``morphosyntactic_features`` is joined onto the currency word with sandhi + (ரூபாய் + ஆக -> ரூபாயாக, ரூபாய் + இல் -> ரூபாயில்). + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="money", kind="verbalize", deterministic=deterministic) + + major_minor_currencies = [ + r for r in load_labels(get_abs_path("data/money/major_minor_currencies.tsv")) if len(r) >= 2 + ] + + optional_suffix = pynini.closure( + pynutil.delete(" morphosyntactic_features: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\""), + 0, + 1, + ) + currency_major = ( + pynutil.delete("currency_maj: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + + optional_suffix + ) + + # A whole-field ஒன்று, or ஒன்று heading a quantity phrase (ஒரு லட்சம்), reads as ஒரு, but not + # before a decimal point (ஒன்று புள்ளி ஐந்து கோடி). + not_point = pynini.difference( + pynini.closure(NEMO_NOT_QUOTE, 1), pynini.accep(POINT_WORD) + pynini.closure(NEMO_NOT_QUOTE) + ) + one_phrase = (pynini.accep("ஒன்று") + pynini.closure(" " + not_point, 0, 1)).optimize() + one_as_oru = pynini.cross("ஒன்று", "ஒரு") + pynini.closure(" " + not_point, 0, 1) | pynini.difference( + pynini.closure(NEMO_NOT_QUOTE, 1), one_phrase + ) + integer_part = pynutil.delete("integer_part: \"") + one_as_oru + pynutil.delete("\"") + fractional_part = pynutil.delete("fractional_part: \"") + one_as_oru + pynutil.delete("\"") + + # Major denomination only. + graph_major_only = integer_part + pynini.accep(NEMO_SPACE) + currency_major + + major_minor_graphs = [] + minor_graphs = [] + for major, minor, *_ in major_minor_currencies: + graph_major = pynutil.delete("currency_maj: \"") + pynini.accep(major) + pynutil.delete("\"") + graph_minor = pynutil.delete("currency_min: \"") + pynini.cross("centiles", minor) + pynutil.delete("\"") + major_minor_graphs.append( + integer_part + + pynini.accep(NEMO_SPACE) + + graph_major + + pynini.accep(NEMO_SPACE) + + fractional_part + + pynini.accep(NEMO_SPACE) + + graph_minor + ) + # Minor denomination only: the zero integer part and the major word are silent. + minor_graphs.append( + pynutil.delete("integer_part: \"பூஜ்யம்\"") + + pynutil.delete(NEMO_SPACE) + + pynutil.delete("currency_maj: \"") + + pynutil.delete(major) + + pynutil.delete("\"") + + pynutil.delete(NEMO_SPACE) + + fractional_part + + pynini.accep(NEMO_SPACE) + + graph_minor + ) + + graph = ( + graph_major_only + | pynini.union(*major_minor_graphs) + | pynutil.add_weight(pynini.union(*minor_graphs), -0.1) + ) + + optional_sign = pynini.closure(pynini.cross("negative: \"true\" ", f"{MINUS_WORD} "), 0, 1) + graph = optional_sign + graph + suffix_sandhi = pynini.cdrewrite( + pynini.union(pynini.cross("்ஆ", "ா"), pynini.cross("்இ", "ி")), "", "", NEMO_SIGMA + ) + graph = graph @ suffix_sandhi + + self.fst = self.delete_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/ta/verbalizers/ordinal.py b/nemo_text_processing/text_normalization/ta/verbalizers/ordinal.py new file mode 100644 index 000000000..3071ecef6 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/verbalizers/ordinal.py @@ -0,0 +1,35 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for verbalizing ordinals, e.g. + ordinal { integer: "பத்தாவது" } -> பத்தாவது + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic) + + graph = pynutil.delete("integer: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + self.fst = self.delete_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/ta/verbalizers/roman.py b/nemo_text_processing/text_normalization/ta/verbalizers/roman.py new file mode 100644 index 000000000..2770f6602 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/verbalizers/roman.py @@ -0,0 +1,53 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + GraphFst, + delete_preserve_order, + delete_space, + insert_space, +) + + +class RomanFst(GraphFst): + """ + Finite state transducer for verbalizing Roman numerals read in context, e.g. + roman { key_cardinal: "வகுப்பு" integer: "பன்னிரண்டு" preserve_order: true } -> வகுப்பு பன்னிரண்டு + roman { integer: "பன்னிரண்டாம்" key_cardinal: "வகுப்பு" preserve_order: true } -> பன்னிரண்டாம் வகுப்பு + roman { integer: "பன்னிரண்டாம்" } -> பன்னிரண்டாம் + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="roman", kind="verbalize", deterministic=deterministic) + + # A multi-word cue travels with U+00A0 NO-BREAK SPACE; speak it with plain spaces. + key = ( + pynutil.delete("key_cardinal: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) @ pynini.cdrewrite(pynini.cross(" ", " "), "", "", NEMO_SIGMA) + integer = pynutil.delete("integer: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + graph = pynini.union( + key + delete_space + insert_space + integer, + integer + delete_space + insert_space + key, + integer, + ) + self.fst = self.delete_tokens(graph + delete_preserve_order).optimize() diff --git a/nemo_text_processing/text_normalization/ta/verbalizers/telephone.py b/nemo_text_processing/text_normalization/ta/verbalizers/telephone.py new file mode 100644 index 000000000..99cee61bf --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/verbalizers/telephone.py @@ -0,0 +1,51 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import ( + MIN_NEG_WEIGHT, + NEMO_NOT_QUOTE, + NEMO_SPACE, + GraphFst, + delete_space, + insert_space, +) + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for verbalizing telephone numbers, e.g. + telephone { country_code: "பிளஸ் ஒன்பது ஒன்று" number_part: "ஒன்பது எட்டு ..." } -> பிளஸ் ஒன்பது ஒன்று ஒன்பது எட்டு ... + telephone { country_code: "பிளஸ் தொண்ணூற்றொன்று" } -> பிளஸ் தொண்ணூற்றொன்று + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="telephone", kind="verbalize", deterministic=deterministic) + + country_code = pynutil.delete("country_code: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + optional_country_code = pynini.closure(country_code + delete_space + insert_space, 0, 1) + number_part = ( + pynutil.delete("number_part: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynini.closure(pynutil.add_weight(pynutil.delete(NEMO_SPACE), MIN_NEG_WEIGHT), 0, 1) + + pynutil.delete("\"") + ) + graph = (optional_country_code + number_part) | country_code + self.fst = self.delete_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/ta/verbalizers/time.py b/nemo_text_processing/text_normalization/ta/verbalizers/time.py new file mode 100644 index 000000000..4cd3f3a28 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/verbalizers/time.py @@ -0,0 +1,60 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing time, e.g. + time { hours: "பத்து" minutes: "முப்பது" } -> பத்து மணி முப்பது நிமிடம் + time { hours: "பத்து" } -> பத்து மணி + time { hours: "பத்து" minutes: "முப்பது" meridiem: "முற்பகல்" } -> முற்பகல் பத்து மணி முப்பது நிமிடம் + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="time", kind="verbalize", deterministic=deterministic) + + hour = pynutil.delete("hours: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + minute = pynutil.delete("minutes: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + second = pynutil.delete("seconds: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + insert_mani = pynutil.insert("மணி") + insert_minute = pynutil.insert("நிமிடம்") + insert_second = pynutil.insert("வினாடி") + + graph_h = hour + insert_space + insert_mani + graph_hm = graph_h + delete_space + insert_space + minute + insert_space + insert_minute + graph_hms = graph_hm + delete_space + insert_space + second + insert_space + insert_second + graph_hs = graph_h + delete_space + insert_space + second + insert_space + insert_second + + # A day-part word or a resolved AM/PM is fronted. + meridiem = pynini.closure( + pynutil.delete("meridiem: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + + delete_space + + insert_space, + 0, + 1, + ) + self.graph = meridiem + (graph_hms | graph_hm | graph_hs | graph_h) + self.fst = self.delete_tokens(self.graph).optimize() diff --git a/nemo_text_processing/text_normalization/ta/verbalizers/verbalize.py b/nemo_text_processing/text_normalization/ta/verbalizers/verbalize.py new file mode 100644 index 000000000..36502ab0a --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/verbalizers/verbalize.py @@ -0,0 +1,70 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.ta.graph_utils import GraphFst +from nemo_text_processing.text_normalization.ta.verbalizers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.ta.verbalizers.date import DateFst +from nemo_text_processing.text_normalization.ta.verbalizers.decimal import DecimalFst +from nemo_text_processing.text_normalization.ta.verbalizers.fraction import FractionFst +from nemo_text_processing.text_normalization.ta.verbalizers.measure import MeasureFst +from nemo_text_processing.text_normalization.ta.verbalizers.money import MoneyFst +from nemo_text_processing.text_normalization.ta.verbalizers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.ta.verbalizers.roman import RomanFst +from nemo_text_processing.text_normalization.ta.verbalizers.telephone import TelephoneFst +from nemo_text_processing.text_normalization.ta.verbalizers.time import TimeFst +from nemo_text_processing.text_normalization.ta.verbalizers.whitelist import WhiteListFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + + cardinal_graph = CardinalFst(deterministic=deterministic).fst + decimal = DecimalFst(deterministic=deterministic) + decimal_graph = decimal.fst + fraction_graph = FractionFst(deterministic=deterministic).fst + date_graph = DateFst(deterministic=deterministic).fst + time_graph = TimeFst(deterministic=deterministic).fst + ordinal_graph = OrdinalFst(deterministic=deterministic).fst + measure_graph = MeasureFst(decimal=decimal, deterministic=deterministic).fst + money_graph = MoneyFst(deterministic=deterministic).fst + telephone_graph = TelephoneFst(deterministic=deterministic).fst + roman_graph = RomanFst(deterministic=deterministic).fst + whitelist_graph = WhiteListFst(deterministic=deterministic).fst + + graph = ( + cardinal_graph + | decimal_graph + | fraction_graph + | date_graph + | time_graph + | ordinal_graph + | measure_graph + | money_graph + | telephone_graph + | roman_graph + | whitelist_graph + ) + + self.fst = graph diff --git a/nemo_text_processing/text_normalization/ta/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/ta/verbalizers/verbalize_final.py new file mode 100644 index 000000000..827f4d5e5 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/verbalizers/verbalize_final.py @@ -0,0 +1,68 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import delete_extra_space, delete_space +from nemo_text_processing.text_normalization.ta.graph_utils import GraphFst, generator_main +from nemo_text_processing.text_normalization.ta.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.text_normalization.ta.verbalizers.word import WordFst + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence, e.g. + tokens { cardinal { integer: "இருபத்துமூன்று" } } tokens { name: "பேர்" } tokens { name: "." } -> இருபத்துமூன்று பேர் . + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"ta_tn_{deterministic}_deterministic_verbalizer.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + logging.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + else: + verbalize = VerbalizeFst(deterministic=deterministic).fst + word = WordFst(deterministic=deterministic).fst + types = verbalize | word + + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + + self.fst = graph.optimize() + if far_file: + generator_main(far_file, {"verbalize": self.fst}) + logging.info(f"VerbalizeFinalFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/ta/verbalizers/whitelist.py b/nemo_text_processing/text_normalization/ta/verbalizers/whitelist.py new file mode 100644 index 000000000..0524bf9ac --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/verbalizers/whitelist.py @@ -0,0 +1,43 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for verbalizing whitelist entries, e.g. + tokens { name: "டாக்டர்" } -> டாக்டர் + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="whitelist", kind="verbalize", deterministic=deterministic) + + graph = ( + pynutil.delete("name:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_CHAR - " ", 1) + + pynutil.delete("\"") + ) + # Multi-word values travel with U+00A0 NO-BREAK SPACE; speak them with plain spaces. + graph = graph @ pynini.cdrewrite(pynini.cross(" ", " "), "", "", NEMO_SIGMA) + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/ta/verbalizers/word.py b/nemo_text_processing/text_normalization/ta/verbalizers/word.py new file mode 100644 index 000000000..a47f93612 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/verbalizers/word.py @@ -0,0 +1,45 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + + +class WordFst(GraphFst): + """ + Finite state transducer for verbalizing plain words, e.g. + tokens { name: "தமிழ்" } -> தமிழ் + + A mark following a word attaches to it without a space. Multi-word values travel with + U+00A0 NO-BREAK SPACE and are spoken with plain spaces. + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="verbalize", deterministic=deterministic) + + chars = pynini.closure(NEMO_CHAR - " ", 1) + punct = pynini.union("!", "?", ".", ",", "-", ":", ";", "।") + char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"") + + graph = char + pynini.closure(delete_space + punct, 0, 1) + graph = graph @ pynini.cdrewrite(pynini.cross(" ", ""), "", punct, NEMO_SIGMA) + graph = graph @ pynini.cdrewrite(pynini.cross(" ", " "), "", "", NEMO_SIGMA) + + self.fst = graph.optimize() diff --git a/tests/nemo_text_processing/ta/__init__.py b/tests/nemo_text_processing/ta/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/tests/nemo_text_processing/ta/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..6aa9a39c5 --- /dev/null +++ b/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,101 @@ +இருபத்துமூன்று பேர் வந்தனர்~23 பேர் வந்தனர் +இருபத்து மூன்று~23 +மைனஸ் நூற்று இருபது~-120 +நூற்று இருபத்துமூன்று~123 +நானூற்று ஐம்பத்தாறு~456 +தொள்ளாயிரம்~900 +ஒன்று இலட்சம்~100000 +ஒன்று கோடி ஐம்பது இலட்சம்~15000000 +பூஜ்யம்~0 +பூஜ்ஜியம்~0 +இருபத்தைந்து ஆயிரம்~25000 +நூற்றுப்பத்து~110 +நூற்றுத்தொண்ணூறு~190 +ஒரு இலட்சம்~100000 +ஆயிரம்~1000 +இரண்டாயிரத்து இருபத்துநான்கு~2024 +ஒரு கோடி ஐம்பது இலட்சம்~15000000 +ஆயிரத்து நூற்று ஒன்று~1101 +இரண்டாயிரத்து நூற்று ஒன்று~2101 +ஒரு கோடி~10000000 +ஒன்னு~1 +ரெண்டு~2 +மூணு~3 +நாலு~4 +அஞ்சு~5 +இருவத்தி மூணு~23 +நூத்தி ஐம்பது~150 +இருபத்திமூன்று~23 +ஆயிரத்தி ஒன்னு~1001 +ஓர் ஆயிரம்~1000 +ஒரு கோடியே ஐம்பது லட்சத்து ஐந்நூறு~15000500 +ஐந்து லட்சத்து அறுபதாயிரம்~560000 +எழுவது~எழுவது +நூத்தி அம்பது~150 +பதினஞ்சு~15 +நாற்பத்தஞ்சு~45 +நாப்பத்தி ஒன்னு~41 +ரெண்டாயிரத்து இருவத்தி நாலு~2024 +இருநூத்தி ஐம்பது~250 +இருபத்திரண்டு~22 +தொண்ணூற்றிரண்டு~92 +ஆயிரத்து இருபத்திரண்டு~1022 +நான்காயிரத்து நூற்றுப்பதினாறு~4116 +ஏழாயிரத்து ஐந்நூற்றுப்பதினெட்டு~7518 +நூற்றிரண்டு~102 +இருநூற்றிரண்டு~202 +முன்னூறு~300 +முன்னூற்று ஐம்பது~350 +இரண்டாயிரத்து இருபத்துநான்கில்~2024ல் +இரண்டாயிரத்து இருபத்துநான்குக்கு~2024க்கு +ஐந்தில்~5ல் +ஆயிரத்தில்~1000ல் +ஐந்தாக~5ஆக +ஒரு ஆயிரம்~1000 +ஒரு ஆயிரம் பேர்~1000 பேர் +ஒன்றரை ஆயிரம்~1500 +ஐந்து புள்ளி ஐந்து ஆயிரம்~5500 +ஒன்று பூஜ்யம் ஒன்று~101 +ஆறு பூஜ்யம் பூஜ்யம் பூஜ்யம் பூஜ்யம் ஒன்று~600001 +ஒரு நாள் ஒரு ராஜா~ஒரு நாள் ஒரு ராஜா +ஒரு பையன்~ஒரு பையன் +ஒரு வேளை~ஒரு வேளை +ஓர் இடம்~ஓர் இடம் +ஒரே ஒரு~ஒரே ஒரு +ஒன்று சேர்~ஒன்று சேர் +ஒன்று கூடி~ஒன்று கூடி +எல்லாம் ஒன்று~எல்லாம் ஒன்று +நூற்றொன்று~101 +ஆயிரத்தொன்று~1001 +நூறுக்கு~100க்கு +ஐந்துகள்~5கள் +பிளஸ் ஐந்து~+5 +பிளஸ் ஐம்பது~+50 +இருபத்து ஒன்று~21 +நாற்பத்து ஐந்து~45 +எழுபத்தி மூன்று~73 +எண்பத்தியொன்று~81 +தொண்ணூற்றியெட்டு~98 +எண்பத்தினான்கு~84 +தொண்ணூற்றிஐந்து~95 +எண்பத்திஏழு~87 +தொன்ணூறு~90 +பூஜ்யம் புள்ளி ஐந்து ஆயிரம்~500 +பூஜ்யம் புள்ளி பூஜ்யம் ஆயிரம்~0 +ஐந்து புள்ளி பூஜ்யம் ஆயிரம்~5000 +இரண்டு இலட்சத்து ஐம்பதாயிரம்~250000 +இரண்டு இலட்சம் ஐம்பது ஆயிரம்~250000 +எண்பத்துமூன்றாயிரத்து அறுநூற்றுப்பதினாறு~83616 +எழுபதாயிரத்து நூற்று ஐந்து~70105 +இருபத்தோராயிரம்~21000 +பத்தாயிரம்~10000 +ஐந்து கோடியே எழுபத்தேழு இலட்சத்து ஐம்பத்துமூன்றாயிரத்து முந்நூற்று முப்பத்தாறு~57753336 +பதினான்கு கோடியே தொண்ணூற்றாறு இலட்சம்~149600000 +பதின்மூன்று~13 +பதிமூன்று~13 +இரண்டாயிரத்து பதின்மூன்று~2013 +ஐந்தால்~5ஆல் +இரண்டாயிரத்தால்~2000ஆல் +நூறும்~100உம் +நூற்றைம்பது~150 +நூற்றெழுபத்து ஒன்று~171 diff --git a/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_date.txt b/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_date.txt new file mode 100644 index 000000000..d3155d4c3 --- /dev/null +++ b/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_date.txt @@ -0,0 +1,11 @@ +பதினைந்து ஜூன் இரண்டாயிரத்து இருபத்துநான்கு~15 ஜூன் 2024 +பதினைந்து ஜூன் இரண்டு ஆயிரம் இருபத்துநான்கு~15 ஜூன் 2024 +ஒன்று ஏப்ரல் இரண்டாயிரத்து இருபத்துநான்கு~1 ஏப்ரல் 2024 +ஐந்நூறு ஜூன்~500 ஜூன் +பதினைந்து ஜூன் இரண்டாயிரத்து இருபத்துநான்கில்~15 ஜூன் 2024ல் +பத்து ஜனவரி இரண்டாயிரத்து இருபது~10 ஜனவரி 2020 +ஜூலை இரண்டாயிரத்து இருபத்தைந்து~ஜூலை 2025 +இரண்டாயிரத்து இருபத்துநான்கு டிசம்பர் முப்பத்தொன்று~2024 டிசம்பர் 31 +மார்ச் ஐந்து~மார்ச் 5 +இருபத்தைந்து டிசம்பர்~25 டிசம்பர் +ஒன்று ஜனவரி ஆயிரத்து தொள்ளாயிரத்து தொண்ணூறு~1 ஜனவரி 1990 diff --git a/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_decimal.txt b/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_decimal.txt new file mode 100644 index 000000000..e38a58666 --- /dev/null +++ b/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_decimal.txt @@ -0,0 +1,30 @@ +பன்னிரண்டு புள்ளி ஐந்து~12.5 +பூஜ்யம் புள்ளி ஐந்து~0.5 +மூன்று புள்ளி ஒன்று நான்கு~3.14 +மைனஸ் இரண்டு புள்ளி ஐந்து~-2.5 +ஒன்றரை~1.5 +இரண்டரை கிலோ~2.5 கிலோ +முக்கால்~0.75 +ஒண்ணரை~1.5 +ரெண்டரை~2.5 +ஒன்னரை~1.5 +ஒன்று புள்ளி இரண்டு புள்ளி மூன்று~1.2.3 +ஐந்து புள்ளி ஐந்து லட்சம்~5.5 லட்சம் +ஒன்று புள்ளி ஐந்து கோடி~1.5 கோடி +பத்து புள்ளி ஐந்து லட்சம்~10.5 லட்சம் +ஒன்றரை லட்சம்~1.5 லட்சம் +பத்தே கால்~10.25 +ஒன்றே கால்~1.25 +ஒன்றே முக்கால்~1.75 +ஒன்றேகால்~1.25 +ஒன்பதே கால்~9.25 +கால் வலிக்கிறது~கால் வலிக்கிறது +அரை நிஜார்~அரை நிஜார் +முக்கால் வட்டம்~முக்கால் வட்டம் +கால் கிலோ~கால் கிலோ +பிளஸ் ஐந்து புள்ளி ஐந்து~+5.5 +பிளஸ் பூஜ்யம் புள்ளி பூஜ்யம்~+0.0 +இருபத்தைந்தரை~25.5 +நூறரை~100.5 +இருபத்தைந்தே கால்~25.25 +ஒன்று புள்ளி இரண்டு ஐந்து லட்சம்~1.25 லட்சம் diff --git a/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_fraction.txt b/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_fraction.txt new file mode 100644 index 000000000..139cb82fd --- /dev/null +++ b/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_fraction.txt @@ -0,0 +1,22 @@ +நான்கில் மூன்று~3/4 +இரண்டில் ஒன்று~1/2 +எட்டில் ஏழு~7/8 +ஏழில் இருபத்திரண்டு~22/7 +ஆயிரத்தில் ஒன்று~1/1000 +இருபத்தைந்தில் மூன்று~3/25 +ஐந்து கீழ் எழுபத்தேழு~5/77 +ஏழு கீழ் எட்டு~7/8 +ஒன்று கீழ் ஆயிரம்~1/1000 +இருபத்திரண்டு கீழ் ஏழு~22/7 +நான்கில் ஒன்று~1/4 +மூன்றில் இரண்டு~2/3 +பத்தில் ஒன்பது~9/10 +நூற்றில் இருபது~20/100 +மூன்று கீழ் எட்டு~3/8 +ஆயிரத்தில் ஐந்து~5/1000 +ஒன்பது கீழ் பத்து~9/10 +இரண்டு மூன்றில் இரண்டு~2 2/3 +நூறில் ஐந்து~5/100 +ஏழில் ஒன்று~1/7 +பதினைந்தில் நான்கு~4/15 +இருபதில் மூன்று~3/20 diff --git a/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_measure.txt new file mode 100644 index 000000000..9465bae84 --- /dev/null +++ b/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_measure.txt @@ -0,0 +1,8 @@ +ஐந்து கிலோமீட்டர் நடந்தேன்~5 கிலோமீட்டர் நடந்தேன் +பன்னிரண்டு புள்ளி ஐந்து கிலோகிராம்~12.5 கிலோகிராம் +மைனஸ் நாற்பது டிகிரி செல்சியஸ்~-40 டிகிரி செல்சியஸ் +இருபது கிலோமீட்டர் தூரம்~20 கிலோமீட்டர் தூரம் +ஐம்பது கிலோ அரிசி~50 கிலோ அரிசி +நூறு மீட்டர் ஓட்டம்~100 மீட்டர் ஓட்டம் +மூன்று லிட்டர் பால்~3 லிட்டர் பால் +நாற்பது டிகிரி வெப்பம்~40 டிகிரி வெப்பம் diff --git a/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_money.txt new file mode 100644 index 000000000..c43fc5415 --- /dev/null +++ b/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_money.txt @@ -0,0 +1,44 @@ +ஐம்பது ரூபாய்~₹50 +ஐம்பது ரூபாய் ஐம்பது பைசா~₹50.50 +நூறு டாலர்~$100 +ஐம்பது பைசா~₹0.50 +ஒரு ரூபாய்~₹1 +ரூபாய் ஐம்பது~₹50 +ஐந்து கோடி ரூபாய்~₹5 கோடி +இரண்டு புள்ளி ஐந்து லட்சம் ரூபாய்~₹2.5 லட்சம் +மைனஸ் ஐந்நூறு ரூபாய்~-₹500 +ஐநூறு ரூபாய்~₹500 +ஐந்து பைசா~₹0.05 +ஒரு பைசா~₹0.01 +ஐந்து ரூபாய் ஐந்து பைசா~₹5.05 +நூறு ரூபாய் ஐந்து பைசா~₹100.05 +மைனஸ் ஐம்பது பைசா~-₹0.50 +ஐந்து ஆயிரம் ரூபாய்~₹5000 +ஐம்பது ஆயிரம் ரூபாய்~₹50000 +ஐந்து லட்சத்து ஐம்பது ஆயிரம் ரூபாய்~₹550000 +ஐந்து கோடி ஐம்பது லட்சம் ரூபாய்~₹55000000 +ஒரு இலட்சம் ஐம்பது ஆயிரம் ரூபாய்~₹150000 +ஒன்றரை லட்சம் ரூபாய்~₹1.5 லட்சம் +இரண்டரை லட்சம் ரூபாய்~₹2.5 லட்சம் +ஐம்பது பவுண்ட்~£50 +நூறு யென்~¥100 +ஆயிரம் வான்~₩1000 +ஐம்பது யூரோக்கள்~€50 +ஐம்பது சென்ட்~$0.50 +ஐந்து பவுண்டு~5 பவுண்டு +ஐந்து ரூபாய் ஐம்பது காசுகள்~₹5.50 +ஐம்பது காசு~₹0.50 +ஐந்து டாலர் ஐம்பது பைசா~$5 ₹0.50 +ஐந்து ரூபாய் ஐம்பது சென்ட்~₹5 $0.50 +ஐந்து பவுண்ட் ஐம்பது பென்ஸ்~£5.50 +ஐந்து யூரோ ஐம்பது சென்ட்~€5.50 +ஐந்து லீரா ஐம்பது குருஸ்~₺5.50 +ஐந்து டாக்கா ஐம்பது பைசா~৳5.50 +ஐந்து யென் ஐம்பது சென்~¥5.50 +ஐந்து நைரா ஐம்பது கோபோ~₦5.50 +ஐந்து வான் ஐம்பது ஜியோன்~₩5.50 +₹5 கோடி ஐந்து பேர்~₹5 கோடி 5 பேர் +ஐம்பது ரூபாய்க்கு~₹50க்கு +ஐந்து முதல் பத்து ரூபாய்~₹5-10 +ஒன்றரை ரூபாய்~₹1.50 +ஒரு லட்சம் கோடி ரூபாய்~₹1 லட்சம் கோடி diff --git a/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_ordinal.txt b/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_ordinal.txt new file mode 100644 index 000000000..fb29960e0 --- /dev/null +++ b/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_ordinal.txt @@ -0,0 +1,16 @@ +ஐந்தாவது வகுப்பு~5வது வகுப்பு +பத்தாம் வகுப்பு~10ஆம் வகுப்பு +முதலாவது~1வது +இருபத்தொன்றாவது~21வது +இரண்டாயிரத்து இருபத்து மூன்றாம் ஆண்டு~2023ஆம் ஆண்டு +ஆயிரமாவது~1000வது +நூறாவது~100வது +நூற்றாவது~100வது +பூஜ்யமாவது~0வது +ஒரு இலட்சமாவது~100000வது +ஆயிரத்தொன்றாவது~1001வது +ஐந்தாவதாக~5வதாக +ஐந்தாவதுக்கு~5வதுக்கு +ஐந்தாவதில்~5வதில் +இருபத்தெட்டாம்~28ஆம் +ஏழாம்~7ஆம் diff --git a/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_telephone.txt b/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_telephone.txt new file mode 100644 index 000000000..71b9da187 --- /dev/null +++ b/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_telephone.txt @@ -0,0 +1,12 @@ +ஒன்பது ஒன்பது நான்கு மூன்று இரண்டு பூஜ்யம் ஆறு எட்டு ஏழு பூஜ்யம்~9943206870 +பூஜ்யம் நான்கு நான்கு இரண்டு எட்டு இரண்டு மூன்று பூஜ்யம் பூஜ்யம் பூஜ்யம் பூஜ்யம்~04428230000 +ஒன்று எட்டு பூஜ்யம் பூஜ்யம் நான்கு இரண்டு ஐந்து ஒன்று இரண்டு மூன்று நான்கு~18004251234 ++91 9876543210~+91 9876543210 +ஒன்று இரண்டு மூன்று~123 +பூஜ்யம் பூஜ்யம் ஏழு~007 +பிளஸ் தொண்ணூற்றொன்று ஒன்பது எட்டு ஏழு ஆறு ஐந்து நான்கு மூன்று இரண்டு ஒன்று பூஜ்யம்~+91 9876543210 +பூஜ்யம் நான்கு நான்கு இரண்டு நான்கு மூன்று மூன்று ஒன்று இரண்டு மூன்று நான்கு~04424331234 +பிளஸ் ஒன்று~+1 +ஒன்று இரண்டு மூன்று நான்கு ஐந்து ஆறு~123456 +எட்டு பூஜ்யம் பூஜ்யம் நான்கு இரண்டு ஐந்து ஒன்று இரண்டு மூன்று நான்கு~8004251234 +பிளஸ் நாற்பத்துநான்கு~+44 diff --git a/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_time.txt new file mode 100644 index 000000000..ffdc67379 --- /dev/null +++ b/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_time.txt @@ -0,0 +1,25 @@ +பத்து மணி முப்பது நிமிடம்~10:30 +பத்து மணிக்கு~10:00 +பத்து மணி முப்பது நிமிடம் நாற்பத்தைந்து வினாடி~10:30:45 +ஒன்பது மணி ஐந்து நிமிடம்~9:05 +இரண்டு மணி நேரம் காத்திருந்தேன்~2 மணி நேரம் காத்திருந்தேன் +பத்து மணிக்கு வா~10:00 வா +பத்து மணி~10 மணி +ஒரு மணிக்கு~1:00 +பத்து மணி அறுபது நிமிடம்~10 மணி 60 நிமிடம் +இருபத்தைந்து மணிக்கு~25 மணிக்கு +இருபத்துநான்கு மணிக்கு~24 மணிக்கு +இருபத்துமூன்று மணிக்கு~23:00 +பத்தரை மணி~10.5 மணி +பத்தரை மணிக்கு~10:30 +ஒன்றரை மணிக்கு~1:30 +பத்தே கால் மணிக்கு~10:15 +பத்தே முக்கால் மணிக்கு~10:45 +பத்து முப்பது மணிக்கு~10:30 +காலை பத்து மணி~காலை 10:00 +பிற்பகல் மூன்று மணி~பிற்பகல் 3:00 +பத்து மணி முப்பது நிமிடத்தில்~10:30 +பத்து மணி முப்பது நிமிடத்துக்கு~10:30 +அரை மணி~அரை மணி +இரண்டரை மணி நேரம்~2.5 மணி நேரம் +பத்து மணி முப்பது வினாடி~10:00:30 diff --git a/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_whitelist.txt b/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_whitelist.txt new file mode 100644 index 000000000..e3662111a --- /dev/null +++ b/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_whitelist.txt @@ -0,0 +1,6 @@ +டாக்டர்~டாக்டர் +ஒன்று சேர்ந்து~ஒன்று சேர்ந்து +ஒன்று போல~ஒன்று போல +ஒரு நாள்~ஒரு நாள் +கால் கிலோ~கால் கிலோ +அரை மணி நேரம்~அரை மணி நேரம் diff --git a/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_word.txt b/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_word.txt new file mode 100644 index 000000000..3cd939a4f --- /dev/null +++ b/tests/nemo_text_processing/ta/data_inverse_text_normalization/test_cases_word.txt @@ -0,0 +1,14 @@ +வணக்கம் நண்பர்களே~வணக்கம் நண்பர்களே +hello world~hello world +-5 டிகிரி~-5 டிகிரி +10:30 ஆகிறது~10:30 ஆகிறது +-₹500~-₹500 +[ஐந்து]~[ 5 ] +"ஐந்து"~" 5 " +5 = 5~5 = 5 +5=5~5 = 5 +a = b~a = b +பத்து முதல் இருபது வயது~10 முதல் 20 வயது +ஐந்து முதல் பத்து கிலோ~5 முதல் 10 கிலோ +நூறு முதல் இருநூறு~100 முதல் 200 +இரண்டு முதல் மூன்று நாட்கள்~2 முதல் 3 நாட்கள் diff --git a/tests/nemo_text_processing/ta/data_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..168a1f1d8 --- /dev/null +++ b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,128 @@ +0~பூஜ்யம் +5~ஐந்து +23~இருபத்துமூன்று +100~நூறு +101~நூற்று ஒன்று +123~நூற்று இருபத்துமூன்று +150~நூற்று ஐம்பது +205~இருநூற்று ஐந்து +300~முந்நூறு +456~நானூற்று ஐம்பத்தாறு +900~தொள்ளாயிரம் +999~தொள்ளாயிரத்து தொண்ணூற்றொன்பது +1000~ஆயிரம் +1500~ஆயிரத்து ஐந்நூறு +25000~இருபத்தைந்தாயிரம் +2024~இரண்டாயிரத்து இருபத்துநான்கு +1,00,000~ஒரு இலட்சம் +1,50,00,000~ஒரு கோடியே ஐம்பது இலட்சம் +-120~மைனஸ் நூற்று இருபது +௧௨௩~நூற்று இருபத்துமூன்று +-௨௩~மைனஸ் இருபத்துமூன்று +௯~ஒன்பது +order 66 confirmed~order அறுபத்தாறு confirmed +10-20 வயது~பத்து முதல் இருபது வயது +2024ல் நடந்தது.~இரண்டாயிரத்து இருபத்துநான்கில் நடந்தது . +5ல் இருந்து~ஐந்தில் இருந்து +007 என்ற எண்.~பூஜ்யம் பூஜ்யம் ஏழு என்ற எண் . +9876543210123~ஒன்பது எட்டு ஏழு ஆறு ஐந்து நான்கு மூன்று இரண்டு ஒன்று பூஜ்யம் ஒன்று இரண்டு மூன்று +110~நூற்றுப்பத்து +5-அவர்கள்~ஐந்து அவர்கள் +0-~பூஜ்யம் - +190~நூற்றுத்தொண்ணூறு +210~இருநூற்றுப்பத்து +315~முந்நூற்றுப்பதினைந்து +2000~இரண்டாயிரம் +3500~மூன்றாயிரத்து ஐந்நூறு +1101~ஆயிரத்து நூற்று ஒன்று +1110~ஆயிரத்து நூற்றுப்பத்து +99999~தொண்ணூற்றொன்பதாயிரத்து தொள்ளாயிரத்து தொண்ணூற்றொன்பது +10000000~ஒரு கோடி +99,99,999~தொண்ணூற்றொன்பது இலட்சத்து தொண்ணூற்றொன்பதாயிரத்து தொள்ளாயிரத்து தொண்ணூற்றொன்பது +100க்கு~நூறுக்கு +1000த்தில்~ஆயிரத்தில் +1990களில்~ஆயிரத்து தொள்ளாயிரத்து தொண்ணூறுகளில் +5-3=2~ஐந்து மைனஸ் மூன்று சமம் இரண்டு +100,000~ஒரு இலட்சம் +10–20~பத்து முதல் இருபது +1,000,000~பத்து இலட்சம் +1,500,000~பதினைந்து இலட்சம் +1,234,567~பன்னிரண்டு இலட்சத்து முப்பத்துநான்காயிரத்து ஐந்நூற்று அறுபத்தேழு +12,500~பன்னிரண்டாயிரத்து ஐந்நூறு +1,00,000க்கு~ஒரு இலட்சத்துக்கு +2024-ல்~இரண்டாயிரத்து இருபத்துநான்கில் +100-க்கு~நூறுக்கு +2024க்குள்~இரண்டாயிரத்து இருபத்துநான்குக்குள் +5ஆக~ஐந்தாக +10உம்~பத்தும் +5+3~ஐந்து கூட்டல் மூன்று +5*3~ஐந்து மடங்கு மூன்று +2^3~இரண்டு கேரட் மூன்று +5<10~ஐந்து விடக் குறைவு பத்து +5&6~ஐந்து மற்றும் ஆறு +எண் #5~எண் ஹாஷ் ஐந்து +5 - 3 = 2~ஐந்து மைனஸ் மூன்று சமம் இரண்டு +5−3~ஐந்து மைனஸ் மூன்று +−5~மைனஸ் ஐந்து +1,00,00,00,000~ஒன்று பூஜ்யம் பூஜ்யம் பூஜ்யம் பூஜ்யம் பூஜ்யம் பூஜ்யம் பூஜ்யம் பூஜ்யம் பூஜ்யம் +1,000,000,000~ஒன்று பூஜ்யம் பூஜ்யம் பூஜ்யம் பூஜ்யம் பூஜ்யம் பூஜ்யம் பூஜ்யம் பூஜ்யம் பூஜ்யம் +5ஆல்~ஐந்தால் +5ால்~ஐந்தால் +5ஓடு~ஐந்தோடு +5உடன்~ஐந்துடன் +5க்கும்~ஐந்துக்கும் +5ஐ~ஐந்தை +5-ஐ~ஐந்தை +5ன்~ஐந்தின் +5இன்~ஐந்தின் +5ும்~ஐந்தும் +5லிருந்து~ஐந்திலிருந்து +5தான்~ஐந்துதான் +1000ஆல்~ஆயிரத்தால் +1,00,000ும்~ஒரு இலட்சமும் +5கிலோ~ஐந்து கிலோ +5கிலோவுக்கு~ஐந்து கிலோவுக்கு +5​6~ஐந்து ஆறு +5⁠6~ஐந்து ஆறு +5.5‎~ஐந்து புள்ளி ஐந்து +‏5~ஐந்து +5­0~ஐம்பது +5‐6~ஐந்து முதல் ஆறு +5―6~ஐந்து முதல் ஆறு +10>5~பத்து விட அதிகம் ஐந்து +5 < 10~ஐந்து விடக் குறைவு பத்து +5~ ஐந்து ++5~பிளஸ் ஐந்து ++000~பிளஸ் பூஜ்யம் பூஜ்யம் பூஜ்யம் ++007~பிளஸ் பூஜ்யம் பூஜ்யம் ஏழு ++50ல்~பிளஸ் ஐம்பதில் ++~+ ++ 5~+ ஐந்து +5+3+2~ஐந்து கூட்டல் மூன்று கூட்டல் இரண்டு +1+2=3~ஒன்று கூட்டல் இரண்டு சமம் மூன்று +250000~இரண்டு இலட்சத்து ஐம்பதாயிரம் +200000~இரண்டு இலட்சம் +108000~ஒரு இலட்சத்து எட்டாயிரம் +1,01,000~ஒரு இலட்சத்து ஓராயிரம் +83616~எண்பத்துமூன்றாயிரத்து அறுநூற்றுப்பதினாறு +70105~எழுபதாயிரத்து நூற்று ஐந்து +11144~பதினொன்றாயிரத்து நூற்று நாற்பத்துநான்கு +17000~பதினேழாயிரம் +10000~பத்தாயிரம் +21000~இருபத்தோராயிரம் +51293~ஐம்பத்தோராயிரத்து இருநூற்றுத்தொண்ணூற்றுமூன்று +57753336~ஐந்து கோடியே எழுபத்தேழு இலட்சத்து ஐம்பத்துமூன்றாயிரத்து முந்நூற்று முப்பத்தாறு +149600000~பதினான்கு கோடியே தொண்ணூற்றாறு இலட்சம் +13~பதின்மூன்று +113~நூற்றுப்பதின்மூன்று +2013~இரண்டாயிரத்து பதின்மூன்று +13000~பதின்மூன்றாயிரம் +15,06~பதினைந்து , பூஜ்யம் ஆறு +1,00~ஒன்று , பூஜ்யம் பூஜ்யம் +10,00~பத்து , பூஜ்யம் பூஜ்யம் +15,06,2024~பதினைந்து , பூஜ்யம் ஆறு , இரண்டாயிரத்து இருபத்துநான்கு +12,34,56,78,901~ஒன்று இரண்டு மூன்று நான்கு ஐந்து ஆறு ஏழு எட்டு ஒன்பது பூஜ்யம் ஒன்று +1,000,00,000~ஆயிரம் , பூஜ்யம் பூஜ்யம் , பூஜ்யம் பூஜ்யம் பூஜ்யம் +௧,௨௫,௦௦௦~ஒரு இலட்சத்து இருபத்தைந்தாயிரம் +151~நூற்று ஐம்பத்தொன்று +170~நூற்று எழுபது diff --git a/tests/nemo_text_processing/ta/data_text_normalization/test_cases_date.txt b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_date.txt new file mode 100644 index 000000000..db97a6ccd --- /dev/null +++ b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_date.txt @@ -0,0 +1,23 @@ +15-06-2024~பதினைந்து ஜூன் இரண்டாயிரத்து இருபத்துநான்கு +01-04-2024~ஒன்று ஏப்ரல் இரண்டாயிரத்து இருபத்துநான்கு +15/06/2024~பதினைந்து ஜூன் இரண்டாயிரத்து இருபத்துநான்கு +௦௧-௦௪-௨௦௨௪~ஒன்று ஏப்ரல் இரண்டாயிரத்து இருபத்துநான்கு +15-06-24~பதினைந்து - பூஜ்யம் ஆறு முதல் இருபத்துநான்கு +13-13-2024~பதின்மூன்று முதல் பதின்மூன்று - இரண்டாயிரத்து இருபத்துநான்கு +06-15-2024~ஜூன் பதினைந்து இரண்டாயிரத்து இருபத்துநான்கு +2024-06-15~இரண்டாயிரத்து இருபத்துநான்கு ஜூன் பதினைந்து +6.1.2024~ஆறு ஜனவரி இரண்டாயிரத்து இருபத்துநான்கு +15-06-2024ல்~பதினைந்து ஜூன் இரண்டாயிரத்து இருபத்துநான்கில் +15.06.2024க்கு~பதினைந்து ஜூன் இரண்டாயிரத்து இருபத்துநான்குக்கு +15-06-2024ஆம் தேதி~பதினைந்து ஜூன் இரண்டாயிரத்து இருபத்துநான்காம் தேதி +15-06-2024க்குள்~பதினைந்து ஜூன் இரண்டாயிரத்து இருபத்துநான்குக்குள் +15-ஜூன்-2024~பதினைந்து ஜூன் இரண்டாயிரத்து இருபத்துநான்கு +01-01-2000ல்~ஒன்று ஜனவரி இரண்டாயிரத்தில் +01-01-2000~ஒன்று ஜனவரி இரண்டாயிரம் +15-06/2024~பதினைந்து முதல் பூஜ்யம் ஆறு / இரண்டாயிரத்து இருபத்துநான்கு +15.06-2024~பதினைந்து புள்ளி பூஜ்யம் ஆறு - இரண்டாயிரத்து இருபத்துநான்கு +2024/06-15~இரண்டாயிரத்து இருபத்துநான்கு / பூஜ்யம் ஆறு முதல் பதினைந்து +15.06.2024~பதினைந்து ஜூன் இரண்டாயிரத்து இருபத்துநான்கு +2024/06/15~இரண்டாயிரத்து இருபத்துநான்கு ஜூன் பதினைந்து +15/06-2024~பதினைந்து / பூஜ்யம் ஆறு முதல் இரண்டாயிரத்து இருபத்துநான்கு +01-01-2000க்கு~ஒன்று ஜனவரி இரண்டாயிரத்துக்கு diff --git a/tests/nemo_text_processing/ta/data_text_normalization/test_cases_decimal.txt b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_decimal.txt new file mode 100644 index 000000000..8d7ce0f29 --- /dev/null +++ b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_decimal.txt @@ -0,0 +1,23 @@ +12.5~பன்னிரண்டு புள்ளி ஐந்து +0.5~பூஜ்யம் புள்ளி ஐந்து +3.14~மூன்று புள்ளி ஒன்று நான்கு +-2.5~மைனஸ் இரண்டு புள்ளி ஐந்து +௧௨.௫~பன்னிரண்டு புள்ளி ஐந்து +100.25~நூறு புள்ளி இரண்டு ஐந்து +3.14-அங்கு~மூன்று புள்ளி ஒன்று நான்கு அங்கு +.5~பூஜ்யம் புள்ளி ஐந்து +1.2.3~ஒன்று புள்ளி இரண்டு புள்ளி மூன்று +192.168.1.1~ஒன்று ஒன்பது இரண்டு டாட் ஒன்று ஆறு எட்டு டாட் ஒன்று டாட் ஒன்று +1 லட்சம்~ஒரு லட்சம் +1 கோடி~ஒரு கோடி +5 lakh~ஐந்து இலட்சம் +5 crore~ஐந்து கோடி +1.5 கோடி~ஒன்று புள்ளி ஐந்து கோடி +3ஆயிரம்~மூன்று ஆயிரம் +5 லட்சம் பேர்~ஐந்து லட்சம் பேர் ++0.0~பிளஸ் பூஜ்யம் புள்ளி பூஜ்யம் ++5.5~பிளஸ் ஐந்து புள்ளி ஐந்து ++5 லட்சம்~பிளஸ் ஐந்து லட்சம் +-5 லட்சம்~மைனஸ் ஐந்து லட்சம் +1,5.5~ஒன்று , ஐந்து புள்ளி ஐந்து +1,250.5~ஆயிரத்து இருநூற்று ஐம்பது புள்ளி ஐந்து diff --git a/tests/nemo_text_processing/ta/data_text_normalization/test_cases_electronic.txt b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_electronic.txt new file mode 100644 index 000000000..cdd319d22 --- /dev/null +++ b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_electronic.txt @@ -0,0 +1,29 @@ +kumar@gmail.com~kumar எட் gmail டாட் காம் +info@example.co.in~info எட் example டாட் கோ டாட் ஐ என் +support@example.com~support எட் example டாட் காம் +user@example.net~user எட் example டாட் நெட் +admin@example.edu~admin எட் example டாட் எடு +test@example.gov~test எட் example டாட் கவ் +a-b@example.org~ஏ டாஷ் பி எட் example டாட் ஆர்க் +x@example.in~எக்ஸ் எட் example டாட் ஐ என் +https://google.com~google டாட் காம் +http://example.com/page2~example டாட் காம் வெட்டுக்கோடு page இரண்டு +https://www.example.com/path/to/page~டபிள்யூ டபிள்யூ டபிள்யூ டாட் example டாட் காம் வெட்டுக்கோடு path வெட்டுக்கோடு to வெட்டுக்கோடு page +www.example.com~டபிள்யூ டபிள்யூ டபிள்யூ டாட் example டாட் காம் +www.google.co.in~டபிள்யூ டபிள்யூ டபிள்யூ டாட் google டாட் கோ டாட் ஐ என் +www.example.co~டபிள்யூ டபிள்யூ டபிள்யூ டாட் example டாட் கோ +example.com~example டாட் காம் +example.org~example டாட் ஆர்க் +example.net/a/b~example டாட் நெட் வெட்டுக்கோடு ஏ வெட்டுக்கோடு பி +example.com/page2~example டாட் காம் வெட்டுக்கோடு page இரண்டு +192.168.1.1~ஒன்று ஒன்பது இரண்டு டாட் ஒன்று ஆறு எட்டு டாட் ஒன்று டாட் ஒன்று +10.0.0.1~ஒன்று பூஜ்யம் டாட் பூஜ்யம் டாட் பூஜ்யம் டாட் ஒன்று +255.255.255.0~இரண்டு ஐந்து ஐந்து டாட் இரண்டு ஐந்து ஐந்து டாட் இரண்டு ஐந்து ஐந்து டாட் பூஜ்யம் +@handle~எட் handle +@user_name~எட் user அடிக்கோடு name +@a_b~எட் ஏ அடிக்கோடு பி +user_name~user_name +a_b~a_b +3.14~மூன்று புள்ளி ஒன்று நான்கு +5@6~ஐந்து எட் ஆறு +விவரங்களுக்கு info@example.com பார்க்கவும்.~விவரங்களுக்கு info எட் example டாட் காம் பார்க்கவும் . diff --git a/tests/nemo_text_processing/ta/data_text_normalization/test_cases_fraction.txt b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_fraction.txt new file mode 100644 index 000000000..44da391b1 --- /dev/null +++ b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_fraction.txt @@ -0,0 +1,32 @@ +3/4~முக்கால் +1/2~அரை +7/8~ஏழு கீழ் எட்டு +௩/௪~முக்கால் +½~அரை +¾~முக்கால் +2¾~இரண்டே முக்கால் +1/0~ஒன்று / பூஜ்யம் +3/45~மூன்று கீழ் நாற்பத்தைந்து +5/25~ஐந்து கீழ் இருபத்தைந்து +1/200~ஒன்று கீழ் இருநூறு +1/1000~ஒன்று கீழ் ஆயிரம் +-1/2~மைனஸ் அரை +15/06~பதினைந்து / பூஜ்யம் ஆறு +5/77~ஐந்து கீழ் எழுபத்தேழு +22/7~இருபத்திரண்டு கீழ் ஏழு +2/3~இரண்டு கீழ் மூன்று +20/100~இருபது கீழ் நூறு +1/4~கால் +¼~கால் +1 1/2~ஒன்றரை +2 3/4~இரண்டே முக்கால் +1 1/4~ஒன்றே கால் +3 3/4~மூன்றே முக்கால் +2 1/2~இரண்டரை +10 1/2~பத்தரை +25 1/2~இருபத்தைந்தரை +1¼~ஒன்றே கால் +1½~ஒன்றரை +-2 3/4~மைனஸ் இரண்டே முக்கால் +2 5/77~இரண்டு மற்றும் ஐந்து கீழ் எழுபத்தேழு +1000 1/2~ஆயிரம் மற்றும் அரை diff --git a/tests/nemo_text_processing/ta/data_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_measure.txt new file mode 100644 index 000000000..1729f1ae0 --- /dev/null +++ b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_measure.txt @@ -0,0 +1,33 @@ +5 கி.மீ. நடந்தேன்.~ஐந்து கிலோமீட்டர் நடந்தேன் . +12 kg~பன்னிரண்டு கிலோகிராம் +12.5kg~பன்னிரண்டு புள்ளி ஐந்து கிலோகிராம் +100 மி.லி.~நூறு மில்லிலிட்டர் +5 செ.மீ~ஐந்து சென்டிமீட்டர் +10 km~பத்து கிலோமீட்டர் +-40°C~மைனஸ் நாற்பது டிகிரி செல்சியஸ் +12 கி.கி. அரிசி~பன்னிரண்டு கிலோகிராம் அரிசி +3 லி.~மூன்று லிட்டர் +1 kg~ஒரு கிலோகிராம் +47A~47A +5t~ஐந்து டன் +5 t~ஐந்து டன் +5KG~ஐந்து கிலோகிராம் +5cm2~ஐந்து சதுர சென்டிமீட்டர் +100 சத.~நூறு சதவீதம் +60கி.மீ/மணி~அறுபது கிலோமீட்டர் ஒரு மணி நேரத்திற்கு +5-10 kg~ஐந்து முதல் பத்து கிலோகிராம் +2-3 கி.மீ.~இரண்டு முதல் மூன்று கிலோமீட்டர் +5G~ஐந்து ஜி +5 g~ஐந்து கிராம் +5 சத~ஐந்து சதவீதம் +5 லி~ஐந்து லிட்டர் +100மீ~நூறு மீட்டர் +1 மீ~ஒரு மீட்டர் +100 கிமீ~நூறு கிலோமீட்டர் +9 மிமீ~ஒன்பது மில்லிமீட்டர் +5 செமீ~ஐந்து சென்டிமீட்டர் +2 கிகி~இரண்டு கிலோகிராம் +250 மிலி~இருநூற்று ஐம்பது மில்லிலிட்டர் +170 C~நூற்று எழுபது செல்சியஸ் +38.5 C~முப்பத்தெட்டு புள்ளி ஐந்து செல்சியஸ் +23 °C~இருபத்துமூன்று டிகிரி செல்சியஸ் diff --git a/tests/nemo_text_processing/ta/data_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_money.txt new file mode 100644 index 000000000..bbcdcff6e --- /dev/null +++ b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_money.txt @@ -0,0 +1,50 @@ +₹50~ஐம்பது ரூபாய் +₹1,250.50~ஆயிரத்து இருநூற்று ஐம்பது ரூபாய் ஐம்பது பைசா +$100~நூறு டாலர் +₹0.75~எழுபத்தைந்து பைசா +Rs 500~ஐந்நூறு ரூபாய் +₹50.5~ஐம்பது ரூபாய் ஐம்பது பைசா +ரூ.100~நூறு ரூபாய் +₹50.05~ஐம்பது ரூபாய் ஐந்து பைசா +₹1~ஒரு ரூபாய் +₹101~நூற்று ஒன்று ரூபாய் +₹0.01~ஒரு பைசா +₹1.50~ஒரு ரூபாய் ஐம்பது பைசா +₹5 கோடி~ஐந்து கோடி ரூபாய் +₹2.5 லட்சம்~இரண்டு புள்ளி ஐந்து லட்சம் ரூபாய் +₹.50~ஐம்பது பைசா +₹1,999.00~ஆயிரத்து தொள்ளாயிரத்து தொண்ணூற்றொன்பது ரூபாய் +₹-500~மைனஸ் ஐந்நூறு ரூபாய் +-₹500~மைனஸ் ஐந்நூறு ரூபாய் +₹1 கோடி~ஒரு கோடி ரூபாய் +$5 மில்லியன்~ஐந்து மில்லியன் டாலர் +₹150க்கு~நூற்று ஐம்பது ரூபாய்க்கு +₹5-10~ஐந்து முதல் பத்து ரூபாய் +₹5-10 கோடி~ஐந்து முதல் பத்து கோடி ரூபாய் +₹2 lakh~இரண்டு இலட்சம் ரூபாய் +Rs 1.5 lakh~ஒன்று புள்ளி ஐந்து இலட்சம் ரூபாய் +$50 million~ஐம்பது மில்லியன் டாலர் +₹15L~பதினைந்து இலட்சம் ரூபாய் +₹5cr~ஐந்து கோடி ரூபாய் +$50M~ஐம்பது மில்லியன் டாலர் +₹5கோடி~ஐந்து கோடி ரூபாய் +₹1 லட்சம் கோடி~ஒரு லட்சம் கோடி ரூபாய் +₹1.5 கோடி~ஒன்று புள்ளி ஐந்து கோடி ரூபாய் +₹50.123~ஐம்பது புள்ளி ஒன்று இரண்டு மூன்று ரூபாய் +₹100.500~நூறு புள்ளி ஐந்து பூஜ்யம் பூஜ்யம் ரூபாய் +₹5 கோடிக்கு~ஐந்து கோடி ரூபாய்க்கு +₹50க்கும்~ஐம்பது ரூபாய்க்கும் +₹50ஆக~ஐம்பது ரூபாயாக +₹50ஆல்~ஐம்பது ரூபாயால் +₹50இல்~ஐம்பது ரூபாயில் +50/-~ஐம்பது ரூபாய் +₺.50~ஐம்பது குருஸ் +৳.50~ஐம்பது பைசா +₦.50~ஐம்பது கோபோ +¥.50~ஐம்பது சென் +₩.50~ஐம்பது ஜியோன் +₹5 கிலோ~ஐந்து ரூபாய் கிலோ +₹5%~ஐந்து ரூபாய் சதவீதம் +₹5 - ₹10~ஐந்து ரூபாய் முதல் பத்து ரூபாய் +₹5-₹10~ஐந்து ரூபாய் முதல் பத்து ரூபாய் +விலை ₹150.0 ஆகும்.~விலை நூற்று ஐம்பது ரூபாய் ஆகும் . diff --git a/tests/nemo_text_processing/ta/data_text_normalization/test_cases_ordinal.txt b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_ordinal.txt new file mode 100644 index 000000000..93c9682a9 --- /dev/null +++ b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_ordinal.txt @@ -0,0 +1,24 @@ +5வது~ஐந்தாவது +1வது~முதலாவது +2ஆவது~இரண்டாவது +10ஆம்~பத்தாம் +21வது~இருபத்தொன்றாவது +௫வது~ஐந்தாவது +100வது~நூறாவது +3-வது~மூன்றாவது +3ஆவதாக~மூன்றாவதாக +5வதுக்கு~ஐந்தாவதுக்கு +5ஆவதற்கு~ஐந்தாவதற்கு +5ஆவதில்~ஐந்தாவதில் +1st~1st +28ம்~இருபத்தெட்டாம் +7ம்~ஏழாம் +20ம்~இருபதாம் +1812ம்~ஆயிரத்து எண்ணூற்றுப்பன்னிரண்டாம் +2009ம்~இரண்டாயிரத்து ஒன்பதாம் +5ம் வகுப்பு~ஐந்தாம் வகுப்பு +1990ம் ஆண்டு~ஆயிரத்து தொள்ளாயிரத்து தொண்ணூறாம் ஆண்டு +01ஆம்~முதலாம் +09ம்~ஒன்பதாம் +04ம்~நான்காம் +007ஆம்~ஏழாம் diff --git a/tests/nemo_text_processing/ta/data_text_normalization/test_cases_range.txt b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_range.txt new file mode 100644 index 000000000..ccb65d72c --- /dev/null +++ b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_range.txt @@ -0,0 +1,13 @@ +10-20 வயது சிறுவர்கள்.~பத்து முதல் இருபது வயது சிறுவர்கள் . +2-3 நாட்கள்~இரண்டு முதல் மூன்று நாட்கள் +100-200~நூறு முதல் இருநூறு +10-20ல்~பத்து முதல் இருபதில் +10-20 லட்சம்~பத்து முதல் இருபது லட்சம் +5-10 கிலோ~ஐந்து முதல் பத்து கிலோ +2024-2025~இரண்டாயிரத்து இருபத்துநான்கு முதல் இரண்டாயிரத்து இருபத்தைந்து +1-2 நாட்கள்~ஒன்று முதல் இரண்டு நாட்கள் +50 - 100~ஐம்பது முதல் நூறு +10-15 நிமிடம்~பத்து முதல் பதினைந்து நிமிடம் +100-200 பேர்~நூறு முதல் இருநூறு பேர் +1,000-2,000~ஆயிரம் முதல் இரண்டாயிரம் +௧௦-௨௦~பத்து முதல் இருபது diff --git a/tests/nemo_text_processing/ta/data_text_normalization/test_cases_roman.txt b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_roman.txt new file mode 100644 index 000000000..21de26cdd --- /dev/null +++ b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_roman.txt @@ -0,0 +1,39 @@ +Class XII~Class பன்னிரண்டு +Chapter IV~Chapter நான்கு +Part II~Part இரண்டு +Volume V~Volume ஐந்து +Vol XXXIX~Vol முப்பத்தொன்பது +Section III~Section மூன்று +Grade VI~Grade ஆறு +Book I~Book ஒன்று +Act II~Act இரண்டு +Scene IV~Scene நான்கு +Round X~Round பத்து +Season II~Season இரண்டு +Type I~Type ஒன்று +Mark V~Mark ஐந்து +Level III~Level மூன்று +Phase II~Phase இரண்டு +Stage I~Stage ஒன்று +வகுப்பு XII~வகுப்பு பன்னிரண்டு +வகுப்பு I~வகுப்பு ஒன்று +வகுப்பு X~வகுப்பு பத்து +அத்தியாயம் VII~அத்தியாயம் ஏழு +XIV Class~பதினான்காம் Class +XII வகுப்பு~பன்னிரண்டாம் வகுப்பு +I வகுப்பு~முதலாம் வகுப்பு +IV வகுப்பு~நான்காம் வகுப்பு +XIIஆம்~பன்னிரண்டாம் +Iஆம்~முதலாம் +IVஆம் வகுப்பு~நான்காம் வகுப்பு +ராஜராஜன்-II~ராஜராஜன் இரண்டு +I~I +V~V +X~X +XII~XII +MIX~MIX +IIII~IIII +XL~XL +XLII~XLII +வகுப்பு XLII~வகுப்பு XLII +Class 12~Class பன்னிரண்டு diff --git a/tests/nemo_text_processing/ta/data_text_normalization/test_cases_serial.txt b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_serial.txt new file mode 100644 index 000000000..ef0ab0567 --- /dev/null +++ b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_serial.txt @@ -0,0 +1,39 @@ +5G~ஐந்து ஜி +4G~நான்கு ஜி +2G~இரண்டு ஜி +3D~மூன்று டி +A12~ஏ பன்னிரண்டு +B12~பி பன்னிரண்டு +X100~எக்ஸ் நூறு +A-1~ஏ ஒன்று +B-60~பி அறுபது +R2D2~ஆர் இரண்டு டி இரண்டு +H1N1~எச் ஒன்று என் ஒன்று +AB123CD~ஏ பி நூற்று இருபத்துமூன்று சி டி +6E204~ஆறு ஈ இருநூற்று நான்கு +47A~47A +170C~170C +COVID-19~COVID பத்தொன்பது +COVID19~COVID பத்தொன்பது +JEE-2024~ஜெ ஈ ஈ இரண்டாயிரத்து இருபத்துநான்கு +FY2024~எஃப் வை இரண்டாயிரத்து இருபத்துநான்கு +SBIN0001234~எஸ் பி ஐ என் பூஜ்யம் பூஜ்யம் பூஜ்யம் ஒன்று இரண்டு மூன்று நான்கு +HDFC0001234~எச் டி எஃப் சி பூஜ்யம் பூஜ்யம் பூஜ்யம் ஒன்று இரண்டு மூன்று நான்கு +IFSC SBIN0001234~ஐ எஃப் எஸ் சி எஸ் பி ஐ என் பூஜ்யம் பூஜ்யம் பூஜ்யம் ஒன்று இரண்டு மூன்று நான்கு +PAN ABCDE1234F~பி ஏ என் ABCDE ஆயிரத்து இருநூற்று முப்பத்துநான்கு எஃப் +ABCDE1234F~ABCDE ஆயிரத்து இருநூற்று முப்பத்துநான்கு எஃப் +PNR 4X7K9M~பி என் ஆர் நான்கு எக்ஸ் ஏழு கே ஒன்பது எம் +4X7K9M~நான்கு எக்ஸ் ஏழு கே ஒன்பது எம் +KA 01 AB 1234~கே ஏ பூஜ்யம் ஒன்று ஏ பி ஒன்று இரண்டு மூன்று நான்கு +TN-09-AB-1234~டி என் பூஜ்யம் ஒன்பது ஏ பி ஒன்று இரண்டு மூன்று நான்கு +MH12DE1433~எம் எச் ஒன்று இரண்டு டி ஈ ஒன்று நான்கு மூன்று மூன்று +1-800-555~ஒன்று எண்ணூறு ஐந்நூற்று ஐம்பத்தைந்து +1-2-3~ஒன்று இரண்டு மூன்று +2024~இரண்டாயிரத்து இருபத்துநான்கு +12345~பன்னிரண்டாயிரத்து முந்நூற்று நாற்பத்தைந்து +15-06-2024~பதினைந்து ஜூன் இரண்டாயிரத்து இருபத்துநான்கு +9876543210~ஒன்பது எட்டு ஏழு ஆறு ஐந்து நான்கு மூன்று இரண்டு ஒன்று பூஜ்யம் +1800-425-1234~ஒன்று எட்டு பூஜ்யம் பூஜ்யம் நான்கு இரண்டு ஐந்து ஒன்று இரண்டு மூன்று நான்கு +5x3~5x3 +10-20~பத்து முதல் இருபது +குறியீடு 5G மற்றும் KA 01 AB 1234 உள்ளது.~குறியீடு ஐந்து ஜி மற்றும் கே ஏ பூஜ்யம் ஒன்று ஏ பி ஒன்று இரண்டு மூன்று நான்கு உள்ளது . diff --git a/tests/nemo_text_processing/ta/data_text_normalization/test_cases_telephone.txt b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_telephone.txt new file mode 100644 index 000000000..053a457df --- /dev/null +++ b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_telephone.txt @@ -0,0 +1,20 @@ +9943206870~ஒன்பது ஒன்பது நான்கு மூன்று இரண்டு பூஜ்யம் ஆறு எட்டு ஏழு பூஜ்யம் ++91 9876543210~பிளஸ் ஒன்பது ஒன்று ஒன்பது எட்டு ஏழு ஆறு ஐந்து நான்கு மூன்று இரண்டு ஒன்று பூஜ்யம் +044-28230000~பூஜ்யம் நான்கு நான்கு இரண்டு எட்டு இரண்டு மூன்று பூஜ்யம் பூஜ்யம் பூஜ்யம் பூஜ்யம் ++91 என்பது குறியீடு.~பிளஸ் தொண்ணூற்றொன்று என்பது குறியீடு . ++91-98765-43210~பிளஸ் ஒன்பது ஒன்று ஒன்பது எட்டு ஏழு ஆறு ஐந்து நான்கு மூன்று இரண்டு ஒன்று பூஜ்யம் +98765 43210~ஒன்பது எட்டு ஏழு ஆறு ஐந்து நான்கு மூன்று இரண்டு ஒன்று பூஜ்யம் +044-2433 1234~பூஜ்யம் நான்கு நான்கு இரண்டு நான்கு மூன்று மூன்று ஒன்று இரண்டு மூன்று நான்கு +1800-425-1234~ஒன்று எட்டு பூஜ்யம் பூஜ்யம் நான்கு இரண்டு ஐந்து ஒன்று இரண்டு மூன்று நான்கு +044 2823 0000~பூஜ்யம் நான்கு நான்கு இரண்டு எட்டு இரண்டு மூன்று பூஜ்யம் பூஜ்யம் பூஜ்யம் பூஜ்யம் +(044) 2823 0000~பூஜ்யம் நான்கு நான்கு இரண்டு எட்டு இரண்டு மூன்று பூஜ்யம் பூஜ்யம் பூஜ்யம் பூஜ்யம் +044 28230000~பூஜ்யம் நான்கு நான்கு இரண்டு எட்டு இரண்டு மூன்று பூஜ்யம் பூஜ்யம் பூஜ்யம் பூஜ்யம் ++91-44-28230000~பிளஸ் ஒன்பது ஒன்று நான்கு நான்கு இரண்டு எட்டு இரண்டு மூன்று பூஜ்யம் பூஜ்யம் பூஜ்யம் பூஜ்யம் ++91 44 2823 0000~பிளஸ் ஒன்பது ஒன்று நான்கு நான்கு இரண்டு எட்டு இரண்டு மூன்று பூஜ்யம் பூஜ்யம் பூஜ்யம் பூஜ்யம் +1-800-425-1234~ஒன்று எட்டு பூஜ்யம் பூஜ்யம் நான்கு இரண்டு ஐந்து ஒன்று இரண்டு மூன்று நான்கு +9876543210க்கு~ஒன்பது எட்டு ஏழு ஆறு ஐந்து நான்கு மூன்று இரண்டு ஒன்று பூஜ்யத்துக்கு +9876543210ல்~ஒன்பது எட்டு ஏழு ஆறு ஐந்து நான்கு மூன்று இரண்டு ஒன்று பூஜ்யத்தில் ++919876543210ல்~பிளஸ் ஒன்பது ஒன்று ஒன்பது எட்டு ஏழு ஆறு ஐந்து நான்கு மூன்று இரண்டு ஒன்று பூஜ்யத்தில் +1800 11 4000~ஒன்று எட்டு பூஜ்யம் பூஜ்யம் ஒன்று ஒன்று நான்கு பூஜ்யம் பூஜ்யம் பூஜ்யம் +அலுவலக எண் (044) 2823-0000 ஆகும்.~அலுவலக எண் பூஜ்யம் நான்கு நான்கு இரண்டு எட்டு இரண்டு மூன்று பூஜ்யம் பூஜ்யம் பூஜ்யம் பூஜ்யம் ஆகும் . +9876543210ஆல்~ஒன்பது எட்டு ஏழு ஆறு ஐந்து நான்கு மூன்று இரண்டு ஒன்று பூஜ்யத்தால் diff --git a/tests/nemo_text_processing/ta/data_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_time.txt new file mode 100644 index 000000000..962b06fa9 --- /dev/null +++ b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_time.txt @@ -0,0 +1,36 @@ +10:30~பத்து மணி முப்பது நிமிடம் +1:00~ஒரு மணி +23:59~இருபத்துமூன்று மணி ஐம்பத்தொன்பது நிமிடம் +10:30:45~பத்து மணி முப்பது நிமிடம் நாற்பத்தைந்து வினாடி +09:05~ஒன்பது மணி ஐந்து நிமிடம் +௧௨:௩௦~பன்னிரண்டு மணி முப்பது நிமிடம் +12:75~பன்னிரண்டு : எழுபத்தைந்து +25:00~இருபத்தைந்து : பூஜ்யம் பூஜ்யம் +1:30~ஒரு மணி முப்பது நிமிடம் +மணி 3:30க்கு~மணி மூன்று மணி முப்பது நிமிடம் +10:30:00~பத்து மணி முப்பது நிமிடம் +10:00:30~பத்து மணி முப்பது வினாடி +10:00:00~பத்து மணி +10:30 AM~முற்பகல் பத்து மணி முப்பது நிமிடம் +12:00 PM~பிற்பகல் பன்னிரண்டு மணி +10:60~பத்து : அறுபது +10:30:60~பத்து மணி முப்பது நிமிடம் : அறுபது +7:00 மணி~ஏழு மணி +10:30 மணி~பத்து மணி முப்பது நிமிடம் +10:30-ல்~பத்து மணி முப்பது நிமிடம் +10.30 மணிக்கு~பத்து மணி முப்பது நிமிடம் +காலை 10.30~காலை பத்து மணி முப்பது நிமிடம் +மாலை 6.30 மணிக்கு~மாலை ஆறு மணி முப்பது நிமிடம் +மு.ப. 10.30~முற்பகல் பத்து மணி முப்பது நிமிடம் +10.30 பி.ப.~பிற்பகல் பத்து மணி முப்பது நிமிடம் +10.30~பத்து புள்ளி மூன்று பூஜ்யம் +24:00~இருபத்துநான்கு மணி +24:30~இருபத்துநான்கு : முப்பது +24:00:30~இருபத்துநான்கு மணி : முப்பது +23:00:30~இருபத்துமூன்று மணி முப்பது வினாடி +13:13:13~பதின்மூன்று மணி பதின்மூன்று நிமிடம் பதின்மூன்று வினாடி +1:13 PM~பிற்பகல் ஒரு மணி பதின்மூன்று நிமிடம் +7 AM~முற்பகல் ஏழு மணி +7pm~பிற்பகல் ஏழு மணி +கூட்டம் 10.30 AM ஆகும்.~கூட்டம் முற்பகல் பத்து மணி முப்பது நிமிடம் ஆகும் . +10:30 இல்~பத்து மணி முப்பது நிமிடம் diff --git a/tests/nemo_text_processing/ta/data_text_normalization/test_cases_whitelist.txt b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_whitelist.txt new file mode 100644 index 000000000..a836065b9 --- /dev/null +++ b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_whitelist.txt @@ -0,0 +1,19 @@ +டாக்டர்.~டாக்டர் . +5×3=15 ஆகும்.~ஐந்து பெருக்கல் மூன்று சமம் பதினைந்து ஆகும் . +5%~ஐந்து சதவீதம் +10÷2~பத்து வகுத்தல் இரண்டு +(1.5%)~( ஒன்று புள்ளி ஐந்து சதவீதம் ) +[5]~[ ஐந்து ] +5%%~ஐந்து சதவீதம் சதவீதம் +10%க்கு~பத்து சதவீதத்துக்கு +→~வலது அம்பு +5 → 10~ஐந்து வலது அம்பு பத்து +™~வர்த்தகச் சின்னம் +10%ஆக~பத்து சதவீதமாக +5%இல்~ஐந்து சதவீதத்தில் +5%க்கும்~ஐந்து சதவீதத்துக்கும் +@ரவி~எட் ரவி +(@)~( எட் ) +ரவி_குமார்~ரவி அடிக்கோடு குமார் +user@example.com~user எட் example டாட் காம் +a_b~a_b diff --git a/tests/nemo_text_processing/ta/data_text_normalization/test_cases_word.txt b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_word.txt new file mode 100644 index 000000000..6645fea58 --- /dev/null +++ b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_word.txt @@ -0,0 +1,15 @@ +வணக்கம்~வணக்கம் +hello world~hello world +அவன் வீட்டுக்கு சென்றான்.~அவன் வீட்டுக்கு சென்றான் . +www.google.com~டபிள்யூ டபிள்யூ டபிள்யூ டாட் google டாட் காம் +https://example.com/path~example டாட் காம் வெட்டுக்கோடு path +ஜி20~ஜி இருபது +5.மணி~ஐந்து . மணி +நான் சென்னைக்கு சென்றேன்~நான் சென்னைக்கு சென்றேன் +இது ஒரு சோதனை.~இது ஒரு சோதனை . +வணக்கம், நண்பரே!~வணக்கம் , நண்பரே ! +தமிழ் மொழி~தமிழ் மொழி +ஆம்~ஆம் +இல்லை?~இல்லை ? +OK~OK +நல்லது; சரி~நல்லது ; சரி diff --git a/tests/nemo_text_processing/ta/test_cardinal.py b/tests/nemo_text_processing/ta/test_cardinal.py new file mode 100644 index 000000000..184a793d8 --- /dev/null +++ b/tests/nemo_text_processing/ta/test_cardinal.py @@ -0,0 +1,42 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestCardinal: + normalizer = Normalizer( + input_case='cased', lang='ta', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + inverse_normalizer = InverseNormalizer(lang='ta', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ta/data_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() + + @parameterized.expand(parse_test_case_file('ta/data_inverse_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/ta/test_date.py b/tests/nemo_text_processing/ta/test_date.py new file mode 100644 index 000000000..8a38a8099 --- /dev/null +++ b/tests/nemo_text_processing/ta/test_date.py @@ -0,0 +1,42 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestDate: + normalizer = Normalizer( + input_case='cased', lang='ta', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + inverse_normalizer = InverseNormalizer(lang='ta', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ta/data_text_normalization/test_cases_date.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() + + @parameterized.expand(parse_test_case_file('ta/data_inverse_text_normalization/test_cases_date.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/ta/test_decimal.py b/tests/nemo_text_processing/ta/test_decimal.py new file mode 100644 index 000000000..1e808d901 --- /dev/null +++ b/tests/nemo_text_processing/ta/test_decimal.py @@ -0,0 +1,42 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestDecimal: + normalizer = Normalizer( + input_case='cased', lang='ta', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + inverse_normalizer = InverseNormalizer(lang='ta', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ta/data_text_normalization/test_cases_decimal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() + + @parameterized.expand(parse_test_case_file('ta/data_inverse_text_normalization/test_cases_decimal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/ta/test_electronic.py b/tests/nemo_text_processing/ta/test_electronic.py new file mode 100644 index 000000000..457ec032c --- /dev/null +++ b/tests/nemo_text_processing/ta/test_electronic.py @@ -0,0 +1,33 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestElectronic: + normalizer = Normalizer( + input_case='cased', lang='ta', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + + @parameterized.expand(parse_test_case_file('ta/data_text_normalization/test_cases_electronic.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() diff --git a/tests/nemo_text_processing/ta/test_fraction.py b/tests/nemo_text_processing/ta/test_fraction.py new file mode 100644 index 000000000..6c9a3cd50 --- /dev/null +++ b/tests/nemo_text_processing/ta/test_fraction.py @@ -0,0 +1,42 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestFraction: + normalizer = Normalizer( + input_case='cased', lang='ta', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + inverse_normalizer = InverseNormalizer(lang='ta', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ta/data_text_normalization/test_cases_fraction.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() + + @parameterized.expand(parse_test_case_file('ta/data_inverse_text_normalization/test_cases_fraction.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/ta/test_measure.py b/tests/nemo_text_processing/ta/test_measure.py new file mode 100644 index 000000000..fe9bc49ec --- /dev/null +++ b/tests/nemo_text_processing/ta/test_measure.py @@ -0,0 +1,42 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestMeasure: + normalizer = Normalizer( + input_case='cased', lang='ta', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + inverse_normalizer = InverseNormalizer(lang='ta', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ta/data_text_normalization/test_cases_measure.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() + + @parameterized.expand(parse_test_case_file('ta/data_inverse_text_normalization/test_cases_measure.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/ta/test_money.py b/tests/nemo_text_processing/ta/test_money.py new file mode 100644 index 000000000..912e86e39 --- /dev/null +++ b/tests/nemo_text_processing/ta/test_money.py @@ -0,0 +1,42 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestMoney: + normalizer = Normalizer( + input_case='cased', lang='ta', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + inverse_normalizer = InverseNormalizer(lang='ta', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ta/data_text_normalization/test_cases_money.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() + + @parameterized.expand(parse_test_case_file('ta/data_inverse_text_normalization/test_cases_money.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/ta/test_ordinal.py b/tests/nemo_text_processing/ta/test_ordinal.py new file mode 100644 index 000000000..2a9397b21 --- /dev/null +++ b/tests/nemo_text_processing/ta/test_ordinal.py @@ -0,0 +1,42 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestOrdinal: + normalizer = Normalizer( + input_case='cased', lang='ta', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + inverse_normalizer = InverseNormalizer(lang='ta', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ta/data_text_normalization/test_cases_ordinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() + + @parameterized.expand(parse_test_case_file('ta/data_inverse_text_normalization/test_cases_ordinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/ta/test_range.py b/tests/nemo_text_processing/ta/test_range.py new file mode 100644 index 000000000..9cbbc3090 --- /dev/null +++ b/tests/nemo_text_processing/ta/test_range.py @@ -0,0 +1,33 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestRange: + normalizer = Normalizer( + input_case='cased', lang='ta', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + + @parameterized.expand(parse_test_case_file('ta/data_text_normalization/test_cases_range.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() diff --git a/tests/nemo_text_processing/ta/test_roman.py b/tests/nemo_text_processing/ta/test_roman.py new file mode 100644 index 000000000..9070b0a93 --- /dev/null +++ b/tests/nemo_text_processing/ta/test_roman.py @@ -0,0 +1,33 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestRoman: + normalizer = Normalizer( + input_case='cased', lang='ta', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + + @parameterized.expand(parse_test_case_file('ta/data_text_normalization/test_cases_roman.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() diff --git a/tests/nemo_text_processing/ta/test_serial.py b/tests/nemo_text_processing/ta/test_serial.py new file mode 100644 index 000000000..255a6fe7f --- /dev/null +++ b/tests/nemo_text_processing/ta/test_serial.py @@ -0,0 +1,33 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestSerial: + normalizer = Normalizer( + input_case='cased', lang='ta', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + + @parameterized.expand(parse_test_case_file('ta/data_text_normalization/test_cases_serial.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() diff --git a/tests/nemo_text_processing/ta/test_sparrowhawk_inverse_text_normalization.sh b/tests/nemo_text_processing/ta/test_sparrowhawk_inverse_text_normalization.sh new file mode 100644 index 000000000..cdafae251 --- /dev/null +++ b/tests/nemo_text_processing/ta/test_sparrowhawk_inverse_text_normalization.sh @@ -0,0 +1,79 @@ +#! /bin/sh + +PROJECT_DIR=/workspace/tests + +runtest () { + input=$1 + cd /workspace/sparrowhawk/documentation/grammars + + # read test file + while IFS= read -r testcase; do + IFS='~' read -r spoken written <<< "$testcase" + denorm_pred=$(echo "$spoken" | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1) + + # trim white space + written="$(echo -e "${written}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + # input expected actual + assertEquals "$spoken" "$written" "$denorm_pred" + done < "$input" +} + +testITNCardinal() { + input=$PROJECT_DIR/ta/data_inverse_text_normalization/test_cases_cardinal.txt + runtest $input +} + +testITNDate() { + input=$PROJECT_DIR/ta/data_inverse_text_normalization/test_cases_date.txt + runtest $input +} + +testITNDecimal() { + input=$PROJECT_DIR/ta/data_inverse_text_normalization/test_cases_decimal.txt + runtest $input +} + +testITNFraction() { + input=$PROJECT_DIR/ta/data_inverse_text_normalization/test_cases_fraction.txt + runtest $input +} + +testITNMeasure() { + input=$PROJECT_DIR/ta/data_inverse_text_normalization/test_cases_measure.txt + runtest $input +} + +testITNMoney() { + input=$PROJECT_DIR/ta/data_inverse_text_normalization/test_cases_money.txt + runtest $input +} + +testITNOrdinal() { + input=$PROJECT_DIR/ta/data_inverse_text_normalization/test_cases_ordinal.txt + runtest $input +} + +testITNTelephone() { + input=$PROJECT_DIR/ta/data_inverse_text_normalization/test_cases_telephone.txt + runtest $input +} + +testITNTime() { + input=$PROJECT_DIR/ta/data_inverse_text_normalization/test_cases_time.txt + runtest $input +} + +testITNWhitelist() { + input=$PROJECT_DIR/ta/data_inverse_text_normalization/test_cases_whitelist.txt + runtest $input +} + +testITNWord() { + input=$PROJECT_DIR/ta/data_inverse_text_normalization/test_cases_word.txt + runtest $input +} + +# Load shUnit2 +. $PROJECT_DIR/../shunit2/shunit2 diff --git a/tests/nemo_text_processing/ta/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/ta/test_sparrowhawk_normalization.sh new file mode 100644 index 000000000..b2647c2b7 --- /dev/null +++ b/tests/nemo_text_processing/ta/test_sparrowhawk_normalization.sh @@ -0,0 +1,102 @@ +#! /bin/sh + +PROJECT_DIR=/workspace/tests + +runtest () { + input=$1 + cd /workspace/sparrowhawk/documentation/grammars + + # read test file + while IFS= read -r testcase; do + IFS='~' read -r written spoken <<< "$testcase" + + # Escape backslashes and replace non breaking space with breaking space + escaped_written=$(printf '%s' "$written" | sed 's/\\/\\\\/g') + denorm_pred=$(echo "$escaped_written" | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1 | sed 's/\xC2\xA0/ /g') + + # trim white space + spoken="$(echo -e "${spoken}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + # input expected actual + assertEquals "$written" "$spoken" "$denorm_pred" + done < "$input" +} + +testTNCardinal() { + input=$PROJECT_DIR/ta/data_text_normalization/test_cases_cardinal.txt + runtest $input +} + +testTNDate() { + input=$PROJECT_DIR/ta/data_text_normalization/test_cases_date.txt + runtest $input +} + +testTNDecimal() { + input=$PROJECT_DIR/ta/data_text_normalization/test_cases_decimal.txt + runtest $input +} + +testTNElectronic() { + input=$PROJECT_DIR/ta/data_text_normalization/test_cases_electronic.txt + runtest $input +} + +testTNFraction() { + input=$PROJECT_DIR/ta/data_text_normalization/test_cases_fraction.txt + runtest $input +} + +testTNMeasure() { + input=$PROJECT_DIR/ta/data_text_normalization/test_cases_measure.txt + runtest $input +} + +testTNMoney() { + input=$PROJECT_DIR/ta/data_text_normalization/test_cases_money.txt + runtest $input +} + +testTNOrdinal() { + input=$PROJECT_DIR/ta/data_text_normalization/test_cases_ordinal.txt + runtest $input +} + +testTNRange() { + input=$PROJECT_DIR/ta/data_text_normalization/test_cases_range.txt + runtest $input +} + +testTNRoman() { + input=$PROJECT_DIR/ta/data_text_normalization/test_cases_roman.txt + runtest $input +} + +testTNSerial() { + input=$PROJECT_DIR/ta/data_text_normalization/test_cases_serial.txt + runtest $input +} + +testTNTelephone() { + input=$PROJECT_DIR/ta/data_text_normalization/test_cases_telephone.txt + runtest $input +} + +testTNTime() { + input=$PROJECT_DIR/ta/data_text_normalization/test_cases_time.txt + runtest $input +} + +testTNWhitelist() { + input=$PROJECT_DIR/ta/data_text_normalization/test_cases_whitelist.txt + runtest $input +} + +testTNWord() { + input=$PROJECT_DIR/ta/data_text_normalization/test_cases_word.txt + runtest $input +} + +# Load shUnit2 +. $PROJECT_DIR/../shunit2/shunit2 diff --git a/tests/nemo_text_processing/ta/test_telephone.py b/tests/nemo_text_processing/ta/test_telephone.py new file mode 100644 index 000000000..67d7899ac --- /dev/null +++ b/tests/nemo_text_processing/ta/test_telephone.py @@ -0,0 +1,42 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestTelephone: + normalizer = Normalizer( + input_case='cased', lang='ta', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + inverse_normalizer = InverseNormalizer(lang='ta', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ta/data_text_normalization/test_cases_telephone.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() + + @parameterized.expand(parse_test_case_file('ta/data_inverse_text_normalization/test_cases_telephone.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/ta/test_time.py b/tests/nemo_text_processing/ta/test_time.py new file mode 100644 index 000000000..8cd708efa --- /dev/null +++ b/tests/nemo_text_processing/ta/test_time.py @@ -0,0 +1,42 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestTime: + normalizer = Normalizer( + input_case='cased', lang='ta', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + inverse_normalizer = InverseNormalizer(lang='ta', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ta/data_text_normalization/test_cases_time.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() + + @parameterized.expand(parse_test_case_file('ta/data_inverse_text_normalization/test_cases_time.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/ta/test_whitelist.py b/tests/nemo_text_processing/ta/test_whitelist.py new file mode 100644 index 000000000..51fb67075 --- /dev/null +++ b/tests/nemo_text_processing/ta/test_whitelist.py @@ -0,0 +1,42 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestWhitelist: + normalizer = Normalizer( + input_case='cased', lang='ta', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + inverse_normalizer = InverseNormalizer(lang='ta', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ta/data_text_normalization/test_cases_whitelist.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() + + @parameterized.expand(parse_test_case_file('ta/data_inverse_text_normalization/test_cases_whitelist.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/ta/test_word.py b/tests/nemo_text_processing/ta/test_word.py new file mode 100644 index 000000000..e2f73bbfe --- /dev/null +++ b/tests/nemo_text_processing/ta/test_word.py @@ -0,0 +1,42 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestWord: + normalizer = Normalizer( + input_case='cased', lang='ta', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + inverse_normalizer = InverseNormalizer(lang='ta', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ta/data_text_normalization/test_cases_word.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() + + @parameterized.expand(parse_test_case_file('ta/data_inverse_text_normalization/test_cases_word.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tools/text_processing_deployment/export_grammars.sh b/tools/text_processing_deployment/export_grammars.sh index 017472ae9..f41766dc8 100644 --- a/tools/text_processing_deployment/export_grammars.sh +++ b/tools/text_processing_deployment/export_grammars.sh @@ -32,7 +32,7 @@ GRAMMARS="itn_grammars" # tn_grammars INPUT_CASE="lower_cased" # cased -LANGUAGE="en" # language, {'en', 'es', 'de','zh'} supports both TN and ITN, {'pt', 'ru', 'fr', 'vi', 'mr'} supports ITN only +LANGUAGE="en" # language, {'en', 'es', 'de','zh', 'ta'} supports both TN and ITN, {'pt', 'ru', 'fr', 'vi', 'mr'} supports ITN only MODE="export" # default is one of {'export', 'interactive', 'test', 'ci'}. Default "export" OVERWRITE_CACHE="True" # Set to False to re-use .far files FORCE_REBUILD="False" # Set to True to re-build docker file diff --git a/tools/text_processing_deployment/pynini_export.py b/tools/text_processing_deployment/pynini_export.py index 73a4fc138..20fff505e 100644 --- a/tools/text_processing_deployment/pynini_export.py +++ b/tools/text_processing_deployment/pynini_export.py @@ -109,6 +109,7 @@ def parse_args(): 'ja', 'rw', 'ko', + 'ta', ], type=str, default='en', @@ -352,6 +353,17 @@ def parse_args(): ClassifyFst as TNClassifyFst, ) from nemo_text_processing.text_normalization.ko.verbalizers.verbalize import VerbalizeFst as TNVerbalizeFst + elif args.language == 'ta': + from nemo_text_processing.inverse_text_normalization.ta.taggers.tokenize_and_classify import ( + ClassifyFst as ITNClassifyFst, + ) + from nemo_text_processing.inverse_text_normalization.ta.verbalizers.verbalize import ( + VerbalizeFst as ITNVerbalizeFst, + ) + from nemo_text_processing.text_normalization.ta.taggers.tokenize_and_classify import ( + ClassifyFst as TNClassifyFst, + ) + from nemo_text_processing.text_normalization.ta.verbalizers.verbalize import VerbalizeFst as TNVerbalizeFst else: raise KeyError(f"Language {args.language} is not defined for export.") output_dir = os.path.join(args.output_dir, f"{args.language}_{args.grammars}_{args.input_case}")