diff --git a/nemo_text_processing/text_normalization/kn/data/ordinals/__init__.py b/nemo_text_processing/text_normalization/kn/data/ordinals/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/kn/data/ordinals/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/kn/data/ordinals/ending.tsv b/nemo_text_processing/text_normalization/kn/data/ordinals/ending.tsv new file mode 100644 index 000000000..13b3ccb88 --- /dev/null +++ b/nemo_text_processing/text_normalization/kn/data/ordinals/ending.tsv @@ -0,0 +1,2 @@ +ು +ಿ \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/kn/data/ordinals/exceptions.tsv b/nemo_text_processing/text_normalization/kn/data/ordinals/exceptions.tsv new file mode 100644 index 000000000..4bba22e10 --- /dev/null +++ b/nemo_text_processing/text_normalization/kn/data/ordinals/exceptions.tsv @@ -0,0 +1,4 @@ +1ನೇ ಮೊದಲನೆಯ +೧ನೇ ಮೊದಲನೆಯ +೦ನೇ ಸೊನ್ನೆಯ +0ನೇ ಸೊನ್ನೆಯ \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/kn/data/ordinals/kn_suffixes.tsv b/nemo_text_processing/text_normalization/kn/data/ordinals/kn_suffixes.tsv new file mode 100644 index 000000000..db35b27c8 --- /dev/null +++ b/nemo_text_processing/text_normalization/kn/data/ordinals/kn_suffixes.tsv @@ -0,0 +1,2 @@ +ನೆ ನೆಯ +ನೇ ನೆಯ \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/kn/taggers/ordinal.py b/nemo_text_processing/text_normalization/kn/taggers/ordinal.py new file mode 100644 index 000000000..d4fea1daa --- /dev/null +++ b/nemo_text_processing/text_normalization/kn/taggers/ordinal.py @@ -0,0 +1,53 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.kn.graph_utils import NEMO_SIGMA, GraphFst +from nemo_text_processing.text_normalization.kn.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.kn.utils import get_abs_path + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying Kannada ordinals, e.g. + ೧೦ನೇ -> ordinal { integer: "ಹತ್ತನೆಯ" } + 12ನೇ -> ordinal { integer: "ಹನ್ನೆರಡನೆಯ" } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: CardinalFst, deterministic: bool = True): + super().__init__(name="ordinal", kind="classify", deterministic=deterministic) + + exceptions = pynini.string_file(get_abs_path("data/ordinals/exceptions.tsv")) + endings = pynini.string_file(get_abs_path("data/ordinals/ending.tsv")) + kn_suffixes = pynini.string_file(get_abs_path("data/ordinals/kn_suffixes.tsv")) + + drop_cardinal_ending = pynini.cdrewrite(pynutil.delete(endings), "", "[EOS]", NEMO_SIGMA).optimize() + + kn_ordinal_graph = cardinal.final_graph @ drop_cardinal_ending + kn_suffixes + + exception_inputs = pynini.project(exceptions, "input").optimize() + ordinal_input = pynini.project(kn_ordinal_graph, "input").optimize() + ordinal_inputs = pynini.difference(ordinal_input, exception_inputs).optimize() + + ordinal_graph = (ordinal_inputs @ kn_ordinal_graph).optimize() + + graph = pynini.union(exceptions, ordinal_graph).optimize() + + final_graph = pynutil.insert('integer: "') + graph + pynutil.insert('"') + self.fst = self.add_tokens(final_graph).optimize() diff --git a/nemo_text_processing/text_normalization/kn/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/kn/taggers/tokenize_and_classify.py index fd279b6bb..b04e90424 100644 --- a/nemo_text_processing/text_normalization/kn/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/text_normalization/kn/taggers/tokenize_and_classify.py @@ -27,6 +27,7 @@ generator_main, ) from nemo_text_processing.text_normalization.kn.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.kn.taggers.ordinal import OrdinalFst from nemo_text_processing.text_normalization.kn.taggers.punctuation import PunctuationFst from nemo_text_processing.text_normalization.kn.taggers.word import WordFst @@ -75,10 +76,13 @@ def __init__( cardinal = CardinalFst(deterministic=deterministic) cardinal_graph = cardinal.fst + ordinal = OrdinalFst(cardinal=cardinal, deterministic=deterministic) + ordinal_graph = ordinal.fst + punctuation = PunctuationFst(deterministic=deterministic) punct_graph = punctuation.fst - classify = pynutil.add_weight(cardinal_graph, 1.1) + classify = pynutil.add_weight(cardinal_graph, 1.1) | pynutil.add_weight(ordinal_graph, 1.1) word_graph = WordFst().fst diff --git a/nemo_text_processing/text_normalization/kn/verbalizers/ordinal.py b/nemo_text_processing/text_normalization/kn/verbalizers/ordinal.py new file mode 100644 index 000000000..0d07ee91f --- /dev/null +++ b/nemo_text_processing/text_normalization/kn/verbalizers/ordinal.py @@ -0,0 +1,38 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.kn.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for verbalizing Kannada ordinals, e.g. + ordinal { integer: "ಮೊದಲನೆಯ" } -> ಮೊದಲನೆಯ + ordinal { integer: "ಹತ್ತನೆಯ" } -> ಹತ್ತನೆಯ + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic) + + integer_value = delete_space + pynutil.delete("\"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + graph = pynutil.delete("integer:") + integer_value + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/kn/verbalizers/verbalize.py b/nemo_text_processing/text_normalization/kn/verbalizers/verbalize.py index 2f68662c4..ee3c7f367 100644 --- a/nemo_text_processing/text_normalization/kn/verbalizers/verbalize.py +++ b/nemo_text_processing/text_normalization/kn/verbalizers/verbalize.py @@ -14,6 +14,7 @@ from nemo_text_processing.text_normalization.kn.graph_utils import GraphFst from nemo_text_processing.text_normalization.kn.verbalizers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.kn.verbalizers.ordinal import OrdinalFst class VerbalizeFst(GraphFst): @@ -33,6 +34,9 @@ def __init__(self, deterministic: bool = True): cardinal = CardinalFst(deterministic=deterministic) cardinal_graph = cardinal.fst - graph = cardinal_graph + ordinal = OrdinalFst(deterministic=deterministic) + ordinal_graph = ordinal.fst + + graph = cardinal_graph | ordinal_graph self.fst = graph diff --git a/tests/nemo_text_processing/kn/data_text_normalization/test_cases_ordinal.txt b/tests/nemo_text_processing/kn/data_text_normalization/test_cases_ordinal.txt new file mode 100644 index 000000000..1fb2ace13 --- /dev/null +++ b/tests/nemo_text_processing/kn/data_text_normalization/test_cases_ordinal.txt @@ -0,0 +1,71 @@ +೧ನೇ~ಮೊದಲನೆಯ +೨ನೇ~ಎರಡನೆಯ +2ನೇ~ಎರಡನೆಯ +೩ನೇ~ಮೂರನೆಯ +೪ನೇ~ನಾಲ್ಕನೆಯ +೫ನೇ~ಐದನೆಯ +5ನೇ~ಐದನೆಯ +೬ನೇ~ಆರನೆಯ +೭ನೇ~ಏಳನೆಯ +7ನೇ~ಏಳನೆಯ +೮ನೇ~ಎಂಟನೆಯ +೯ನೇ~ಒಂಬತ್ತನೆಯ +೧೦ನೇ~ಹತ್ತನೆಯ +೧೨ನೇ~ಹನ್ನೆರಡನೆಯ +12ನೇ~ಹನ್ನೆರಡನೆಯ +೧೪ನೇ~ಹದಿನಾಲ್ಕನೆಯ +15ನೇ~ಹದಿನೈದನೆಯ +೧೬ನೇ~ಹದಿನಾರನೆಯ +೧೭ನೇ~ಹದಿನೇಳನೆಯ +೧೮ನೇ~ಹದಿನೆಂಟನೆಯ +೧೯ನೇ~ಹತ್ತೊಂಬತ್ತನೆಯ +19ನೇ~ಹತ್ತೊಂಬತ್ತನೆಯ +೨೦ನೇ~ಇಪ್ಪತ್ತನೆಯ +೨೧ನೇ~ಇಪ್ಪತ್ತೊಂದನೆಯ +೨೫ನೇ~ಇಪ್ಪತ್ತೈದನೆಯ +೨೭ನೇ~ಇಪ್ಪತ್ತೇಳನೆಯ +೩೦ನೇ~ಮೂವತ್ತನೆಯ +೩೩ನೇ~ಮೂವತ್ತಮೂರನೆಯ +೪೦ನೇ~ನಲವತ್ತನೆಯ +೪೫ನೇ~ನಲವತ್ತೈದನೆಯ +೫೦ನೇ~ಐವತ್ತನೆಯ +೫೬ನೇ~ಐವತ್ತಾರನೆಯ +೬೦ನೇ~ಅರವತ್ತನೆಯ +೬೭ನೇ~ಅರವತ್ತೇಳನೆಯ +67ನೇ~ಅರವತ್ತೇಳನೆಯ +೭೫ನೇ~ಎಪ್ಪತ್ತೈದನೆಯ +೮೦ನೇ~ಎಂಬತ್ತನೆಯ +೮೮ನೇ~ಎಂಬತ್ತೆಂಟನೆಯ +೯೧ನೇ~ತೊಂಬತ್ತೊಂದನೆಯ +೯೯ನೇ~ತೊಂಬತ್ತೊಂಬತ್ತನೆಯ +೧೦೦ನೇ~ನೂರನೆಯ +100899823000ನೇ~ಹತ್ತು ಸಾವಿರದ ಎಂಬತ್ತೊಂಬತ್ತು ಕೋಟಿಯ ತೊಂಬತ್ತೆಂಟು ಲಕ್ಷದ ಇಪ್ಪತ್ತಮೂರು ಸಾವಿರನೆಯ +೧೦೧ನೇ~ನೂರ ಒಂದನೆಯ +೧೧೧ನೇ~ನೂರ ಹನ್ನೊಂದನೆಯ +೧೨೫ನೇ~ನೂರ ಇಪ್ಪತ್ತೈದನೆಯ +೧೫೩ನೇ~ನೂರ ಐವತ್ತಮೂರನೆಯ +೨೦೦ನೇ~ಇನ್ನೂರನೆಯ +೨೧೯ನೇ~ಇನ್ನೂರ ಹತ್ತೊಂಬತ್ತನೆಯ +೨೪೦ನೇ~ಇನ್ನೂರ ನಲವತ್ತನೆಯ +೩೨೯ನೇ~ಮುನ್ನೂರ ಇಪ್ಪತ್ತೊಂಬತ್ತನೆಯ +೩೬೫ನೇ~ಮುನ್ನೂರ ಅರವತ್ತೈದನೆಯ +೪೫೫ನೇ~ನಾನೂರ ಐವತ್ತೈದನೆಯ +೫೫೫ನೇ~ಐನೂರ ಐವತ್ತೈದನೆಯ +೬೪೦ನೇ~ಆರುನೂರ ನಲವತ್ತನೆಯ +೮೯೦ನೇ~ಎಂಟುನೂರ ತೊಂಬತ್ತನೆಯ +೧೦೦೧ನೇ~ಒಂದು ಸಾವಿರದ ಒಂದನೆಯ +೧೦೯೧ನೇ~ಒಂದು ಸಾವಿರದ ತೊಂಬತ್ತೊಂದನೆಯ +೧೭೮೨ನೆ~ಒಂದು ಸಾವಿರದ ಏಳುನೂರ ಎಂಬತ್ತೆರಡನೆಯ +೧೮೯೦ನೇ~ಒಂದು ಸಾವಿರದ ಎಂಟುನೂರ ತೊಂಬತ್ತನೆಯ +೧೯೮೧ನೇ~ಒಂದು ಸಾವಿರದ ಒಂಬೈನೂರ ಎಂಬತ್ತೊಂದನೆಯ +೯೮೨೬ನೇ~ಒಂಬತ್ತು ಸಾವಿರದ ಎಂಟುನೂರ ಇಪ್ಪತ್ತಾರನೆಯ +6789876ನೆ~ಅರವತ್ತೇಳು ಲಕ್ಷದ ಎಂಬತ್ತೊಂಬತ್ತು ಸಾವಿರದ ಎಂಟುನೂರ ಎಪ್ಪತ್ತಾರನೆಯ +10000000000000ನೆ~ಹತ್ತು ಲಕ್ಷ ಕೋಟನೆಯ +10000001ನೇ~ಒಂದು ಕೋಟಿಯ ಒಂದನೆಯ +1000ನೇ~ಒಂದು ಸಾವಿರನೆಯ +೭೦೦೦೦೦ನೇ~ಏಳು ಲಕ್ಷನೆಯ +ಈ ಮಗುವಿನ ಶೈಕ್ಷಣಿಕ ಫಲಿತಾಂಶವು ಯಾವಾಗಲೂ ಇಡೀ ತರಗತಿಯಲ್ಲಿ ೧ನೇ ಸ್ಥಾನದಲ್ಲಿದೆ.~ಈ ಮಗುವಿನ ಶೈಕ್ಷಣಿಕ ಫಲಿತಾಂಶವು ಯಾವಾಗಲೂ ಇಡೀ ತರಗತಿಯಲ್ಲಿ ಮೊದಲನೆಯ ಸ್ಥಾನದಲ್ಲಿದೆ. +ಈ ಸಾಲಿನಿಂದ ಕೆಳಕ್ಕೆ ಎಣಿಸಿದಾಗ 5ನೇ ರಾಘವೇಂದ್ರ.~ಈ ಸಾಲಿನಿಂದ ಕೆಳಕ್ಕೆ ಎಣಿಸಿದಾಗ ಐದನೆಯ ರಾಘವೇಂದ್ರ. +ಅಭಿನಂದನೆಗಳು! ನೀವು ನಮ್ಮ ಅಂಗಡಿಯ ೧೦೧ನೇ ಗ್ರಾಹಕರಾಗಿದ್ದೀರಿ.~ಅಭಿನಂದನೆಗಳು! ನೀವು ನಮ್ಮ ಅಂಗಡಿಯ ನೂರ ಒಂದನೆಯ ಗ್ರಾಹಕರಾಗಿದ್ದೀರಿ. +ಈ ಪಟ್ಟಿಯ ಆರಂಭದಿಂದ ೧೦೦ನೇ ವ್ಯಕ್ತಿಯವರೆಗೆ ಇರುವ ಎಲ್ಲರೂ ನಿಮ್ಮ ಗುರಿ ಗ್ರಾಹಕರು.~ಈ ಪಟ್ಟಿಯ ಆರಂಭದಿಂದ ನೂರನೆಯ ವ್ಯಕ್ತಿಯವರೆಗೆ ಇರುವ ಎಲ್ಲರೂ ನಿಮ್ಮ ಗುರಿ ಗ್ರಾಹಕರು. +0ನೇ~ಸೊನ್ನೆಯ \ No newline at end of file diff --git a/tests/nemo_text_processing/kn/test_ordinal.py b/tests/nemo_text_processing/kn/test_ordinal.py new file mode 100644 index 000000000..a4fb9ba8f --- /dev/null +++ b/tests/nemo_text_processing/kn/test_ordinal.py @@ -0,0 +1,33 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestOrdinal: + normalizer = Normalizer( + input_case='cased', lang='kn', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + + @parameterized.expand(parse_test_case_file('kn/data_text_normalization/test_cases_ordinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/kn/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/kn/test_sparrowhawk_normalization.sh index d652a756e..d85f93837 100644 --- a/tests/nemo_text_processing/kn/test_sparrowhawk_normalization.sh +++ b/tests/nemo_text_processing/kn/test_sparrowhawk_normalization.sh @@ -37,5 +37,10 @@ testTNPunctuation() { runtest $input } +testTNOrdinal() { + input=$PROJECT_DIR/kn/data_text_normalization/test_cases_ordinal.txt + runtest $input +} + # Load shUnit2 . $PROJECT_DIR/../shunit2/shunit2 \ No newline at end of file