diff --git a/Jenkinsfile b/Jenkinsfile index 39972c461..8558ed322 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,7 +11,7 @@ pipeline { } environment { AR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-11-26-0' - DE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-23-24-0' + DE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-11-26-0' EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-11-26-1' ES_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-25-24-0' ES_EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/08-30-24-0' diff --git a/nemo_text_processing/inverse_text_normalization/de/data/__init__.py b/nemo_text_processing/inverse_text_normalization/de/data/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/__init__.py b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/conjunction.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/conjunction.tsv new file mode 100644 index 000000000..2babcce6d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/conjunction.tsv @@ -0,0 +1 @@ +und \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/digits.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/digits.tsv new file mode 100644 index 000000000..370e42b6e --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/digits.tsv @@ -0,0 +1,13 @@ +eine 1 +eins 1 +ein 1 +einer 1 +zwei 2 +zwo 2 +drei 3 +vier 4 +fünf 5 +sechs 6 +sieben 7 +acht 8 +neun 9 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/irregular_teens.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/irregular_teens.tsv new file mode 100644 index 000000000..090760e59 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/irregular_teens.tsv @@ -0,0 +1,3 @@ +zehn 10 +elf 11 +zwölf 12 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/regular_teens.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/regular_teens.tsv new file mode 100644 index 000000000..acf4411a7 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/regular_teens.tsv @@ -0,0 +1,7 @@ +dreizehn 13 +vierzehn 14 +fünfzehn 15 +sechzehn 16 +siebzehn 17 +achtzehn 18 +neunzehn 19 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/tens.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/tens.tsv new file mode 100644 index 000000000..28a3f8032 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/tens.tsv @@ -0,0 +1,8 @@ +zwanzig 2 +dreißig 3 +vierzig 4 +fünfzig 5 +sechzig 6 +siebzig 7 +achtzig 8 +neunzig 9 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/zero.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/zero.tsv new file mode 100644 index 000000000..e27b1c640 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/zero.tsv @@ -0,0 +1 @@ +null 0 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/graph_utils.py b/nemo_text_processing/inverse_text_normalization/de/graph_utils.py new file mode 100644 index 000000000..cf281bad8 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/graph_utils.py @@ -0,0 +1,319 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import string +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.examples import plurals +from pynini.export import export +from pynini.lib import byte, pynutil, utf8 + +from nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels +from nemo_text_processing.utils.logging import logger + +NEMO_CHAR = utf8.VALID_UTF8_CHAR + +NEMO_DIGIT = byte.DIGIT +NEMO_LOWER = pynini.union(*string.ascii_lowercase).optimize() +NEMO_UPPER = pynini.union(*string.ascii_uppercase).optimize() +NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize() +NEMO_ALNUM = pynini.union(NEMO_DIGIT, NEMO_ALPHA).optimize() +NEMO_HEX = pynini.union(*string.hexdigits).optimize() +NEMO_NON_BREAKING_SPACE = "\u00a0" +NEMO_SPACE = " " +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", "\u00a0").optimize() +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() + +NEMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize() +NEMO_GRAPH = pynini.union(NEMO_ALNUM, NEMO_PUNCT).optimize() + +NEMO_SIGMA = pynini.closure(NEMO_CHAR) +NEMO_LOWER_NOT_A = pynini.union( + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", +).optimize() + +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +delete_zero_or_one_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE, 0, 1)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") +delete_preserve_order = pynini.closure( + pynutil.delete(" preserve_order: true") + | (pynutil.delete(' field_order: "') + NEMO_NOT_QUOTE + pynutil.delete('"')) +) + +suppletive = pynini.string_file(get_abs_path("data/suppletive.tsv")) +# _v = pynini.union("a", "e", "i", "o", "u") +_c = pynini.union( + "b", + "c", + "d", + "f", + "g", + "h", + "j", + "k", + "l", + "m", + "n", + "p", + "q", + "r", + "s", + "t", + "v", + "w", + "x", + "y", + "z", +) +_ies = NEMO_SIGMA + _c + pynini.cross("y", "ies") +_es = NEMO_SIGMA + pynini.union("s", "sh", "ch", "x", "z") + pynutil.insert("es") +_s = NEMO_SIGMA + pynutil.insert("s") + +graph_plural = plurals._priority_union( + suppletive, + plurals._priority_union(_ies, plurals._priority_union(_es, _s, NEMO_SIGMA), NEMO_SIGMA), + NEMO_SIGMA, +).optimize() + +SINGULAR_TO_PLURAL = graph_plural +PLURAL_TO_SINGULAR = pynini.invert(graph_plural) +TO_LOWER = pynini.union(*[pynini.cross(x, y) for x, y in zip(string.ascii_uppercase, string.ascii_lowercase)]) +TO_UPPER = pynini.invert(TO_LOWER) +MIN_NEG_WEIGHT = -0.0001 +MIN_POS_WEIGHT = 0.0001 +INPUT_CASED = "cased" +INPUT_LOWER_CASED = "lower_cased" +MINUS = pynini.union("minus", "Minus").optimize() + + +def capitalized_input_graph( + graph: "pynini.FstLike", + original_graph_weight: float = None, + capitalized_graph_weight: float = None, +) -> "pynini.FstLike": + """ + Allow graph input to be capitalized, e.g. for ITN) + + Args: + graph: FstGraph + original_graph_weight: weight to add to the original `graph` + capitalized_graph_weight: weight to add to the capitalized graph + """ + capitalized_graph = pynini.compose(TO_LOWER + NEMO_SIGMA, graph).optimize() + + if original_graph_weight is not None: + graph = pynutil.add_weight(graph, weight=original_graph_weight) + + if capitalized_graph_weight is not None: + capitalized_graph = pynutil.add_weight(capitalized_graph, weight=capitalized_graph_weight) + + graph |= capitalized_graph + return graph + + +def generator_main(file_name: str, graphs: Dict[str, "pynini.FstLike"]): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logger.info(f"Created {file_name}") + + +def get_plurals(fst): + """ + Given singular returns plurals + + Args: + fst: Fst + + Returns plurals to given singular forms + """ + return SINGULAR_TO_PLURAL @ fst + + +def get_singulars(fst): + """ + Given plural returns singulars + + Args: + fst: Fst + + Returns singulars to given plural forms + """ + return PLURAL_TO_SINGULAR @ fst + + +def convert_space(fst) -> "pynini.FstLike": + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA) + + +def string_map_cased(input_file: str, input_case: str = INPUT_LOWER_CASED): + labels = load_labels(input_file) + + if input_case == INPUT_CASED: + additional_labels = [] + for written, spoken, *weight in labels: + written_capitalized = written[0].upper() + written[1:] + additional_labels.extend( + [ + [ + written_capitalized, + spoken.capitalize(), + ], # first letter capitalized + [ + written_capitalized, + spoken.upper().replace(" AND ", " and "), + ], # # add pairs with the all letters capitalized + ] + ) + + spoken_no_space = spoken.replace(" ", "") + # add abbreviations without spaces (both lower and upper case), i.e. "BMW" not "B M W" + if len(spoken) == (2 * len(spoken_no_space) - 1): + logger.debug(f"This is weight {weight}") + if len(weight) == 0: + additional_labels.extend( + [ + [written, spoken_no_space], + [written_capitalized, spoken_no_space.upper()], + ] + ) + else: + additional_labels.extend( + [ + [written, spoken_no_space, weight[0]], + [written_capitalized, spoken_no_space.upper(), weight[0]], + ] + ) + labels += additional_labels + + whitelist = pynini.string_map(labels).invert().optimize() + return whitelist + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + "/grammars/" + kind + "/" + name + ".far") + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> "pynini.FstLike": + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> "pynini.FstLike": + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> "pynini.FstLike": + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross("\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py index 46fdca4e3..252378b97 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py @@ -12,57 +12,229 @@ # See the License for the specific language governing permissions and # limitations under the License. +from collections import defaultdict + import pynini from pynini.lib import pynutil -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, GraphFst +from nemo_text_processing.inverse_text_normalization.de.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_WHITE_SPACE, + GraphFst, + delete_space, +) +from nemo_text_processing.inverse_text_normalization.de.utils import get_abs_path, load_labels + + +def get_tens_digit(digit_path: str, tens_path: str, conjunction_path: str) -> 'pynini.FstLike': + """ + getting all denormalizations for numbers between 21 - 99. German says the ones digit + before the tens digit (ein-und-zwanzig = 21), so the words cannot be read left to right + + Args: + digit_path: file to digits tsv + tens_path: file to tens tsv, e.g. zwanzig -> 2 + conjunction_path: file to the conjunction tsv, e.g. und + Returns: + res: fst that converts the verbalization of a number to its digits + """ + + conjunction = load_labels(conjunction_path)[0][0] + digits = defaultdict(list) + ties = defaultdict(list) + for k, v in load_labels(digit_path): + digits[v].append(k) + + for k, v in load_labels(tens_path): + ties[v].append(k) + + d = [] + for i in range(21, 100): + s = str(i) + if s[1] == "0": + continue + + for di in digits[s[1]]: + for ti in ties[s[0]]: + word = di + conjunction + ti + d.append((word, s)) + + res = pynini.string_map(d) + return res class CardinalFst(GraphFst): """ - Finite state transducer for classifying cardinals. Numbers below ten are not converted. + Finite state transducer for classifying cardinals. Numbers below thirteen are not converted + unless they carry a minus sign. Allows both compound numeral strings or separated by whitespace. - "und" (en: "and") can be inserted between "hundert" and following number or "tausend" and following single or double digit number. - - e.g. minus drei und zwanzig -> cardinal { negative: "-" integer: "23" } } - e.g. minus dreiundzwanzig -> cardinal { integer: "23" } } - e.g. dreizehn -> cardinal { integer: "13" } } - e.g. ein hundert -> cardinal { integer: "100" } } - e.g. einhundert -> cardinal { integer: "100" } } - e.g. ein tausend -> cardinal { integer: "1000" } } - e.g. eintausend -> cardinal { integer: "1000" } } - e.g. ein tausend zwanzig -> cardinal { integer: "1020" } } + "und" (en: "and") after "hundert", "tausend" or a larger magnitude word is never part of the + number, it joins two numbers, whether written glued or spaced: both "einhundertundzwei" and + "ein hundert und zwei" -> 100 und 2. German writes 102 as "ein hundert zwei". - Args: - tn_cardinal_tagger: TN cardinal tagger + e.g. minus drei und zwanzig -> cardinal { negative: "-" integer: "23" } + e.g. minus dreiundzwanzig -> cardinal { negative: "-" integer: "23" } + e.g. dreizehn -> cardinal { integer: "13" } + e.g. ein hundert -> cardinal { integer: "100" } + e.g. einhundert -> cardinal { integer: "100" } + e.g. ein tausend -> cardinal { integer: "1.000" } + e.g. eintausend -> cardinal { integer: "1.000" } + e.g. ein tausend zwanzig -> cardinal { integer: "1.020" } """ - def __init__(self, tn_cardinal_tagger: GraphFst, deterministic: bool = True): - super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + def __init__(self): + super().__init__(name="cardinal", kind="classify") - # add_space_between_chars = pynini.cdrewrite(pynini.closure(insert_space, 0, 1), NEMO_CHAR, NEMO_CHAR, NEMO_SIGMA) - optional_delete_space = pynini.closure(NEMO_SIGMA | pynutil.delete(" ")) + zero = pynini.string_file(get_abs_path("data/cardinal/zero.tsv")) + digits = pynini.string_file(get_abs_path("data/cardinal/digits.tsv")) + irregular_teens = pynini.string_file(get_abs_path("data/cardinal/irregular_teens.tsv")) + to_denormalize = zero | digits | irregular_teens + + regular_teens = pynini.string_file(get_abs_path("data/cardinal/regular_teens.tsv")) + teens = irregular_teens | regular_teens + tens = pynini.string_file(get_abs_path("data/cardinal/tens.tsv")) + ties = tens + pynutil.insert("0") + conjunction = load_labels(get_abs_path("data/cardinal/conjunction.tsv"))[0][0] + + # the map is keyed on the compound spelling, so whitespace is stripped before lookup + delete_all_spaces = pynini.cdrewrite(pynutil.delete(NEMO_WHITE_SPACE), "", "", NEMO_SIGMA) + ties_digit = delete_all_spaces @ get_tens_digit( + get_abs_path("data/cardinal/digits.tsv"), + get_abs_path("data/cardinal/tens.tsv"), + get_abs_path("data/cardinal/conjunction.tsv"), + ) - graph = (tn_cardinal_tagger.graph @ optional_delete_space).invert().optimize() + graph_10_99 = teens | ties | ties_digit + + hundreds = ( + ((digits | pynutil.insert("1")) + delete_space + pynutil.delete("hundert") + delete_space + graph_10_99) + | ((digits | pynutil.insert("1")) + delete_space + pynini.cross("hundert", "0") + delete_space + digits) + | ((digits | pynutil.insert("1")) + delete_space + pynini.cross("hundert", "00")) + ) + + # Digits are grouped in clusters of three: {hundreds}{tens}{ones}. + non_zero_digit_cluster = (hundreds) | (pynutil.insert("0") + graph_10_99) | (pynutil.insert("00") + digits) + digit_cluster = non_zero_digit_cluster | pynutil.insert("000") + # a magnitude word with no multiplier in front of it means "one" of that magnitude + leading_cluster = non_zero_digit_cluster | pynutil.insert("001") + + thousands = ( + ((leading_cluster + delete_space + pynini.cross("tausend", ".")) | pynutil.insert("000.")) + + delete_space + + digit_cluster + ) + + million = pynini.accep("million") | pynini.accep("millionen") + millions = ( + ((leading_cluster + delete_space + pynini.cross(million, ".")) | pynutil.insert("000.")) + + delete_space + + thousands + ) + + billion = pynini.accep("milliarde") | pynini.accep("milliarden") + billions = ( + ((leading_cluster + delete_space + pynini.cross(billion, ".")) | pynutil.insert("000.")) + + delete_space + + millions + ) + + trillion = pynini.accep("billion") | pynini.accep("billionen") + trillions = ( + ((leading_cluster + delete_space + pynini.cross(trillion, ".")) | pynutil.insert("000.")) + + delete_space + + billions + ) + + quadrillion = pynini.accep("billiarde") | pynini.accep("billiarden") + quadrillions = ( + ((leading_cluster + delete_space + pynini.cross(quadrillion, ".")) | pynutil.insert("000.")) + + delete_space + + trillions + ) + + quintillion = pynini.accep("trillion") | pynini.accep("trillionen") + quintillions = ( + ((leading_cluster + delete_space + pynini.cross(quintillion, ".")) | pynutil.insert("000.")) + + delete_space + + quadrillions + ) + + sextillion = pynini.accep("trilliarde") | pynini.accep("trilliarden") + sextillions = ( + ((leading_cluster + delete_space + pynini.cross(sextillion, ".")) | pynutil.insert("000.")) + + delete_space + + quintillions + ) + + non_zero_digits = pynini.difference(NEMO_DIGIT, "0") + chars_to_remove = pynini.accep("0") | pynini.accep(".") + remove_chars = pynutil.delete(pynini.closure(chars_to_remove)) + remove_leading_zeros = pynini.cdrewrite(remove_chars, "[BOS]", non_zero_digits, NEMO_SIGMA) + + grammars = [ + sextillions, + quintillions, + quadrillions, + trillions, + billions, + millions, + thousands, + digit_cluster, + zero, + ] + + graph_cardinals = "" + for grammar in grammars: + graph_cardinals |= grammar + + # the graph the other German semiotic classes consume, without the first-dozen exception + self.graph_no_exception = (graph_cardinals @ remove_leading_zeros).optimize() + + # 1-999 without leading zeros, consumed by the decimal tagger's get_quantity self.graph_hundred_component_at_least_one_none_zero_digit = ( - (tn_cardinal_tagger.graph_hundred_component_at_least_one_none_zero_digit @ optional_delete_space) - .invert() - .optimize() + non_zero_digit_cluster @ remove_leading_zeros + ).optimize() + + # the block below leaves numerals 1 - 12 spelled out + accept_denormalized_first_dozen = pynini.project(to_denormalize, "input") + accept_denormalized_everything = pynini.project(self.graph_no_exception, "input") + accept_without_first_dozen = accept_denormalized_everything - accept_denormalized_first_dozen + transduce_without_first_dozen = accept_without_first_dozen @ self.graph_no_exception + graph = accept_denormalized_first_dozen | transduce_without_first_dozen + self.graph = graph.optimize() + + magnitude_word = pynini.union( + "hundert", "tausend", million, billion, trillion, quadrillion, quintillion, sextillion + ) + ends_in_magnitude = pynini.compose(NEMO_SIGMA + magnitude_word, self.graph_no_exception) + graph_magnitude_und = ( + ends_in_magnitude + + delete_space + + pynutil.insert(" ") + + pynini.accep(conjunction) + + pynutil.insert(" ") + + delete_space + + self.graph_hundred_component_at_least_one_none_zero_digit ) - self.graph_ties = (tn_cardinal_tagger.two_digit_non_zero @ optional_delete_space).invert().optimize() - # this is to make sure if there is an ambiguity with decimal, decimal is chosen, e.g. 1000000 vs. 1 million - graph = pynutil.add_weight(graph, weight=0.001) - self.graph_no_exception = graph - self.digit = pynini.arcmap(tn_cardinal_tagger.digit, map_type="rmweight").invert().optimize() - graph_exception = pynini.project(self.digit, 'input') - self.graph = (pynini.project(graph, "input") - graph_exception.arcsort()) @ graph + negative = pynutil.insert("negative: ") + pynini.cross("minus ", '"-"') + pynutil.insert(" ") + self.optional_minus_graph = pynini.closure(negative, 0, 1) + integer = pynutil.insert('integer: "') + (self.graph | graph_magnitude_und) + pynutil.insert('"') + # a sign in front of zero carries no meaning, so "minus null" is not a cardinal + accept_zero = pynini.project(zero, "input") + graph_no_exception_non_zero = ( + pynini.difference(accept_denormalized_everything, accept_zero) @ self.graph_no_exception + ) - self.optional_minus_graph = pynini.closure( - pynutil.insert("negative: ") + pynini.cross("minus ", "\"-\" "), 0, 1 + negative_integer = ( + negative + + pynutil.insert('integer: "') + + (graph_no_exception_non_zero | graph_magnitude_und) + + pynutil.insert('"') ) - final_graph = self.optional_minus_graph + pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + final_graph = integer | negative_integer final_graph = self.add_tokens(final_graph) self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py index 1d60d071a..e0dbc4a42 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py @@ -93,7 +93,7 @@ def __init__( tn_electronic_verbalizer = TNElectronicVerbalizer(deterministic=False) tn_whitelist_tagger = TNWhitelistTagger(input_case="cased", deterministic=False, input_file=whitelist) - cardinal = CardinalFst(tn_cardinal_tagger=tn_cardinal_tagger) + cardinal = CardinalFst() cardinal_graph = cardinal.fst ordinal = OrdinalFst(itn_cardinal_tagger=cardinal, tn_ordinal_verbalizer=tn_ordinal_verbalizer) diff --git a/nemo_text_processing/inverse_text_normalization/de/utils.py b/nemo_text_processing/inverse_text_normalization/de/utils.py new file mode 100644 index 000000000..461631f46 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/utils.py @@ -0,0 +1,42 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + "/" + rel_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels diff --git a/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py index b13382a8e..019b09b19 100644 --- a/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py @@ -15,22 +15,34 @@ import pynini from pynini.lib import pynutil -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst +from nemo_text_processing.inverse_text_normalization.de.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space class CardinalFst(GraphFst): """ Finite state transducer for verbalizing cardinal - e.g. cardinal { integer: "23" negative: "-" } -> -23 - - Args: - tn_cardinal_verbalizer: TN cardinal verbalizer + e.g. cardinal { negative: "-" integer: "23" } -> -23 """ - def __init__(self, tn_cardinal_verbalizer: GraphFst, deterministic: bool = True): - super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) - self.numbers = tn_cardinal_verbalizer.numbers - optional_sign = pynini.closure(pynutil.delete("negative: \"") + NEMO_NOT_QUOTE + pynutil.delete("\" "), 0, 1) - graph = optional_sign + self.numbers + def __init__(self): + super().__init__(name="cardinal", kind="verbalize") + + # the tagger writes the sign itself, so the verbalizer just reads it out of the field + optional_minus = pynini.closure( + pynutil.delete('negative: "') + NEMO_NOT_QUOTE + pynutil.delete('"') + delete_space, 0, 1 + ) + + # removes the 'integer:' label + just_integers = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + + delete_space + ) + + graph = optional_minus + just_integers + self.numbers = graph.optimize() delete_tokens = self.delete_tokens(graph) self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/de/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/de/verbalizers/verbalize.py index 9c921a63a..014777890 100644 --- a/nemo_text_processing/inverse_text_normalization/de/verbalizers/verbalize.py +++ b/nemo_text_processing/inverse_text_normalization/de/verbalizers/verbalize.py @@ -34,7 +34,7 @@ def __init__(self, deterministic: bool = True): tn_cardinal_verbalizer = TNCardinalVerbalizer(deterministic=False) tn_decimal_verbalizer = TNDecimalVerbalizer(deterministic=False) - cardinal = CardinalFst(tn_cardinal_verbalizer=tn_cardinal_verbalizer) + cardinal = CardinalFst() cardinal_graph = cardinal.fst decimal = DecimalFst(tn_decimal_verbalizer=tn_decimal_verbalizer) decimal_graph = decimal.fst diff --git a/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt index 0b2064296..e8d51ecb5 100644 --- a/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt +++ b/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt @@ -1,46 +1,54 @@ ein hundert~100 einhundert~100 -ein hundert und zwei~102 einhundertzwei~102 -ein hundert und zwanzig~120 -ein hundert und elf~111 -ein tausend~1000 -eintausend~1000 +ein hundert zwei~102 +ein hundert und zwei~100 und 2 +ein hundert und zwanzig~100 und 20 +ein hundert und elf~100 und 11 +einhundertundelf~100 und 11 +ein tausend~1.000 +eintausend~1.000 +ein hundert zwölf~112 ein hundert zwanzig~120 -ein tausend zwanzig~1020 -eintausendzwanzig~1020 -neun billionen sieben hundert neun und achtzig milliarden drei hundert zwei und achtzig millionen fünf hundert sechs und dreißig tausend ein hundert dreißig~9789382536130 +hundert zwanzig~120 +ein tausend zwanzig~1.020 +eintausendzwanzig~1.020 +neun billionen sieben hundert neun und achtzig milliarden drei hundert zwei und achtzig millionen fünf hundert sechs und dreißig tausend ein hundert dreißig~9.789.382.536.130 zwei hundert vier und fünfzig~254 -ein hundert sieben und vierzig tausend vier hundert ein und fünfzig~147451 -eine million ein hundert sechs und fünfzig tausend ein hundert drei und siebzig~1156173 -eine milliarde fünf hundert drei und neunzig millionen zwei und siebzig tausend neun hundert ein und sechzig~1593072961 -sieben und neunzig billiarden acht hundert acht billionen zwei hundert vier und sechzig milliarden sieben hundert zwei und siebzig millionen sieben hundert zwei und neunzig tausend fünf~97808264772792005 -zehn billiarden zehn billionen zehn millionen ein hundert tausend zehn~10010000010100010 -zehn billiarden zehn billionen zehn millionen einhunderttausendzehn~10010000010100010 -minus fünf und zwanzig tausend sieben und dreißig~-25037 -minus fünf und zwanzig tausend sieben und dreißig~-25037 -minus fünfundzwanzigtausendsiebenunddreißig~-25037 -eine billiarde zwei hundert vier und sechzig billionen drei hundert eins milliarden neun hundert acht und dreißig millionen ein hundert vier~1264301938000104 -eine billiarde zweihundertvierundsechzig billionen dreihunderteins milliarden neunhundertachtunddreißig millionen einhundertvier~1264301938000104 +zwo hundert~200 +zwoundzwanzig~22 +ein hundert sieben und vierzig tausend vier hundert ein und fünfzig~147.451 +eine million ein hundert sechs und fünfzig tausend ein hundert drei und siebzig~1.156.173 +eine milliarde fünf hundert drei und neunzig millionen zwei und siebzig tausend neun hundert ein und sechzig~1.593.072.961 +sieben und neunzig billiarden acht hundert acht billionen zwei hundert vier und sechzig milliarden sieben hundert zwei und siebzig millionen sieben hundert zwei und neunzig tausend fünf~97.808.264.772.792.005 +zehn billiarden zehn billionen zehn millionen ein hundert tausend zehn~10.010.000.010.100.010 +zehn billiarden zehn billionen zehn millionen einhunderttausendzehn~10.010.000.010.100.010 +eine trillion ein hundert~1.000.000.000.000.000.100 +elf trilliarden drei tausend sieben und fünfzig~11.000.000.000.000.000.003.057 +minus fünf und zwanzig tausend sieben und dreißig~-25.037 +minus fünfundzwanzigtausendsiebenunddreißig~-25.037 +eine billiarde zwei hundert vier und sechzig billionen drei hundert eins milliarden neun hundert acht und dreißig millionen ein hundert vier~1.264.301.938.000.104 +eine billiarde zweihundertvierundsechzig billionen dreihunderteins milliarden neunhundertachtunddreißig millionen einhundertvier~1.264.301.938.000.104 minus sechzig~-60 -sechs und vierzig tausend sechs hundert vier und sechzig~46664 -sechzig~60 +sechsundvierzig tausend sechshundert vierundsechzig~46.664 null~null eins~eins ein~ein eine~eine einer~einer zwei~zwei -neun~neun -zehn~10 -elf~11 -zwölf~12 +zwo~zwo +drei~drei +zehn~zehn +elf~elf +zwölf~zwölf dreizehn~13 vierzehn~14 fünfzehn~15 sechzehn~16 siebzehn~17 achtzehn~18 +neunzehn~19 zwanzig~20 dreißig~30 vierzig~40 @@ -49,14 +57,38 @@ sechzig~60 siebzig~70 achtzig~80 neunzig~90 -zwei millionen drei~2000003 -ein tausend dreizehn~1013 -ein tausend eins~1001 -ein tausend ein hundert~1100 -ein tausend sechs und zwanzig~1026 -ein tausend ein hundert sechs und zwanzig~1126 -achtzehn millionen vier hundert fünfzig tausend neun hundert neunzig~18450990 -achtzehn millionen neun hundert vierzig tausend sieben hundert zwei und zwanzig~18940722 -achtzehn millionen sechs hundert neunzig tausend neun hundert sechzehn~18690916 -achtzehn millionen sechshundertneunzigtausendneunhundertsechzehn~18690916 -achtzehn tausend acht hundert achtzig~18880 +hundert~100 +eine million ein tausend~1.001.000 +zwei millionen drei~2.000.003 +tausend~1.000 +tausend zwanzig~1.020 +ein tausend dreizehn~1.013 +ein tausend eins~1.001 +ein tausend ein hundert~1.100 +ein tausend sechs und zwanzig~1.026 +ein tausend ein hundert sechs und zwanzig~1.126 +achtzehn millionen vier hundert fünfzig tausend neun hundert neunzig~18.450.990 +achtzehn millionen neun hundert vierzig tausend sieben hundert zwei und zwanzig~18.940.722 +achtzehn millionen sechs hundert neunzig tausend neun hundert sechzehn~18.690.916 +achtzehn millionen sechshundertneunzigtausendneunhundertsechzehn~18.690.916 +zwei millionen ein tausend acht~2.001.008 +achtzehn tausend acht hundert achtzig~18.880 +einhunderteins~101 +ein tausend einhundert sechsundzwanzig~1.126 +ein tausend und zwanzig~1.000 und 20 +eintausendundzwanzig~1.000 und 20 +zwei tausend und vier hundert zwei und zwanzig~2.000 und 422 +tausend ein hundert~1.100 +eine milliarde ein tausend~1.000.001.000 +eine milliarde tausend~1.000.001.000 +zwei millionen ein tausend drei~2.001.003 +zwei millionen tausend drei~2.001.003 +eine million tausend~1.001.000 +minus drei~-3 +minus ein tausend und zwanzig~-1.000 und 20 +trillion~1.000.000.000.000.000.000 +eine million und zwanzig~1.000.000 und 20 +eine million und drei und zwanzig~1.000.000 und 23 +eine trilliarde zwei trillionen eine billion vier~1.002.000.001.000.000.000.004 +eine milliarde und vierzehn~1.000.000.000 und 14 +eine billion und fünf und siebzig~1.000.000.000.000 und 75