From 51e2d2b083ea5a53866ccc835fc8a076ce4a3e18 Mon Sep 17 00:00:00 2001 From: Adelina Dunina Date: Fri, 11 Sep 2026 16:49:17 -0400 Subject: [PATCH 01/13] De ITN: decouple cardinal grammar from TN Signed-off-by: Adelina Dunina --- Jenkinsfile | 2 +- .../de/data/__init__.py | 13 + .../de/data/cardinal/__init__.py | 13 + .../de/data/cardinal/digits.tsv | 14 + .../de/data/cardinal/irregular_teens.tsv | 3 + .../de/data/cardinal/teens.tsv | 10 + .../de/data/cardinal/tens.tsv | 8 + .../de/data/cardinal/zero.tsv | 1 + .../de/graph_utils.py | 319 ++++++++++++++++++ .../de/taggers/cardinal.py | 280 +++++++++++++-- .../de/taggers/tokenize_and_classify.py | 2 +- .../inverse_text_normalization/de/utils.py | 27 ++ .../de/verbalizers/cardinal.py | 34 +- .../de/verbalizers/verbalize.py | 2 +- .../test_cases_cardinal.txt | 66 ++-- 15 files changed, 714 insertions(+), 80 deletions(-) create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/digits.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/irregular_teens.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/teens.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/tens.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/zero.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/de/graph_utils.py create mode 100644 nemo_text_processing/inverse_text_normalization/de/utils.py diff --git a/Jenkinsfile b/Jenkinsfile index 39972c461..8558ed322 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,7 +11,7 @@ pipeline { } environment { AR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-11-26-0' - DE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-23-24-0' + DE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-11-26-0' EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-11-26-1' ES_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-25-24-0' ES_EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/08-30-24-0' diff --git a/nemo_text_processing/inverse_text_normalization/de/data/__init__.py b/nemo_text_processing/inverse_text_normalization/de/data/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/__init__.py b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/digits.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/digits.tsv new file mode 100644 index 000000000..223c8db47 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/digits.tsv @@ -0,0 +1,14 @@ +eine 1 +eins 1 +ein 1 +einer 1 +zwei 2 +zwo 2 +zwö 2 +drei 3 +vier 4 +fünf 5 +sechs 6 +sieben 7 +acht 8 +neun 9 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/irregular_teens.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/irregular_teens.tsv new file mode 100644 index 000000000..090760e59 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/irregular_teens.tsv @@ -0,0 +1,3 @@ +zehn 10 +elf 11 +zwölf 12 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/teens.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/teens.tsv new file mode 100644 index 000000000..854712ea6 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/teens.tsv @@ -0,0 +1,10 @@ +zehn 10 +elf 11 +zwölf 12 +dreizehn 13 +vierzehn 14 +fünfzehn 15 +sechzehn 16 +siebzehn 17 +achtzehn 18 +neunzehn 19 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/tens.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/tens.tsv new file mode 100644 index 000000000..28a3f8032 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/tens.tsv @@ -0,0 +1,8 @@ +zwanzig 2 +dreißig 3 +vierzig 4 +fünfzig 5 +sechzig 6 +siebzig 7 +achtzig 8 +neunzig 9 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/zero.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/zero.tsv new file mode 100644 index 000000000..e27b1c640 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/zero.tsv @@ -0,0 +1 @@ +null 0 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/graph_utils.py b/nemo_text_processing/inverse_text_normalization/de/graph_utils.py new file mode 100644 index 000000000..8dc9c6391 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/graph_utils.py @@ -0,0 +1,319 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import string +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.examples import plurals +from pynini.export import export +from pynini.lib import byte, pynutil, utf8 + +from nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels +from nemo_text_processing.utils.logging import logger + +NEMO_CHAR = utf8.VALID_UTF8_CHAR + +NEMO_DIGIT = byte.DIGIT +NEMO_LOWER = pynini.union(*string.ascii_lowercase).optimize() +NEMO_UPPER = pynini.union(*string.ascii_uppercase).optimize() +NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize() +NEMO_ALNUM = pynini.union(NEMO_DIGIT, NEMO_ALPHA).optimize() +NEMO_HEX = pynini.union(*string.hexdigits).optimize() +NEMO_NON_BREAKING_SPACE = "\u00a0" +NEMO_SPACE = " " +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", "\u00a0").optimize() +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() + +NEMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize() +NEMO_GRAPH = pynini.union(NEMO_ALNUM, NEMO_PUNCT).optimize() + +NEMO_SIGMA = pynini.closure(NEMO_CHAR) +NEMO_LOWER_NOT_A = pynini.union( + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", +).optimize() + +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +delete_zero_or_one_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE, 0, 1)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") +delete_preserve_order = pynini.closure( + pynutil.delete(" preserve_order: true") + | (pynutil.delete(' field_order: "') + NEMO_NOT_QUOTE + pynutil.delete('"')) +) + +suppletive = pynini.string_file(get_abs_path("data/suppletive.tsv")) +# _v = pynini.union("a", "e", "i", "o", "u") +_c = pynini.union( + "b", + "c", + "d", + "f", + "g", + "h", + "j", + "k", + "l", + "m", + "n", + "p", + "q", + "r", + "s", + "t", + "v", + "w", + "x", + "y", + "z", +) +_ies = NEMO_SIGMA + _c + pynini.cross("y", "ies") +_es = NEMO_SIGMA + pynini.union("s", "sh", "ch", "x", "z") + pynutil.insert("es") +_s = NEMO_SIGMA + pynutil.insert("s") + +graph_plural = plurals._priority_union( + suppletive, + plurals._priority_union(_ies, plurals._priority_union(_es, _s, NEMO_SIGMA), NEMO_SIGMA), + NEMO_SIGMA, +).optimize() + +SINGULAR_TO_PLURAL = graph_plural +PLURAL_TO_SINGULAR = pynini.invert(graph_plural) +TO_LOWER = pynini.union(*[pynini.cross(x, y) for x, y in zip(string.ascii_uppercase, string.ascii_lowercase)]) +TO_UPPER = pynini.invert(TO_LOWER) +MIN_NEG_WEIGHT = -0.0001 +MIN_POS_WEIGHT = 0.0001 +INPUT_CASED = "cased" +INPUT_LOWER_CASED = "lower_cased" +MINUS = pynini.union("minus", "Minus").optimize() + + +def capitalized_input_graph( + graph: "pynini.FstLike", + original_graph_weight: float = None, + capitalized_graph_weight: float = None, +) -> "pynini.FstLike": + """ + Allow graph input to be capitalized, e.g. for ITN) + + Args: + graph: FstGraph + original_graph_weight: weight to add to the original `graph` + capitalized_graph_weight: weight to add to the capitalized graph + """ + capitalized_graph = pynini.compose(TO_LOWER + NEMO_SIGMA, graph).optimize() + + if original_graph_weight is not None: + graph = pynutil.add_weight(graph, weight=original_graph_weight) + + if capitalized_graph_weight is not None: + capitalized_graph = pynutil.add_weight(capitalized_graph, weight=capitalized_graph_weight) + + graph |= capitalized_graph + return graph + + +def generator_main(file_name: str, graphs: Dict[str, "pynini.FstLike"]): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logger.info(f"Created {file_name}") + + +def get_plurals(fst): + """ + Given singular returns plurals + + Args: + fst: Fst + + Returns plurals to given singular forms + """ + return SINGULAR_TO_PLURAL @ fst + + +def get_singulars(fst): + """ + Given plural returns singulars + + Args: + fst: Fst + + Returns singulars to given plural forms + """ + return PLURAL_TO_SINGULAR @ fst + + +def convert_space(fst) -> "pynini.FstLike": + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA) + + +def string_map_cased(input_file: str, input_case: str = INPUT_LOWER_CASED): + labels = load_labels(input_file) + + if input_case == INPUT_CASED: + additional_labels = [] + for written, spoken, *weight in labels: + written_capitalized = written[0].upper() + written[1:] + additional_labels.extend( + [ + [ + written_capitalized, + spoken.capitalize(), + ], # first letter capitalized + [ + written_capitalized, + spoken.upper().replace(" AND ", " and "), + ], # # add pairs with the all letters capitalized + ] + ) + + spoken_no_space = spoken.replace(" ", "") + # add abbreviations without spaces (both lower and upper case), i.e. "BMW" not "B M W" + if len(spoken) == (2 * len(spoken_no_space) - 1): + logger.debug(f"This is weight {weight}") + if len(weight) == 0: + additional_labels.extend( + [ + [written, spoken_no_space], + [written_capitalized, spoken_no_space.upper()], + ] + ) + else: + additional_labels.extend( + [ + [written, spoken_no_space, weight[0]], + [written_capitalized, spoken_no_space.upper(), weight[0]], + ] + ) + labels += additional_labels + + whitelist = pynini.string_map(labels).invert().optimize() + return whitelist + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + "/grammars/" + kind + "/" + name + ".far") + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> "pynini.FstLike": + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> "pynini.FstLike": + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> "pynini.FstLike": + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross("\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py index 46fdca4e3..4a71159e9 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py @@ -12,57 +12,269 @@ # See the License for the specific language governing permissions and # limitations under the License. + import pynini from pynini.lib import pynutil -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, GraphFst +from nemo_text_processing.inverse_text_normalization.de.graph_utils import NEMO_DIGIT, NEMO_SIGMA, NEMO_SPACE, GraphFst +from nemo_text_processing.inverse_text_normalization.de.utils import get_abs_path + + +def swap_tens_and_ones(digits: 'pynini.FstLike', tens: 'pynini.FstLike') -> 'pynini.FstLike': + """ + German says the ones digit before the tens digit (ein-und-zwanzig = 21), so digits + arrive reversed. An FST cannot reorder without enumerating, so this enumerates every + ones/tens pair present in the digit and tens tables. + """ + ones_digits = sorted({output for _, output, _ in digits.paths().items()}) + tens_digits = sorted({output for _, output, _ in tens.paths().items()}) + return pynini.string_map([(one + ten, ten + one) for one in ones_digits for ten in tens_digits]) class CardinalFst(GraphFst): """ - Finite state transducer for classifying cardinals. Numbers below ten are not converted. - Allows both compound numeral strings or separated by whitespace. - "und" (en: "and") can be inserted between "hundert" and following number or "tausend" and following single or double digit number. - - e.g. minus drei und zwanzig -> cardinal { negative: "-" integer: "23" } } - e.g. minus dreiundzwanzig -> cardinal { integer: "23" } } - e.g. dreizehn -> cardinal { integer: "13" } } - e.g. ein hundert -> cardinal { integer: "100" } } - e.g. einhundert -> cardinal { integer: "100" } } - e.g. ein tausend -> cardinal { integer: "1000" } } - e.g. eintausend -> cardinal { integer: "1000" } } - e.g. ein tausend zwanzig -> cardinal { integer: "1020" } } - - Args: - tn_cardinal_tagger: TN cardinal tagger + Finite state transducer for classifying cardinals + e.g. minus dreiundzwanzig -> cardinal { negative: "true" integer: "23" } + e.g. eintausend -> cardinal { integer: "1.000" } + Numbers below thirteen are not converted. + The transducer implements a period separator every three digits by default. """ - def __init__(self, tn_cardinal_tagger: GraphFst, deterministic: bool = True): - super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + def __init__(self): + super().__init__(name="cardinal", kind="classify") - # add_space_between_chars = pynini.cdrewrite(pynini.closure(insert_space, 0, 1), NEMO_CHAR, NEMO_CHAR, NEMO_SIGMA) - optional_delete_space = pynini.closure(NEMO_SIGMA | pynutil.delete(" ")) + # WFST mappings for numbers 0-99 + zero = pynini.string_file(get_abs_path("data/cardinal/zero.tsv")) + digits = pynini.string_file(get_abs_path("data/cardinal/digits.tsv")) + # Isolates single digit cardinals to pass to other graphs + self.digits = digits.optimize() + irregular_teens = pynini.string_file(get_abs_path("data/cardinal/irregular_teens.tsv")) + to_denormalize = zero | digits | irregular_teens - graph = (tn_cardinal_tagger.graph @ optional_delete_space).invert().optimize() - self.graph_hundred_component_at_least_one_none_zero_digit = ( - (tn_cardinal_tagger.graph_hundred_component_at_least_one_none_zero_digit @ optional_delete_space) - .invert() - .optimize() + # Isolates the first dozen + self.dozen = to_denormalize.optimize() + teens = pynini.string_file(get_abs_path("data/cardinal/teens.tsv")) + tens = pynini.string_file(get_abs_path("data/cardinal/tens.tsv")) + ties = tens + pynutil.insert("0") + # German flips ones and tens in two-digit numbers. The WFST below handles these flips. + delete_space = pynutil.delete(NEMO_SPACE) + delete_und = pynutil.delete("und") + + # Accepts normalized digits+ties (ein+und+zwanzig) + digit_ties = digits + delete_space.ques + delete_und + delete_space.ques + tens + # Flips ties and digits for denormalization + ties_digit = digit_ties @ swap_tens_and_ones(digits, tens) + + # WFST grammar for hundreds + graph_10_99 = irregular_teens | teens | ties | ties_digit + self.graph_double_digits = graph_10_99 + # Isolates single and double-digit cardinals to pass to other graphs + graph_single_and_double_digits = digits | graph_10_99 + self.graph_single_and_double_digits = graph_single_and_double_digits.optimize() + + hundert = pynini.accep("hundert") | pynini.accep("ein hundert") + hundreds = (pynini.cross(hundert, "100")) | ( + ( + (digits | pynutil.insert("1")) + + delete_space.ques + + pynutil.delete("hundert") + + delete_space.ques + + delete_und.ques + + delete_space.ques + + graph_10_99 + ) + | ( + (digits | pynutil.insert("1")) + + delete_space.ques + + pynini.cross("hundert", "0") + + delete_space.ques + + delete_und.ques + + delete_space.ques + + digits + ) + | ((digits | pynutil.insert("1")) + delete_space.ques + pynini.cross("hundert", "00")) + ) + + # Digits are grouped in clusters of three: {hundreds}{tens}{ones}. + # Clusters of three are separated by periods, applied right to left. + non_zero_digit_cluster = (hundreds) | (pynutil.insert("0") + graph_10_99) | (pynutil.insert("00") + digits) + digit_cluster = non_zero_digit_cluster | pynutil.insert("000") + + # WFST grammar for thousands + thousands = (pynini.cross("tausend", "1.000")) | ( + ( + (pynini.cross("tausend", "1.") + delete_space.ques + delete_und.ques) + | (digit_cluster + delete_space.ques + pynini.cross("tausend", ".") + delete_und.ques) + | pynutil.insert("000.") + ) + + delete_space.ques + + digit_cluster + ) + + # WFST grammar for millions + million = pynini.accep("million") | pynini.accep("millionen") + millions = (pynini.cross("million", "1.000.000")) | ( + ( + (pynini.cross("million", "1.") + delete_space.ques + delete_und.ques) + | (digit_cluster + delete_space.ques + pynini.cross(million, ".") + delete_und.ques) + | pynutil.insert("000.") + ) + + delete_space.ques + + thousands + ) + + # WFST grammar for billions + billion = ( + pynini.accep("milliarde") + | pynini.accep("milliarden") + # include the consonant-final stem for ordinal declensions e.g "milliardste" + # "e" -> "" / _[ordinal morpheme] + | pynini.accep("milliard") + ) + billions = (pynini.cross("milliarde", "1.000.000.000")) | ( + ( + ( + pynini.cross((pynini.accep("milliarde") | pynini.accep("milliard")), "1.") + + delete_space.ques + + delete_und.ques + ) + | (digit_cluster + delete_space.ques + pynini.cross(billion, ".") + delete_und.ques) + | pynutil.insert("000.") + ) + + delete_space.ques + + millions + ) + + # WFST grammar for trillions + trillion = pynini.accep("billion") | pynini.accep("billionen") + trillions = (pynini.cross("billion", "1.000.000.000.000")) | ( + ( + (pynini.cross("billion", "1.") + delete_space.ques + delete_und.ques) + | (digit_cluster + delete_space.ques + pynini.cross(trillion, ".") + delete_und.ques) + | pynutil.insert("000.") + ) + + delete_space.ques + + billions ) - self.graph_ties = (tn_cardinal_tagger.two_digit_non_zero @ optional_delete_space).invert().optimize() - # this is to make sure if there is an ambiguity with decimal, decimal is chosen, e.g. 1000000 vs. 1 million - graph = pynutil.add_weight(graph, weight=0.001) - self.graph_no_exception = graph - self.digit = pynini.arcmap(tn_cardinal_tagger.digit, map_type="rmweight").invert().optimize() - graph_exception = pynini.project(self.digit, 'input') - self.graph = (pynini.project(graph, "input") - graph_exception.arcsort()) @ graph + # WFST grammar for quadrillions + quadrillion = ( + pynini.accep("billiarde") + | pynini.accep("billiarden") + # include the consonant-final stem for ordinal declensions e.g "billiardste" + # "e" -> "" / _[ordinal morpheme] + | pynini.accep("billiard") + ) + quadrillions = (pynini.cross("billiarde", "1.000.000.000.000.000")) | ( + ( + (pynini.cross(quadrillion, "1.") + delete_space.ques + delete_und.ques) + | (digit_cluster + delete_space.ques + pynini.cross(quadrillion, ".") + delete_und.ques) + | pynutil.insert("000.") + ) + + delete_space.ques + + trillions + ) + + # WFST grammar for quintillions + quintillion = pynini.accep("trillion") | pynini.accep("trillionen") + quintillions = (pynini.cross("trillion", "1.000.000.000.000.000.000")) | ( + ( + (pynini.cross("trillion", "1.") + delete_space.ques + delete_und.ques) + | (digit_cluster + delete_space.ques + pynini.cross(quintillion, ".") + delete_und.ques) + | pynutil.insert("000.") + ) + + delete_space.ques + + quadrillions + ) + + # WFST grammar for sextillions + sextillion = ( + pynini.accep("trilliarde") + | pynini.accep("trilliarden") + # include the consonant-final stem for ordinal declensions e.g "trilliardste" + # "e" -> "" / _[ordinal morpheme] + | pynini.accep("trilliard") + ) + sextillions = (pynini.cross("billiarde", "1.000.000.000.000.000.000.000")) | ( + ( + (pynini.cross(sextillion, "1.") + delete_space.ques + delete_und.ques) + | (digit_cluster + delete_space.ques + pynini.cross(sextillion, ".") + delete_und.ques) + | pynutil.insert("000.") + ) + + delete_space.ques + + quintillions + ) + + # Remove the leading zeros + non_zero_digits = pynini.difference(NEMO_DIGIT, "0") + chars_to_remove = pynini.accep("0") | pynini.accep(".") + remove_chars = pynutil.delete(pynini.closure(chars_to_remove)) + remove_leading_zeros = pynini.cdrewrite(remove_chars, "[BOS]", non_zero_digits, NEMO_SIGMA) + + # All together now + grammars = [ + sextillions, + quintillions, + quadrillions, + trillions, + billions, + millions, + thousands, + digit_cluster, + zero, + ] + + graph_cardinals = "" + for grammar in grammars: + graph_cardinals |= grammar + # Generates a graph accepting all digits to be passed to other semiotic classes + graph_everything = graph_cardinals @ remove_leading_zeros + self.graph_all_cardinals = graph_everything.optimize() + + # the name the other German semiotic classes use for the graph without the first-dozen exception + self.graph_no_exception = self.graph_all_cardinals + + # 1-999 without leading zeros, consumed by the decimal tagger's get_quantity + self.graph_hundred_component_at_least_one_none_zero_digit = ( + non_zero_digit_cluster @ remove_leading_zeros + ).optimize() + + # The block below leaves numerals 1 - 12 canonically normalized + accept_denormalized_first_dozen = pynini.project(to_denormalize, "input") # acceptor for null - zwölf + accept_denormalized_everything = pynini.project( + self.graph_all_cardinals, "input" + ) # acceptor for all verbalized cardinals + accept_without_first_dozen = ( + accept_denormalized_everything - accept_denormalized_first_dozen + ) # acceptor for all verbalized cardinals greater than 12 + transduce_without_first_dozen = ( + accept_without_first_dozen @ self.graph_all_cardinals + ) # transducer for all verbalized cardinals greater than 12 + graph = accept_denormalized_first_dozen | transduce_without_first_dozen + self.graph = graph.optimize() + + self.optional_negative = pynini.closure( + pynutil.insert("negative: ") + pynini.cross("minus ", '"true"') + pynutil.insert(" "), + 0, + 1, + ) + + # the decimal verbalizer reads a single character out of the negative field, so the graph + # handed to the other classes keeps the minus sign rather than the "true" flag self.optional_minus_graph = pynini.closure( - pynutil.insert("negative: ") + pynini.cross("minus ", "\"-\" "), 0, 1 + pynutil.insert("negative: ") + pynini.cross("minus ", '"true"') + pynutil.insert(" "), 0, 1 + ) + + all_cardinals_graph = ( + self.optional_negative + pynutil.insert('integer: "') + self.graph_all_cardinals + pynutil.insert('"') ) + self.all_cardinals_graph = all_cardinals_graph.optimize() - final_graph = self.optional_minus_graph + pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + # The final graph for this semiotic class leaves the first dozen normalized + final_graph = self.optional_negative + pynutil.insert('integer: "') + self.graph + pynutil.insert('"') + # Canonical representation with the first dozen normalized + self.canonical_cardinals_graph = final_graph.optimize() final_graph = self.add_tokens(final_graph) self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py index 1d60d071a..e0dbc4a42 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py @@ -93,7 +93,7 @@ def __init__( tn_electronic_verbalizer = TNElectronicVerbalizer(deterministic=False) tn_whitelist_tagger = TNWhitelistTagger(input_case="cased", deterministic=False, input_file=whitelist) - cardinal = CardinalFst(tn_cardinal_tagger=tn_cardinal_tagger) + cardinal = CardinalFst() cardinal_graph = cardinal.fst ordinal = OrdinalFst(itn_cardinal_tagger=cardinal, tn_ordinal_verbalizer=tn_ordinal_verbalizer) diff --git a/nemo_text_processing/inverse_text_normalization/de/utils.py b/nemo_text_processing/inverse_text_normalization/de/utils.py new file mode 100644 index 000000000..78fdd87f2 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/utils.py @@ -0,0 +1,27 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + "/" + rel_path diff --git a/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py index b13382a8e..ccd961ef1 100644 --- a/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,22 +15,32 @@ import pynini from pynini.lib import pynutil -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst +from nemo_text_processing.inverse_text_normalization.de.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space class CardinalFst(GraphFst): """ - Finite state transducer for verbalizing cardinal - e.g. cardinal { integer: "23" negative: "-" } -> -23 - - Args: - tn_cardinal_verbalizer: TN cardinal verbalizer + Finite state transducer for verbalizing cardinal numbers. Note that the verbalizer retains period-separated formatting. + e.g. 'cardinal { negative: "true" integer: "1.234.512.102" }' -> -1.234.512.102 """ - def __init__(self, tn_cardinal_verbalizer: GraphFst, deterministic: bool = True): - super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) - self.numbers = tn_cardinal_verbalizer.numbers - optional_sign = pynini.closure(pynutil.delete("negative: \"") + NEMO_NOT_QUOTE + pynutil.delete("\" "), 0, 1) - graph = optional_sign + self.numbers + def __init__(self): + super().__init__(name="cardinal", kind="verbalize") + + # removes the 'negative:' label and leaves the optional '-' sign in place + optional_minus = pynini.closure(pynini.cross('negative: "true"', "-") + delete_space, 0, 1) + + # removes the 'integer:' label + just_integers = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + + delete_space + ) + + graph = optional_minus + just_integers + self.numbers = graph.optimize() delete_tokens = self.delete_tokens(graph) self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/de/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/de/verbalizers/verbalize.py index 9c921a63a..014777890 100644 --- a/nemo_text_processing/inverse_text_normalization/de/verbalizers/verbalize.py +++ b/nemo_text_processing/inverse_text_normalization/de/verbalizers/verbalize.py @@ -34,7 +34,7 @@ def __init__(self, deterministic: bool = True): tn_cardinal_verbalizer = TNCardinalVerbalizer(deterministic=False) tn_decimal_verbalizer = TNDecimalVerbalizer(deterministic=False) - cardinal = CardinalFst(tn_cardinal_verbalizer=tn_cardinal_verbalizer) + cardinal = CardinalFst() cardinal_graph = cardinal.fst decimal = DecimalFst(tn_decimal_verbalizer=tn_decimal_verbalizer) decimal_graph = decimal.fst diff --git a/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt index 0b2064296..1049b5052 100644 --- a/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt +++ b/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt @@ -4,26 +4,26 @@ ein hundert und zwei~102 einhundertzwei~102 ein hundert und zwanzig~120 ein hundert und elf~111 -ein tausend~1000 -eintausend~1000 +ein tausend~1.000 +eintausend~1.000 ein hundert zwanzig~120 -ein tausend zwanzig~1020 -eintausendzwanzig~1020 -neun billionen sieben hundert neun und achtzig milliarden drei hundert zwei und achtzig millionen fünf hundert sechs und dreißig tausend ein hundert dreißig~9789382536130 +ein tausend zwanzig~1.020 +eintausendzwanzig~1.020 +neun billionen sieben hundert neun und achtzig milliarden drei hundert zwei und achtzig millionen fünf hundert sechs und dreißig tausend ein hundert dreißig~9.789.382.536.130 zwei hundert vier und fünfzig~254 -ein hundert sieben und vierzig tausend vier hundert ein und fünfzig~147451 -eine million ein hundert sechs und fünfzig tausend ein hundert drei und siebzig~1156173 -eine milliarde fünf hundert drei und neunzig millionen zwei und siebzig tausend neun hundert ein und sechzig~1593072961 -sieben und neunzig billiarden acht hundert acht billionen zwei hundert vier und sechzig milliarden sieben hundert zwei und siebzig millionen sieben hundert zwei und neunzig tausend fünf~97808264772792005 -zehn billiarden zehn billionen zehn millionen ein hundert tausend zehn~10010000010100010 -zehn billiarden zehn billionen zehn millionen einhunderttausendzehn~10010000010100010 -minus fünf und zwanzig tausend sieben und dreißig~-25037 -minus fünf und zwanzig tausend sieben und dreißig~-25037 -minus fünfundzwanzigtausendsiebenunddreißig~-25037 -eine billiarde zwei hundert vier und sechzig billionen drei hundert eins milliarden neun hundert acht und dreißig millionen ein hundert vier~1264301938000104 -eine billiarde zweihundertvierundsechzig billionen dreihunderteins milliarden neunhundertachtunddreißig millionen einhundertvier~1264301938000104 +ein hundert sieben und vierzig tausend vier hundert ein und fünfzig~147.451 +eine million ein hundert sechs und fünfzig tausend ein hundert drei und siebzig~1.156.173 +eine milliarde fünf hundert drei und neunzig millionen zwei und siebzig tausend neun hundert ein und sechzig~1.593.072.961 +sieben und neunzig billiarden acht hundert acht billionen zwei hundert vier und sechzig milliarden sieben hundert zwei und siebzig millionen sieben hundert zwei und neunzig tausend fünf~97.808.264.772.792.005 +zehn billiarden zehn billionen zehn millionen ein hundert tausend zehn~10.010.000.010.100.010 +zehn billiarden zehn billionen zehn millionen einhunderttausendzehn~10.010.000.010.100.010 +minus fünf und zwanzig tausend sieben und dreißig~-25.037 +minus fünf und zwanzig tausend sieben und dreißig~-25.037 +minus fünfundzwanzigtausendsiebenunddreißig~-25.037 +eine billiarde zwei hundert vier und sechzig billionen drei hundert eins milliarden neun hundert acht und dreißig millionen ein hundert vier~1.264.301.938.000.104 +eine billiarde zweihundertvierundsechzig billionen dreihunderteins milliarden neunhundertachtunddreißig millionen einhundertvier~1.264.301.938.000.104 minus sechzig~-60 -sechs und vierzig tausend sechs hundert vier und sechzig~46664 +sechsundvierzig tausend sechshundert vierundsechzig~46.664 sechzig~60 null~null eins~eins @@ -31,10 +31,12 @@ ein~ein eine~eine einer~einer zwei~zwei +zwö~zwö +zwo~zwo neun~neun -zehn~10 -elf~11 -zwölf~12 +zehn~zehn +elf~elf +zwölf~zwölf dreizehn~13 vierzehn~14 fünfzehn~15 @@ -49,14 +51,16 @@ sechzig~60 siebzig~70 achtzig~80 neunzig~90 -zwei millionen drei~2000003 -ein tausend dreizehn~1013 -ein tausend eins~1001 -ein tausend ein hundert~1100 -ein tausend sechs und zwanzig~1026 -ein tausend ein hundert sechs und zwanzig~1126 -achtzehn millionen vier hundert fünfzig tausend neun hundert neunzig~18450990 -achtzehn millionen neun hundert vierzig tausend sieben hundert zwei und zwanzig~18940722 -achtzehn millionen sechs hundert neunzig tausend neun hundert sechzehn~18690916 -achtzehn millionen sechshundertneunzigtausendneunhundertsechzehn~18690916 -achtzehn tausend acht hundert achtzig~18880 +zwei millionen drei~2.000.003 +ein tausend dreizehn~1.013 +ein tausend eins~1.001 +ein tausend ein hundert~1.100 +ein tausend sechs und zwanzig~1.026 +ein tausend ein hundert sechs und zwanzig~1.126 +achtzehn millionen vier hundert fünfzig tausend neun hundert neunzig~18.450.990 +achtzehn millionen neun hundert vierzig tausend sieben hundert zwei und zwanzig~18.940.722 +achtzehn millionen sechs hundert neunzig tausend neun hundert sechzehn~18.690.916 +achtzehn millionen sechshundertneunzigtausendneunhundertsechzehn~18.690.916 +achtzehn tausend acht hundert achtzig~18.880 +einhunderteins~101 +ein tausend einhundert sechsundzwanzig~1.126 From ad134533c0dcdf83d03dc6a4b59ef52ebd2b7145 Mon Sep 17 00:00:00 2001 From: Adelina Dunina Date: Mon, 14 Sep 2026 08:46:33 -0400 Subject: [PATCH 02/13] De ITN: split regular and irregular teens Signed-off-by: Adelina Dunina --- .../de/data/cardinal/{teens.tsv => regular_teens.tsv} | 3 --- 1 file changed, 3 deletions(-) rename nemo_text_processing/inverse_text_normalization/de/data/cardinal/{teens.tsv => regular_teens.tsv} (77%) diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/teens.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/regular_teens.tsv similarity index 77% rename from nemo_text_processing/inverse_text_normalization/de/data/cardinal/teens.tsv rename to nemo_text_processing/inverse_text_normalization/de/data/cardinal/regular_teens.tsv index 854712ea6..acf4411a7 100644 --- a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/teens.tsv +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/regular_teens.tsv @@ -1,6 +1,3 @@ -zehn 10 -elf 11 -zwölf 12 dreizehn 13 vierzehn 14 fünfzehn 15 From df35324edf20fcc6509866b395aa99c571c19953 Mon Sep 17 00:00:00 2001 From: Adelina Dunina Date: Mon, 14 Sep 2026 10:44:30 -0400 Subject: [PATCH 03/13] De ITN: address review comments on cardinal grammar Signed-off-by: Adelina Dunina --- .../de/taggers/cardinal.py | 84 +++++++++++-------- .../de/verbalizers/cardinal.py | 6 +- 2 files changed, 51 insertions(+), 39 deletions(-) diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py index 4a71159e9..3fb81c81c 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py @@ -16,7 +16,12 @@ import pynini from pynini.lib import pynutil -from nemo_text_processing.inverse_text_normalization.de.graph_utils import NEMO_DIGIT, NEMO_SIGMA, NEMO_SPACE, GraphFst +from nemo_text_processing.inverse_text_normalization.de.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + GraphFst, + delete_space, +) from nemo_text_processing.inverse_text_normalization.de.utils import get_abs_path @@ -33,11 +38,18 @@ def swap_tens_and_ones(digits: 'pynini.FstLike', tens: 'pynini.FstLike') -> 'pyn class CardinalFst(GraphFst): """ - Finite state transducer for classifying cardinals + Finite state transducer for classifying cardinals. Numbers below thirteen are not converted. + Allows both compound numeral strings or separated by whitespace. + "und" (en: "and") can be inserted between "hundert" and following number or "tausend" and following single or double digit number. + + e.g. minus drei und zwanzig -> cardinal { negative: "true" integer: "23" } e.g. minus dreiundzwanzig -> cardinal { negative: "true" integer: "23" } + e.g. dreizehn -> cardinal { integer: "13" } + e.g. ein hundert -> cardinal { integer: "100" } + e.g. einhundert -> cardinal { integer: "100" } + e.g. ein tausend -> cardinal { integer: "1.000" } e.g. eintausend -> cardinal { integer: "1.000" } - Numbers below thirteen are not converted. - The transducer implements a period separator every three digits by default. + e.g. ein tausend zwanzig -> cardinal { integer: "1.020" } """ def __init__(self): @@ -53,20 +65,20 @@ def __init__(self): # Isolates the first dozen self.dozen = to_denormalize.optimize() - teens = pynini.string_file(get_abs_path("data/cardinal/teens.tsv")) + regular_teens = pynini.string_file(get_abs_path("data/cardinal/regular_teens.tsv")) + teens = irregular_teens | regular_teens tens = pynini.string_file(get_abs_path("data/cardinal/tens.tsv")) ties = tens + pynutil.insert("0") # German flips ones and tens in two-digit numbers. The WFST below handles these flips. - delete_space = pynutil.delete(NEMO_SPACE) delete_und = pynutil.delete("und") # Accepts normalized digits+ties (ein+und+zwanzig) - digit_ties = digits + delete_space.ques + delete_und + delete_space.ques + tens + digit_ties = digits + delete_space + delete_und + delete_space + tens # Flips ties and digits for denormalization ties_digit = digit_ties @ swap_tens_and_ones(digits, tens) # WFST grammar for hundreds - graph_10_99 = irregular_teens | teens | ties | ties_digit + graph_10_99 = teens | ties | ties_digit self.graph_double_digits = graph_10_99 # Isolates single and double-digit cardinals to pass to other graphs graph_single_and_double_digits = digits | graph_10_99 @@ -76,23 +88,23 @@ def __init__(self): hundreds = (pynini.cross(hundert, "100")) | ( ( (digits | pynutil.insert("1")) - + delete_space.ques + + delete_space + pynutil.delete("hundert") - + delete_space.ques + + delete_space + delete_und.ques - + delete_space.ques + + delete_space + graph_10_99 ) | ( (digits | pynutil.insert("1")) - + delete_space.ques + + delete_space + pynini.cross("hundert", "0") - + delete_space.ques + + delete_space + delete_und.ques - + delete_space.ques + + delete_space + digits ) - | ((digits | pynutil.insert("1")) + delete_space.ques + pynini.cross("hundert", "00")) + | ((digits | pynutil.insert("1")) + delete_space + pynini.cross("hundert", "00")) ) # Digits are grouped in clusters of three: {hundreds}{tens}{ones}. @@ -103,11 +115,11 @@ def __init__(self): # WFST grammar for thousands thousands = (pynini.cross("tausend", "1.000")) | ( ( - (pynini.cross("tausend", "1.") + delete_space.ques + delete_und.ques) - | (digit_cluster + delete_space.ques + pynini.cross("tausend", ".") + delete_und.ques) + (pynini.cross("tausend", "1.") + delete_space + delete_und.ques) + | (digit_cluster + delete_space + pynini.cross("tausend", ".") + delete_und.ques) | pynutil.insert("000.") ) - + delete_space.ques + + delete_space + digit_cluster ) @@ -115,11 +127,11 @@ def __init__(self): million = pynini.accep("million") | pynini.accep("millionen") millions = (pynini.cross("million", "1.000.000")) | ( ( - (pynini.cross("million", "1.") + delete_space.ques + delete_und.ques) - | (digit_cluster + delete_space.ques + pynini.cross(million, ".") + delete_und.ques) + (pynini.cross("million", "1.") + delete_space + delete_und.ques) + | (digit_cluster + delete_space + pynini.cross(million, ".") + delete_und.ques) | pynutil.insert("000.") ) - + delete_space.ques + + delete_space + thousands ) @@ -135,13 +147,13 @@ def __init__(self): ( ( pynini.cross((pynini.accep("milliarde") | pynini.accep("milliard")), "1.") - + delete_space.ques + + delete_space + delete_und.ques ) - | (digit_cluster + delete_space.ques + pynini.cross(billion, ".") + delete_und.ques) + | (digit_cluster + delete_space + pynini.cross(billion, ".") + delete_und.ques) | pynutil.insert("000.") ) - + delete_space.ques + + delete_space + millions ) @@ -149,11 +161,11 @@ def __init__(self): trillion = pynini.accep("billion") | pynini.accep("billionen") trillions = (pynini.cross("billion", "1.000.000.000.000")) | ( ( - (pynini.cross("billion", "1.") + delete_space.ques + delete_und.ques) - | (digit_cluster + delete_space.ques + pynini.cross(trillion, ".") + delete_und.ques) + (pynini.cross("billion", "1.") + delete_space + delete_und.ques) + | (digit_cluster + delete_space + pynini.cross(trillion, ".") + delete_und.ques) | pynutil.insert("000.") ) - + delete_space.ques + + delete_space + billions ) @@ -167,11 +179,11 @@ def __init__(self): ) quadrillions = (pynini.cross("billiarde", "1.000.000.000.000.000")) | ( ( - (pynini.cross(quadrillion, "1.") + delete_space.ques + delete_und.ques) - | (digit_cluster + delete_space.ques + pynini.cross(quadrillion, ".") + delete_und.ques) + (pynini.cross(quadrillion, "1.") + delete_space + delete_und.ques) + | (digit_cluster + delete_space + pynini.cross(quadrillion, ".") + delete_und.ques) | pynutil.insert("000.") ) - + delete_space.ques + + delete_space + trillions ) @@ -179,11 +191,11 @@ def __init__(self): quintillion = pynini.accep("trillion") | pynini.accep("trillionen") quintillions = (pynini.cross("trillion", "1.000.000.000.000.000.000")) | ( ( - (pynini.cross("trillion", "1.") + delete_space.ques + delete_und.ques) - | (digit_cluster + delete_space.ques + pynini.cross(quintillion, ".") + delete_und.ques) + (pynini.cross("trillion", "1.") + delete_space + delete_und.ques) + | (digit_cluster + delete_space + pynini.cross(quintillion, ".") + delete_und.ques) | pynutil.insert("000.") ) - + delete_space.ques + + delete_space + quadrillions ) @@ -197,11 +209,11 @@ def __init__(self): ) sextillions = (pynini.cross("billiarde", "1.000.000.000.000.000.000.000")) | ( ( - (pynini.cross(sextillion, "1.") + delete_space.ques + delete_und.ques) - | (digit_cluster + delete_space.ques + pynini.cross(sextillion, ".") + delete_und.ques) + (pynini.cross(sextillion, "1.") + delete_space + delete_und.ques) + | (digit_cluster + delete_space + pynini.cross(sextillion, ".") + delete_und.ques) | pynutil.insert("000.") ) - + delete_space.ques + + delete_space + quintillions ) diff --git a/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py index ccd961ef1..1eab3bab2 100644 --- a/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,8 +20,8 @@ class CardinalFst(GraphFst): """ - Finite state transducer for verbalizing cardinal numbers. Note that the verbalizer retains period-separated formatting. - e.g. 'cardinal { negative: "true" integer: "1.234.512.102" }' -> -1.234.512.102 + Finite state transducer for verbalizing cardinal Note that the verbalizer retains period-separated formatting. + e.g. cardinal { negative: "true" integer: "23" } -> -23 """ def __init__(self): From 5921ed5eb05889c0e9aa236dd808f43cf34c80f6 Mon Sep 17 00:00:00 2001 From: Adelina Dunina Date: Mon, 14 Sep 2026 10:50:42 -0400 Subject: [PATCH 04/13] De ITN: address review comments on cardinal grammar Signed-off-by: Adelina Dunina --- .../inverse_text_normalization/de/graph_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo_text_processing/inverse_text_normalization/de/graph_utils.py b/nemo_text_processing/inverse_text_normalization/de/graph_utils.py index 8dc9c6391..cf281bad8 100644 --- a/nemo_text_processing/inverse_text_normalization/de/graph_utils.py +++ b/nemo_text_processing/inverse_text_normalization/de/graph_utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. # Copyright 2015 and onwards Google, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); From e6abccbdf1a4a89367b2508864fa68375e64110e Mon Sep 17 00:00:00 2001 From: Adelina Dunina Date: Mon, 14 Sep 2026 10:57:26 -0400 Subject: [PATCH 05/13] De ITN: address review comments on cardinal grammar Signed-off-by: Adelina Dunina --- .../inverse_text_normalization/de/verbalizers/cardinal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py index 1eab3bab2..2f30e8808 100644 --- a/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py @@ -20,7 +20,7 @@ class CardinalFst(GraphFst): """ - Finite state transducer for verbalizing cardinal Note that the verbalizer retains period-separated formatting. + Finite state transducer for verbalizing cardinal e.g. cardinal { negative: "true" integer: "23" } -> -23 """ From c5130fa8341a4e94b956297cd96696594876be95 Mon Sep 17 00:00:00 2001 From: Adelina Dunina Date: Mon, 14 Sep 2026 13:45:14 -0400 Subject: [PATCH 06/13] De ITN: address review comments on cardinal grammar Signed-off-by: Adelina Dunina --- .../de/taggers/cardinal.py | 56 ++++++++++++++----- .../inverse_text_normalization/de/utils.py | 15 +++++ 2 files changed, 58 insertions(+), 13 deletions(-) diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py index 3fb81c81c..e7149f41d 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from collections import defaultdict import pynini from pynini.lib import pynutil @@ -19,21 +20,49 @@ from nemo_text_processing.inverse_text_normalization.de.graph_utils import ( NEMO_DIGIT, NEMO_SIGMA, + NEMO_WHITE_SPACE, GraphFst, delete_space, ) -from nemo_text_processing.inverse_text_normalization.de.utils import get_abs_path +from nemo_text_processing.inverse_text_normalization.de.utils import get_abs_path, load_labels -def swap_tens_and_ones(digits: 'pynini.FstLike', tens: 'pynini.FstLike') -> 'pynini.FstLike': +AND = "und" + + +def get_tens_digit(digit_path: str, tens_path: str) -> 'pynini.FstLike': """ - German says the ones digit before the tens digit (ein-und-zwanzig = 21), so digits - arrive reversed. An FST cannot reorder without enumerating, so this enumerates every - ones/tens pair present in the digit and tens tables. + getting all denormalizations for numbers between 21 - 99. German says the ones digit + before the tens digit (ein-und-zwanzig = 21), so the words cannot be read left to right + + Args: + digit_path: file to digits tsv + tens_path: file to tens tsv, e.g. zwanzig -> 2 + Returns: + res: fst that converts the verbalization of a number to its digits """ - ones_digits = sorted({output for _, output, _ in digits.paths().items()}) - tens_digits = sorted({output for _, output, _ in tens.paths().items()}) - return pynini.string_map([(one + ten, ten + one) for one in ones_digits for ten in tens_digits]) + + digits = defaultdict(list) + ties = defaultdict(list) + for k, v in load_labels(digit_path): + digits[v].append(k) + + for k, v in load_labels(tens_path): + ties[v].append(k) + + d = [] + for i in range(21, 100): + s = str(i) + if s[1] == "0": + continue + + for di in digits[s[1]]: + for ti in ties[s[0]]: + word = di + AND + ti + d.append((word, s)) + + res = pynini.string_map(d) + return res class CardinalFst(GraphFst): @@ -70,12 +99,13 @@ def __init__(self): tens = pynini.string_file(get_abs_path("data/cardinal/tens.tsv")) ties = tens + pynutil.insert("0") # German flips ones and tens in two-digit numbers. The WFST below handles these flips. - delete_und = pynutil.delete("und") + delete_und = pynutil.delete(AND) - # Accepts normalized digits+ties (ein+und+zwanzig) - digit_ties = digits + delete_space + delete_und + delete_space + tens - # Flips ties and digits for denormalization - ties_digit = digit_ties @ swap_tens_and_ones(digits, tens) + # the map is keyed on the compound spelling, so whitespace is stripped before lookup + delete_all_spaces = pynini.cdrewrite(pynutil.delete(NEMO_WHITE_SPACE), "", "", NEMO_SIGMA) + ties_digit = delete_all_spaces @ get_tens_digit( + get_abs_path("data/cardinal/digits.tsv"), get_abs_path("data/cardinal/tens.tsv") + ) # WFST grammar for hundreds graph_10_99 = teens | ties | ties_digit diff --git a/nemo_text_processing/inverse_text_normalization/de/utils.py b/nemo_text_processing/inverse_text_normalization/de/utils.py index 78fdd87f2..144bd9208 100644 --- a/nemo_text_processing/inverse_text_normalization/de/utils.py +++ b/nemo_text_processing/inverse_text_normalization/de/utils.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import csv import os @@ -25,3 +26,17 @@ def get_abs_path(rel_path): Returns absolute path """ return os.path.dirname(os.path.abspath(__file__)) + "/" + rel_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels From 4476e9972178c5ade4859a2fb49e1fc8fdcaab09 Mon Sep 17 00:00:00 2001 From: Adelina Dunina Date: Tue, 15 Sep 2026 13:30:01 -0400 Subject: [PATCH 07/13] De ITN: fix cardinal tagger bugs and extend test coverage Signed-off-by: Adelina Dunina --- .../de/taggers/cardinal.py | 106 +++++------------- .../test_cases_cardinal.txt | 29 ++++- 2 files changed, 56 insertions(+), 79 deletions(-) diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py index e7149f41d..4691a0af8 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py @@ -87,13 +87,9 @@ def __init__(self): # WFST mappings for numbers 0-99 zero = pynini.string_file(get_abs_path("data/cardinal/zero.tsv")) digits = pynini.string_file(get_abs_path("data/cardinal/digits.tsv")) - # Isolates single digit cardinals to pass to other graphs - self.digits = digits.optimize() irregular_teens = pynini.string_file(get_abs_path("data/cardinal/irregular_teens.tsv")) to_denormalize = zero | digits | irregular_teens - # Isolates the first dozen - self.dozen = to_denormalize.optimize() regular_teens = pynini.string_file(get_abs_path("data/cardinal/regular_teens.tsv")) teens = irregular_teens | regular_teens tens = pynini.string_file(get_abs_path("data/cardinal/tens.tsv")) @@ -109,13 +105,8 @@ def __init__(self): # WFST grammar for hundreds graph_10_99 = teens | ties | ties_digit - self.graph_double_digits = graph_10_99 - # Isolates single and double-digit cardinals to pass to other graphs - graph_single_and_double_digits = digits | graph_10_99 - self.graph_single_and_double_digits = graph_single_and_double_digits.optimize() - hundert = pynini.accep("hundert") | pynini.accep("ein hundert") - hundreds = (pynini.cross(hundert, "100")) | ( + hundreds = ( ( (digits | pynutil.insert("1")) + delete_space @@ -141,12 +132,13 @@ def __init__(self): # Clusters of three are separated by periods, applied right to left. non_zero_digit_cluster = (hundreds) | (pynutil.insert("0") + graph_10_99) | (pynutil.insert("00") + digits) digit_cluster = non_zero_digit_cluster | pynutil.insert("000") + # a magnitude word with no multiplier in front of it means "one" of that magnitude + leading_cluster = non_zero_digit_cluster | pynutil.insert("001") # WFST grammar for thousands - thousands = (pynini.cross("tausend", "1.000")) | ( + thousands = ( ( - (pynini.cross("tausend", "1.") + delete_space + delete_und.ques) - | (digit_cluster + delete_space + pynini.cross("tausend", ".") + delete_und.ques) + (leading_cluster + delete_space + pynini.cross("tausend", ".") + delete_space + delete_und.ques) | pynutil.insert("000.") ) + delete_space @@ -155,10 +147,9 @@ def __init__(self): # WFST grammar for millions million = pynini.accep("million") | pynini.accep("millionen") - millions = (pynini.cross("million", "1.000.000")) | ( + millions = ( ( - (pynini.cross("million", "1.") + delete_space + delete_und.ques) - | (digit_cluster + delete_space + pynini.cross(million, ".") + delete_und.ques) + (leading_cluster + delete_space + pynini.cross(million, ".") + delete_space + delete_und.ques) | pynutil.insert("000.") ) + delete_space @@ -166,21 +157,10 @@ def __init__(self): ) # WFST grammar for billions - billion = ( - pynini.accep("milliarde") - | pynini.accep("milliarden") - # include the consonant-final stem for ordinal declensions e.g "milliardste" - # "e" -> "" / _[ordinal morpheme] - | pynini.accep("milliard") - ) - billions = (pynini.cross("milliarde", "1.000.000.000")) | ( + billion = pynini.accep("milliarde") | pynini.accep("milliarden") + billions = ( ( - ( - pynini.cross((pynini.accep("milliarde") | pynini.accep("milliard")), "1.") - + delete_space - + delete_und.ques - ) - | (digit_cluster + delete_space + pynini.cross(billion, ".") + delete_und.ques) + (leading_cluster + delete_space + pynini.cross(billion, ".") + delete_space + delete_und.ques) | pynutil.insert("000.") ) + delete_space @@ -189,10 +169,9 @@ def __init__(self): # WFST grammar for trillions trillion = pynini.accep("billion") | pynini.accep("billionen") - trillions = (pynini.cross("billion", "1.000.000.000.000")) | ( + trillions = ( ( - (pynini.cross("billion", "1.") + delete_space + delete_und.ques) - | (digit_cluster + delete_space + pynini.cross(trillion, ".") + delete_und.ques) + (leading_cluster + delete_space + pynini.cross(trillion, ".") + delete_space + delete_und.ques) | pynutil.insert("000.") ) + delete_space @@ -200,17 +179,10 @@ def __init__(self): ) # WFST grammar for quadrillions - quadrillion = ( - pynini.accep("billiarde") - | pynini.accep("billiarden") - # include the consonant-final stem for ordinal declensions e.g "billiardste" - # "e" -> "" / _[ordinal morpheme] - | pynini.accep("billiard") - ) - quadrillions = (pynini.cross("billiarde", "1.000.000.000.000.000")) | ( + quadrillion = pynini.accep("billiarde") | pynini.accep("billiarden") + quadrillions = ( ( - (pynini.cross(quadrillion, "1.") + delete_space + delete_und.ques) - | (digit_cluster + delete_space + pynini.cross(quadrillion, ".") + delete_und.ques) + (leading_cluster + delete_space + pynini.cross(quadrillion, ".") + delete_space + delete_und.ques) | pynutil.insert("000.") ) + delete_space @@ -219,10 +191,9 @@ def __init__(self): # WFST grammar for quintillions quintillion = pynini.accep("trillion") | pynini.accep("trillionen") - quintillions = (pynini.cross("trillion", "1.000.000.000.000.000.000")) | ( + quintillions = ( ( - (pynini.cross("trillion", "1.") + delete_space + delete_und.ques) - | (digit_cluster + delete_space + pynini.cross(quintillion, ".") + delete_und.ques) + (leading_cluster + delete_space + pynini.cross(quintillion, ".") + delete_space + delete_und.ques) | pynutil.insert("000.") ) + delete_space @@ -230,17 +201,10 @@ def __init__(self): ) # WFST grammar for sextillions - sextillion = ( - pynini.accep("trilliarde") - | pynini.accep("trilliarden") - # include the consonant-final stem for ordinal declensions e.g "trilliardste" - # "e" -> "" / _[ordinal morpheme] - | pynini.accep("trilliard") - ) - sextillions = (pynini.cross("billiarde", "1.000.000.000.000.000.000.000")) | ( + sextillion = pynini.accep("trilliarde") | pynini.accep("trilliarden") + sextillions = ( ( - (pynini.cross(sextillion, "1.") + delete_space + delete_und.ques) - | (digit_cluster + delete_space + pynini.cross(sextillion, ".") + delete_und.ques) + (leading_cluster + delete_space + pynini.cross(sextillion, ".") + delete_space + delete_und.ques) | pynutil.insert("000.") ) + delete_space @@ -270,12 +234,8 @@ def __init__(self): for grammar in grammars: graph_cardinals |= grammar - # Generates a graph accepting all digits to be passed to other semiotic classes - graph_everything = graph_cardinals @ remove_leading_zeros - self.graph_all_cardinals = graph_everything.optimize() - - # the name the other German semiotic classes use for the graph without the first-dozen exception - self.graph_no_exception = self.graph_all_cardinals + # the graph the other German semiotic classes consume, without the first-dozen exception + self.graph_no_exception = (graph_cardinals @ remove_leading_zeros).optimize() # 1-999 without leading zeros, consumed by the decimal tagger's get_quantity self.graph_hundred_component_at_least_one_none_zero_digit = ( @@ -285,38 +245,30 @@ def __init__(self): # The block below leaves numerals 1 - 12 canonically normalized accept_denormalized_first_dozen = pynini.project(to_denormalize, "input") # acceptor for null - zwölf accept_denormalized_everything = pynini.project( - self.graph_all_cardinals, "input" + self.graph_no_exception, "input" ) # acceptor for all verbalized cardinals accept_without_first_dozen = ( accept_denormalized_everything - accept_denormalized_first_dozen ) # acceptor for all verbalized cardinals greater than 12 transduce_without_first_dozen = ( - accept_without_first_dozen @ self.graph_all_cardinals + accept_without_first_dozen @ self.graph_no_exception ) # transducer for all verbalized cardinals greater than 12 graph = accept_denormalized_first_dozen | transduce_without_first_dozen self.graph = graph.optimize() - self.optional_negative = pynini.closure( - pynutil.insert("negative: ") + pynini.cross("minus ", '"true"') + pynutil.insert(" "), - 0, - 1, + # the cardinal verbalizer turns the "true" flag back into a minus sign + optional_negative = pynini.closure( + pynutil.insert("negative: ") + pynini.cross("minus ", '"true"') + pynutil.insert(" "), 0, 1 ) # the decimal verbalizer reads a single character out of the negative field, so the graph # handed to the other classes keeps the minus sign rather than the "true" flag self.optional_minus_graph = pynini.closure( - pynutil.insert("negative: ") + pynini.cross("minus ", '"true"') + pynutil.insert(" "), 0, 1 - ) - - all_cardinals_graph = ( - self.optional_negative + pynutil.insert('integer: "') + self.graph_all_cardinals + pynutil.insert('"') + pynutil.insert("negative: ") + pynini.cross("minus ", '"-"') + pynutil.insert(" "), 0, 1 ) - self.all_cardinals_graph = all_cardinals_graph.optimize() # The final graph for this semiotic class leaves the first dozen normalized - final_graph = self.optional_negative + pynutil.insert('integer: "') + self.graph + pynutil.insert('"') - # Canonical representation with the first dozen normalized - self.canonical_cardinals_graph = final_graph.optimize() + final_graph = optional_negative + pynutil.insert('integer: "') + self.graph + pynutil.insert('"') final_graph = self.add_tokens(final_graph) self.fst = final_graph.optimize() diff --git a/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt index 1049b5052..29fd87359 100644 --- a/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt +++ b/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt @@ -6,25 +6,30 @@ ein hundert und zwanzig~120 ein hundert und elf~111 ein tausend~1.000 eintausend~1.000 +ein hundert zwölf~112 ein hundert zwanzig~120 +hundert zwanzig~120 ein tausend zwanzig~1.020 eintausendzwanzig~1.020 neun billionen sieben hundert neun und achtzig milliarden drei hundert zwei und achtzig millionen fünf hundert sechs und dreißig tausend ein hundert dreißig~9.789.382.536.130 zwei hundert vier und fünfzig~254 +zwo hundert~200 +zwoundzwanzig~22 +zwöundzwanzig~22 ein hundert sieben und vierzig tausend vier hundert ein und fünfzig~147.451 eine million ein hundert sechs und fünfzig tausend ein hundert drei und siebzig~1.156.173 eine milliarde fünf hundert drei und neunzig millionen zwei und siebzig tausend neun hundert ein und sechzig~1.593.072.961 sieben und neunzig billiarden acht hundert acht billionen zwei hundert vier und sechzig milliarden sieben hundert zwei und siebzig millionen sieben hundert zwei und neunzig tausend fünf~97.808.264.772.792.005 zehn billiarden zehn billionen zehn millionen ein hundert tausend zehn~10.010.000.010.100.010 zehn billiarden zehn billionen zehn millionen einhunderttausendzehn~10.010.000.010.100.010 -minus fünf und zwanzig tausend sieben und dreißig~-25.037 +eine trillion ein hundert~1.000.000.000.000.000.100 +elf trilliarden drei tausend sieben und fünfzig~11.000.000.000.000.000.003.057 minus fünf und zwanzig tausend sieben und dreißig~-25.037 minus fünfundzwanzigtausendsiebenunddreißig~-25.037 eine billiarde zwei hundert vier und sechzig billionen drei hundert eins milliarden neun hundert acht und dreißig millionen ein hundert vier~1.264.301.938.000.104 eine billiarde zweihundertvierundsechzig billionen dreihunderteins milliarden neunhundertachtunddreißig millionen einhundertvier~1.264.301.938.000.104 minus sechzig~-60 sechsundvierzig tausend sechshundert vierundsechzig~46.664 -sechzig~60 null~null eins~eins ein~ein @@ -33,6 +38,12 @@ einer~einer zwei~zwei zwö~zwö zwo~zwo +drei~drei +vier~vier +fünf~fünf +sechs~sechs +sieben~sieben +acht~acht neun~neun zehn~zehn elf~elf @@ -43,6 +54,7 @@ fünfzehn~15 sechzehn~16 siebzehn~17 achtzehn~18 +neunzehn~19 zwanzig~20 dreißig~30 vierzig~40 @@ -51,7 +63,11 @@ sechzig~60 siebzig~70 achtzig~80 neunzig~90 +hundert~100 +eine million tausend~1.001.000 zwei millionen drei~2.000.003 +tausend~1.000 +tausend zwanzig~1.020 ein tausend dreizehn~1.013 ein tausend eins~1.001 ein tausend ein hundert~1.100 @@ -61,6 +77,15 @@ achtzehn millionen vier hundert fünfzig tausend neun hundert neunzig~18.450.990 achtzehn millionen neun hundert vierzig tausend sieben hundert zwei und zwanzig~18.940.722 achtzehn millionen sechs hundert neunzig tausend neun hundert sechzehn~18.690.916 achtzehn millionen sechshundertneunzigtausendneunhundertsechzehn~18.690.916 +zwei millionen ein tausend acht~2.001.008 achtzehn tausend acht hundert achtzig~18.880 einhunderteins~101 ein tausend einhundert sechsundzwanzig~1.126 +ein tausend und zwanzig~1.020 +tausend ein hundert~1.100 +eine milliarde tausend~1.000.001.000 +zwei millionen tausend drei~2.001.003 +minus drei~-drei +trillion~1.000.000.000.000.000.000 +eine million und zwanzig~1.000.020 +eine trilliarde zwei trillionen eine billion vier~1.002.000.001.000.000.000.004 \ No newline at end of file From 512f2ec602220ece629e8c92859ab1a188fed600 Mon Sep 17 00:00:00 2001 From: Adelina Dunina Date: Tue, 15 Sep 2026 13:31:58 -0400 Subject: [PATCH 08/13] De ITN: align copyright year in de ITN utils Signed-off-by: Adelina Dunina --- nemo_text_processing/inverse_text_normalization/de/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo_text_processing/inverse_text_normalization/de/utils.py b/nemo_text_processing/inverse_text_normalization/de/utils.py index 144bd9208..461631f46 100644 --- a/nemo_text_processing/inverse_text_normalization/de/utils.py +++ b/nemo_text_processing/inverse_text_normalization/de/utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 6fb86825a9098bd99c8a5ad7957fda1baf5c8d88 Mon Sep 17 00:00:00 2001 From: Adelina Dunina Date: Tue, 15 Sep 2026 13:44:51 -0400 Subject: [PATCH 09/13] De ITN: move the cardinal conjunction into a tsv file Signed-off-by: Adelina Dunina --- .../de/data/cardinal/conjunction.tsv | 1 + .../de/taggers/cardinal.py | 16 +++++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/conjunction.tsv diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/conjunction.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/conjunction.tsv new file mode 100644 index 000000000..2babcce6d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/conjunction.tsv @@ -0,0 +1 @@ +und \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py index 4691a0af8..d61c2b9ba 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py @@ -27,10 +27,7 @@ from nemo_text_processing.inverse_text_normalization.de.utils import get_abs_path, load_labels -AND = "und" - - -def get_tens_digit(digit_path: str, tens_path: str) -> 'pynini.FstLike': +def get_tens_digit(digit_path: str, tens_path: str, conjunction_path: str) -> 'pynini.FstLike': """ getting all denormalizations for numbers between 21 - 99. German says the ones digit before the tens digit (ein-und-zwanzig = 21), so the words cannot be read left to right @@ -38,10 +35,12 @@ def get_tens_digit(digit_path: str, tens_path: str) -> 'pynini.FstLike': Args: digit_path: file to digits tsv tens_path: file to tens tsv, e.g. zwanzig -> 2 + conjunction_path: file to the conjunction tsv, e.g. und Returns: res: fst that converts the verbalization of a number to its digits """ + conjunction = load_labels(conjunction_path)[0][0] digits = defaultdict(list) ties = defaultdict(list) for k, v in load_labels(digit_path): @@ -58,7 +57,7 @@ def get_tens_digit(digit_path: str, tens_path: str) -> 'pynini.FstLike': for di in digits[s[1]]: for ti in ties[s[0]]: - word = di + AND + ti + word = di + conjunction + ti d.append((word, s)) res = pynini.string_map(d) @@ -95,12 +94,15 @@ def __init__(self): tens = pynini.string_file(get_abs_path("data/cardinal/tens.tsv")) ties = tens + pynutil.insert("0") # German flips ones and tens in two-digit numbers. The WFST below handles these flips. - delete_und = pynutil.delete(AND) + conjunction = load_labels(get_abs_path("data/cardinal/conjunction.tsv"))[0][0] + delete_und = pynutil.delete(conjunction) # the map is keyed on the compound spelling, so whitespace is stripped before lookup delete_all_spaces = pynini.cdrewrite(pynutil.delete(NEMO_WHITE_SPACE), "", "", NEMO_SIGMA) ties_digit = delete_all_spaces @ get_tens_digit( - get_abs_path("data/cardinal/digits.tsv"), get_abs_path("data/cardinal/tens.tsv") + get_abs_path("data/cardinal/digits.tsv"), + get_abs_path("data/cardinal/tens.tsv"), + get_abs_path("data/cardinal/conjunction.tsv"), ) # WFST grammar for hundreds From 3d8e076681e6e2db7770a0c34bc664263bf2d85a Mon Sep 17 00:00:00 2001 From: Adelina Dunina Date: Tue, 15 Sep 2026 13:54:29 -0400 Subject: [PATCH 10/13] De ITN: make the cardinal conjunction optional via closure Signed-off-by: Adelina Dunina --- .../de/taggers/cardinal.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py index d61c2b9ba..8bba9eaee 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py @@ -95,7 +95,7 @@ def __init__(self): ties = tens + pynutil.insert("0") # German flips ones and tens in two-digit numbers. The WFST below handles these flips. conjunction = load_labels(get_abs_path("data/cardinal/conjunction.tsv"))[0][0] - delete_und = pynutil.delete(conjunction) + optional_delete_und = pynini.closure(pynutil.delete(conjunction), 0, 1) # the map is keyed on the compound spelling, so whitespace is stripped before lookup delete_all_spaces = pynini.cdrewrite(pynutil.delete(NEMO_WHITE_SPACE), "", "", NEMO_SIGMA) @@ -114,7 +114,7 @@ def __init__(self): + delete_space + pynutil.delete("hundert") + delete_space - + delete_und.ques + + optional_delete_und + delete_space + graph_10_99 ) @@ -123,7 +123,7 @@ def __init__(self): + delete_space + pynini.cross("hundert", "0") + delete_space - + delete_und.ques + + optional_delete_und + delete_space + digits ) @@ -140,7 +140,7 @@ def __init__(self): # WFST grammar for thousands thousands = ( ( - (leading_cluster + delete_space + pynini.cross("tausend", ".") + delete_space + delete_und.ques) + (leading_cluster + delete_space + pynini.cross("tausend", ".") + delete_space + optional_delete_und) | pynutil.insert("000.") ) + delete_space @@ -151,7 +151,7 @@ def __init__(self): million = pynini.accep("million") | pynini.accep("millionen") millions = ( ( - (leading_cluster + delete_space + pynini.cross(million, ".") + delete_space + delete_und.ques) + (leading_cluster + delete_space + pynini.cross(million, ".") + delete_space + optional_delete_und) | pynutil.insert("000.") ) + delete_space @@ -162,7 +162,7 @@ def __init__(self): billion = pynini.accep("milliarde") | pynini.accep("milliarden") billions = ( ( - (leading_cluster + delete_space + pynini.cross(billion, ".") + delete_space + delete_und.ques) + (leading_cluster + delete_space + pynini.cross(billion, ".") + delete_space + optional_delete_und) | pynutil.insert("000.") ) + delete_space @@ -173,7 +173,7 @@ def __init__(self): trillion = pynini.accep("billion") | pynini.accep("billionen") trillions = ( ( - (leading_cluster + delete_space + pynini.cross(trillion, ".") + delete_space + delete_und.ques) + (leading_cluster + delete_space + pynini.cross(trillion, ".") + delete_space + optional_delete_und) | pynutil.insert("000.") ) + delete_space @@ -184,7 +184,7 @@ def __init__(self): quadrillion = pynini.accep("billiarde") | pynini.accep("billiarden") quadrillions = ( ( - (leading_cluster + delete_space + pynini.cross(quadrillion, ".") + delete_space + delete_und.ques) + (leading_cluster + delete_space + pynini.cross(quadrillion, ".") + delete_space + optional_delete_und) | pynutil.insert("000.") ) + delete_space @@ -195,7 +195,7 @@ def __init__(self): quintillion = pynini.accep("trillion") | pynini.accep("trillionen") quintillions = ( ( - (leading_cluster + delete_space + pynini.cross(quintillion, ".") + delete_space + delete_und.ques) + (leading_cluster + delete_space + pynini.cross(quintillion, ".") + delete_space + optional_delete_und) | pynutil.insert("000.") ) + delete_space @@ -206,7 +206,7 @@ def __init__(self): sextillion = pynini.accep("trilliarde") | pynini.accep("trilliarden") sextillions = ( ( - (leading_cluster + delete_space + pynini.cross(sextillion, ".") + delete_space + delete_und.ques) + (leading_cluster + delete_space + pynini.cross(sextillion, ".") + delete_space + optional_delete_und) | pynutil.insert("000.") ) + delete_space From 8c61318b72c2f2f7335fa9be005d1af6957dfb02 Mon Sep 17 00:00:00 2001 From: Adelina Dunina Date: Wed, 16 Sep 2026 13:23:47 -0400 Subject: [PATCH 11/13] De ITN: split a conjunction after a magnitude word, extend tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixed: "und" following "hundert", "tausend" or a larger magnitude word no longer folds into the number, it joins two numbers, so "ein hundert und zwei" -> 100 und 2, since German writes 102 as "ein hundert zwei". A signed number is now always written in digits ("minus drei" -> -3), and a sign in front of zero is rejected. Dropped the "zwö" digit variant, since native-speakers don't use it Added: test cases for a conjunction after hundert, tausend, million, milliarde and billion in both glued and spaced spelling, for a signed number carrying a conjunction, and for digit, teen and two-digit composite values on the right-hand side of the conjunction. Signed-off-by: Adelina Dunina --- .../de/data/cardinal/digits.tsv | 1 - .../de/taggers/cardinal.py | 103 +++++++++--------- .../test_cases_cardinal.txt | 35 +++--- 3 files changed, 68 insertions(+), 71 deletions(-) diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/digits.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/digits.tsv index 223c8db47..370e42b6e 100644 --- a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/digits.tsv +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/digits.tsv @@ -4,7 +4,6 @@ ein 1 einer 1 zwei 2 zwo 2 -zwö 2 drei 3 vier 4 fünf 5 diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py index 8bba9eaee..2cd6f7671 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py @@ -66,9 +66,12 @@ def get_tens_digit(digit_path: str, tens_path: str, conjunction_path: str) -> 'p class CardinalFst(GraphFst): """ - Finite state transducer for classifying cardinals. Numbers below thirteen are not converted. + Finite state transducer for classifying cardinals. Numbers below thirteen are not converted + unless they carry a minus sign. Allows both compound numeral strings or separated by whitespace. - "und" (en: "and") can be inserted between "hundert" and following number or "tausend" and following single or double digit number. + "und" (en: "and") after "hundert", "tausend" or a larger magnitude word is never part of the + number, it joins two numbers, whether written glued or spaced: both "einhundertundzwei" and + "ein hundert und zwei" -> 100 und 2. German writes 102 as "ein hundert zwei". e.g. minus drei und zwanzig -> cardinal { negative: "true" integer: "23" } e.g. minus dreiundzwanzig -> cardinal { negative: "true" integer: "23" } @@ -95,7 +98,6 @@ def __init__(self): ties = tens + pynutil.insert("0") # German flips ones and tens in two-digit numbers. The WFST below handles these flips. conjunction = load_labels(get_abs_path("data/cardinal/conjunction.tsv"))[0][0] - optional_delete_und = pynini.closure(pynutil.delete(conjunction), 0, 1) # the map is keyed on the compound spelling, so whitespace is stripped before lookup delete_all_spaces = pynini.cdrewrite(pynutil.delete(NEMO_WHITE_SPACE), "", "", NEMO_SIGMA) @@ -109,24 +111,8 @@ def __init__(self): graph_10_99 = teens | ties | ties_digit hundreds = ( - ( - (digits | pynutil.insert("1")) - + delete_space - + pynutil.delete("hundert") - + delete_space - + optional_delete_und - + delete_space - + graph_10_99 - ) - | ( - (digits | pynutil.insert("1")) - + delete_space - + pynini.cross("hundert", "0") - + delete_space - + optional_delete_und - + delete_space - + digits - ) + ((digits | pynutil.insert("1")) + delete_space + pynutil.delete("hundert") + delete_space + graph_10_99) + | ((digits | pynutil.insert("1")) + delete_space + pynini.cross("hundert", "0") + delete_space + digits) | ((digits | pynutil.insert("1")) + delete_space + pynini.cross("hundert", "00")) ) @@ -139,10 +125,7 @@ def __init__(self): # WFST grammar for thousands thousands = ( - ( - (leading_cluster + delete_space + pynini.cross("tausend", ".") + delete_space + optional_delete_und) - | pynutil.insert("000.") - ) + ((leading_cluster + delete_space + pynini.cross("tausend", ".")) | pynutil.insert("000.")) + delete_space + digit_cluster ) @@ -150,10 +133,7 @@ def __init__(self): # WFST grammar for millions million = pynini.accep("million") | pynini.accep("millionen") millions = ( - ( - (leading_cluster + delete_space + pynini.cross(million, ".") + delete_space + optional_delete_und) - | pynutil.insert("000.") - ) + ((leading_cluster + delete_space + pynini.cross(million, ".")) | pynutil.insert("000.")) + delete_space + thousands ) @@ -161,10 +141,7 @@ def __init__(self): # WFST grammar for billions billion = pynini.accep("milliarde") | pynini.accep("milliarden") billions = ( - ( - (leading_cluster + delete_space + pynini.cross(billion, ".") + delete_space + optional_delete_und) - | pynutil.insert("000.") - ) + ((leading_cluster + delete_space + pynini.cross(billion, ".")) | pynutil.insert("000.")) + delete_space + millions ) @@ -172,10 +149,7 @@ def __init__(self): # WFST grammar for trillions trillion = pynini.accep("billion") | pynini.accep("billionen") trillions = ( - ( - (leading_cluster + delete_space + pynini.cross(trillion, ".") + delete_space + optional_delete_und) - | pynutil.insert("000.") - ) + ((leading_cluster + delete_space + pynini.cross(trillion, ".")) | pynutil.insert("000.")) + delete_space + billions ) @@ -183,10 +157,7 @@ def __init__(self): # WFST grammar for quadrillions quadrillion = pynini.accep("billiarde") | pynini.accep("billiarden") quadrillions = ( - ( - (leading_cluster + delete_space + pynini.cross(quadrillion, ".") + delete_space + optional_delete_und) - | pynutil.insert("000.") - ) + ((leading_cluster + delete_space + pynini.cross(quadrillion, ".")) | pynutil.insert("000.")) + delete_space + trillions ) @@ -194,10 +165,7 @@ def __init__(self): # WFST grammar for quintillions quintillion = pynini.accep("trillion") | pynini.accep("trillionen") quintillions = ( - ( - (leading_cluster + delete_space + pynini.cross(quintillion, ".") + delete_space + optional_delete_und) - | pynutil.insert("000.") - ) + ((leading_cluster + delete_space + pynini.cross(quintillion, ".")) | pynutil.insert("000.")) + delete_space + quadrillions ) @@ -205,10 +173,7 @@ def __init__(self): # WFST grammar for sextillions sextillion = pynini.accep("trilliarde") | pynini.accep("trilliarden") sextillions = ( - ( - (leading_cluster + delete_space + pynini.cross(sextillion, ".") + delete_space + optional_delete_und) - | pynutil.insert("000.") - ) + ((leading_cluster + delete_space + pynini.cross(sextillion, ".")) | pynutil.insert("000.")) + delete_space + quintillions ) @@ -258,10 +223,25 @@ def __init__(self): graph = accept_denormalized_first_dozen | transduce_without_first_dozen self.graph = graph.optimize() - # the cardinal verbalizer turns the "true" flag back into a minus sign - optional_negative = pynini.closure( - pynutil.insert("negative: ") + pynini.cross("minus ", '"true"') + pynutil.insert(" "), 0, 1 + # "und" after "hundert" or a magnitude word is a conjunction, never part of the number, so + # the phrase stays one token with both sides written out and the spacing normalized: + # "eintausendundzwanzig" and "ein tausend und zwanzig" both give 1.000 und 20 + magnitude_word = pynini.union( + "hundert", "tausend", million, billion, trillion, quadrillion, quintillion, sextillion ) + ends_in_magnitude = pynini.compose(NEMO_SIGMA + magnitude_word, self.graph_no_exception) + graph_magnitude_und = ( + ends_in_magnitude + + delete_space + + pynutil.insert(" ") + + pynini.accep(conjunction) + + pynutil.insert(" ") + + delete_space + + self.graph_hundred_component_at_least_one_none_zero_digit + ) + + # the cardinal verbalizer turns the "true" flag back into a minus sign + negative = pynutil.insert("negative: ") + pynini.cross("minus ", '"true"') + pynutil.insert(" ") # the decimal verbalizer reads a single character out of the negative field, so the graph # handed to the other classes keeps the minus sign rather than the "true" flag @@ -269,8 +249,23 @@ def __init__(self): pynutil.insert("negative: ") + pynini.cross("minus ", '"-"') + pynutil.insert(" "), 0, 1 ) - # The final graph for this semiotic class leaves the first dozen normalized - final_graph = optional_negative + pynutil.insert('integer: "') + self.graph + pynutil.insert('"') + # On its own the first dozen stays verbalized, but a signed number is always written + # in digits: "drei" -> "drei" while "minus drei" -> "-3" + integer = pynutil.insert('integer: "') + (self.graph | graph_magnitude_und) + pynutil.insert('"') + # a sign in front of zero carries no meaning, so "minus null" is not a cardinal + accept_zero = pynini.project(zero, "input") + graph_no_exception_non_zero = ( + pynini.difference(accept_denormalized_everything, accept_zero) @ self.graph_no_exception + ) + + negative_integer = ( + negative + + pynutil.insert('integer: "') + + (graph_no_exception_non_zero | graph_magnitude_und) + + pynutil.insert('"') + ) + + final_graph = integer | negative_integer final_graph = self.add_tokens(final_graph) self.fst = final_graph.optimize() diff --git a/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt index 29fd87359..e8d51ecb5 100644 --- a/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt +++ b/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt @@ -1,9 +1,11 @@ ein hundert~100 einhundert~100 -ein hundert und zwei~102 einhundertzwei~102 -ein hundert und zwanzig~120 -ein hundert und elf~111 +ein hundert zwei~102 +ein hundert und zwei~100 und 2 +ein hundert und zwanzig~100 und 20 +ein hundert und elf~100 und 11 +einhundertundelf~100 und 11 ein tausend~1.000 eintausend~1.000 ein hundert zwölf~112 @@ -15,7 +17,6 @@ neun billionen sieben hundert neun und achtzig milliarden drei hundert zwei und zwei hundert vier und fünfzig~254 zwo hundert~200 zwoundzwanzig~22 -zwöundzwanzig~22 ein hundert sieben und vierzig tausend vier hundert ein und fünfzig~147.451 eine million ein hundert sechs und fünfzig tausend ein hundert drei und siebzig~1.156.173 eine milliarde fünf hundert drei und neunzig millionen zwei und siebzig tausend neun hundert ein und sechzig~1.593.072.961 @@ -36,15 +37,8 @@ ein~ein eine~eine einer~einer zwei~zwei -zwö~zwö zwo~zwo drei~drei -vier~vier -fünf~fünf -sechs~sechs -sieben~sieben -acht~acht -neun~neun zehn~zehn elf~elf zwölf~zwölf @@ -64,7 +58,7 @@ siebzig~70 achtzig~80 neunzig~90 hundert~100 -eine million tausend~1.001.000 +eine million ein tausend~1.001.000 zwei millionen drei~2.000.003 tausend~1.000 tausend zwanzig~1.020 @@ -81,11 +75,20 @@ zwei millionen ein tausend acht~2.001.008 achtzehn tausend acht hundert achtzig~18.880 einhunderteins~101 ein tausend einhundert sechsundzwanzig~1.126 -ein tausend und zwanzig~1.020 +ein tausend und zwanzig~1.000 und 20 +eintausendundzwanzig~1.000 und 20 +zwei tausend und vier hundert zwei und zwanzig~2.000 und 422 tausend ein hundert~1.100 +eine milliarde ein tausend~1.000.001.000 eine milliarde tausend~1.000.001.000 +zwei millionen ein tausend drei~2.001.003 zwei millionen tausend drei~2.001.003 -minus drei~-drei +eine million tausend~1.001.000 +minus drei~-3 +minus ein tausend und zwanzig~-1.000 und 20 trillion~1.000.000.000.000.000.000 -eine million und zwanzig~1.000.020 -eine trilliarde zwei trillionen eine billion vier~1.002.000.001.000.000.000.004 \ No newline at end of file +eine million und zwanzig~1.000.000 und 20 +eine million und drei und zwanzig~1.000.000 und 23 +eine trilliarde zwei trillionen eine billion vier~1.002.000.001.000.000.000.004 +eine milliarde und vierzehn~1.000.000.000 und 14 +eine billion und fünf und siebzig~1.000.000.000.000 und 75 From 7fe42bca1cc3640860fe7f27bcacd30d225d5423 Mon Sep 17 00:00:00 2001 From: Adelina Dunina Date: Wed, 16 Sep 2026 16:53:11 -0400 Subject: [PATCH 12/13] De ITN: unify the cardinal negative field on the minus sign Fixed: the cardinal tagger wrote negative: "-" while its verbalizer still matched the literal negative: "true", so every signed cardinal failed to verbalize and fell back to the input text. The verbalizer now reads the sign character out of the field, matching how the decimal verbalizer already reads it, and the tagger keeps a single definition of the negative graph instead of two copies. Signed-off-by: Adelina Dunina --- .../de/taggers/cardinal.py | 16 ++++++---------- .../de/verbalizers/cardinal.py | 8 +++++--- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py index 2cd6f7671..8f135d0e9 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py @@ -73,8 +73,8 @@ class CardinalFst(GraphFst): number, it joins two numbers, whether written glued or spaced: both "einhundertundzwei" and "ein hundert und zwei" -> 100 und 2. German writes 102 as "ein hundert zwei". - e.g. minus drei und zwanzig -> cardinal { negative: "true" integer: "23" } - e.g. minus dreiundzwanzig -> cardinal { negative: "true" integer: "23" } + e.g. minus drei und zwanzig -> cardinal { negative: "-" integer: "23" } + e.g. minus dreiundzwanzig -> cardinal { negative: "-" integer: "23" } e.g. dreizehn -> cardinal { integer: "13" } e.g. ein hundert -> cardinal { integer: "100" } e.g. einhundert -> cardinal { integer: "100" } @@ -240,14 +240,10 @@ def __init__(self): + self.graph_hundred_component_at_least_one_none_zero_digit ) - # the cardinal verbalizer turns the "true" flag back into a minus sign - negative = pynutil.insert("negative: ") + pynini.cross("minus ", '"true"') + pynutil.insert(" ") - - # the decimal verbalizer reads a single character out of the negative field, so the graph - # handed to the other classes keeps the minus sign rather than the "true" flag - self.optional_minus_graph = pynini.closure( - pynutil.insert("negative: ") + pynini.cross("minus ", '"-"') + pynutil.insert(" "), 0, 1 - ) + # every verbalizer reads a single character out of the negative field, so the tagger + # writes the minus sign itself instead of a boolean flag + negative = pynutil.insert("negative: ") + pynini.cross("minus ", '"-"') + pynutil.insert(" ") + self.optional_minus_graph = pynini.closure(negative, 0, 1) # On its own the first dozen stays verbalized, but a signed number is always written # in digits: "drei" -> "drei" while "minus drei" -> "-3" diff --git a/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py index 2f30e8808..019b09b19 100644 --- a/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py @@ -21,14 +21,16 @@ class CardinalFst(GraphFst): """ Finite state transducer for verbalizing cardinal - e.g. cardinal { negative: "true" integer: "23" } -> -23 + e.g. cardinal { negative: "-" integer: "23" } -> -23 """ def __init__(self): super().__init__(name="cardinal", kind="verbalize") - # removes the 'negative:' label and leaves the optional '-' sign in place - optional_minus = pynini.closure(pynini.cross('negative: "true"', "-") + delete_space, 0, 1) + # the tagger writes the sign itself, so the verbalizer just reads it out of the field + optional_minus = pynini.closure( + pynutil.delete('negative: "') + NEMO_NOT_QUOTE + pynutil.delete('"') + delete_space, 0, 1 + ) # removes the 'integer:' label just_integers = ( From c17585461123c026e9cec4993c3c26601838deb9 Mon Sep 17 00:00:00 2001 From: Adelina Dunina Date: Wed, 16 Sep 2026 17:29:47 -0400 Subject: [PATCH 13/13] De ITN: drop redundant comments from the cardinal tagger Removed the comments that only restated the name of the variable below them, the ones duplicating the class docstring or the get_tens_digit docstring, and the inline labels on the first-dozen block. Reworded the remaining first-dozen comment, since "canonically normalized" reads ambiguously in inverse text normalization. Comment density now matches the other language taggers. No grammar changes. Signed-off-by: Adelina Dunina --- .../de/taggers/cardinal.py | 37 +++---------------- 1 file changed, 5 insertions(+), 32 deletions(-) diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py index 8f135d0e9..252378b97 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py @@ -86,7 +86,6 @@ class CardinalFst(GraphFst): def __init__(self): super().__init__(name="cardinal", kind="classify") - # WFST mappings for numbers 0-99 zero = pynini.string_file(get_abs_path("data/cardinal/zero.tsv")) digits = pynini.string_file(get_abs_path("data/cardinal/digits.tsv")) irregular_teens = pynini.string_file(get_abs_path("data/cardinal/irregular_teens.tsv")) @@ -96,7 +95,6 @@ def __init__(self): teens = irregular_teens | regular_teens tens = pynini.string_file(get_abs_path("data/cardinal/tens.tsv")) ties = tens + pynutil.insert("0") - # German flips ones and tens in two-digit numbers. The WFST below handles these flips. conjunction = load_labels(get_abs_path("data/cardinal/conjunction.tsv"))[0][0] # the map is keyed on the compound spelling, so whitespace is stripped before lookup @@ -107,7 +105,6 @@ def __init__(self): get_abs_path("data/cardinal/conjunction.tsv"), ) - # WFST grammar for hundreds graph_10_99 = teens | ties | ties_digit hundreds = ( @@ -117,20 +114,17 @@ def __init__(self): ) # Digits are grouped in clusters of three: {hundreds}{tens}{ones}. - # Clusters of three are separated by periods, applied right to left. non_zero_digit_cluster = (hundreds) | (pynutil.insert("0") + graph_10_99) | (pynutil.insert("00") + digits) digit_cluster = non_zero_digit_cluster | pynutil.insert("000") # a magnitude word with no multiplier in front of it means "one" of that magnitude leading_cluster = non_zero_digit_cluster | pynutil.insert("001") - # WFST grammar for thousands thousands = ( ((leading_cluster + delete_space + pynini.cross("tausend", ".")) | pynutil.insert("000.")) + delete_space + digit_cluster ) - # WFST grammar for millions million = pynini.accep("million") | pynini.accep("millionen") millions = ( ((leading_cluster + delete_space + pynini.cross(million, ".")) | pynutil.insert("000.")) @@ -138,7 +132,6 @@ def __init__(self): + thousands ) - # WFST grammar for billions billion = pynini.accep("milliarde") | pynini.accep("milliarden") billions = ( ((leading_cluster + delete_space + pynini.cross(billion, ".")) | pynutil.insert("000.")) @@ -146,7 +139,6 @@ def __init__(self): + millions ) - # WFST grammar for trillions trillion = pynini.accep("billion") | pynini.accep("billionen") trillions = ( ((leading_cluster + delete_space + pynini.cross(trillion, ".")) | pynutil.insert("000.")) @@ -154,7 +146,6 @@ def __init__(self): + billions ) - # WFST grammar for quadrillions quadrillion = pynini.accep("billiarde") | pynini.accep("billiarden") quadrillions = ( ((leading_cluster + delete_space + pynini.cross(quadrillion, ".")) | pynutil.insert("000.")) @@ -162,7 +153,6 @@ def __init__(self): + trillions ) - # WFST grammar for quintillions quintillion = pynini.accep("trillion") | pynini.accep("trillionen") quintillions = ( ((leading_cluster + delete_space + pynini.cross(quintillion, ".")) | pynutil.insert("000.")) @@ -170,7 +160,6 @@ def __init__(self): + quadrillions ) - # WFST grammar for sextillions sextillion = pynini.accep("trilliarde") | pynini.accep("trilliarden") sextillions = ( ((leading_cluster + delete_space + pynini.cross(sextillion, ".")) | pynutil.insert("000.")) @@ -178,13 +167,11 @@ def __init__(self): + quintillions ) - # Remove the leading zeros non_zero_digits = pynini.difference(NEMO_DIGIT, "0") chars_to_remove = pynini.accep("0") | pynini.accep(".") remove_chars = pynutil.delete(pynini.closure(chars_to_remove)) remove_leading_zeros = pynini.cdrewrite(remove_chars, "[BOS]", non_zero_digits, NEMO_SIGMA) - # All together now grammars = [ sextillions, quintillions, @@ -209,23 +196,14 @@ def __init__(self): non_zero_digit_cluster @ remove_leading_zeros ).optimize() - # The block below leaves numerals 1 - 12 canonically normalized - accept_denormalized_first_dozen = pynini.project(to_denormalize, "input") # acceptor for null - zwölf - accept_denormalized_everything = pynini.project( - self.graph_no_exception, "input" - ) # acceptor for all verbalized cardinals - accept_without_first_dozen = ( - accept_denormalized_everything - accept_denormalized_first_dozen - ) # acceptor for all verbalized cardinals greater than 12 - transduce_without_first_dozen = ( - accept_without_first_dozen @ self.graph_no_exception - ) # transducer for all verbalized cardinals greater than 12 + # the block below leaves numerals 1 - 12 spelled out + accept_denormalized_first_dozen = pynini.project(to_denormalize, "input") + accept_denormalized_everything = pynini.project(self.graph_no_exception, "input") + accept_without_first_dozen = accept_denormalized_everything - accept_denormalized_first_dozen + transduce_without_first_dozen = accept_without_first_dozen @ self.graph_no_exception graph = accept_denormalized_first_dozen | transduce_without_first_dozen self.graph = graph.optimize() - # "und" after "hundert" or a magnitude word is a conjunction, never part of the number, so - # the phrase stays one token with both sides written out and the spacing normalized: - # "eintausendundzwanzig" and "ein tausend und zwanzig" both give 1.000 und 20 magnitude_word = pynini.union( "hundert", "tausend", million, billion, trillion, quadrillion, quintillion, sextillion ) @@ -240,13 +218,8 @@ def __init__(self): + self.graph_hundred_component_at_least_one_none_zero_digit ) - # every verbalizer reads a single character out of the negative field, so the tagger - # writes the minus sign itself instead of a boolean flag negative = pynutil.insert("negative: ") + pynini.cross("minus ", '"-"') + pynutil.insert(" ") self.optional_minus_graph = pynini.closure(negative, 0, 1) - - # On its own the first dozen stays verbalized, but a signed number is always written - # in digits: "drei" -> "drei" while "minus drei" -> "-3" integer = pynutil.insert('integer: "') + (self.graph | graph_magnitude_und) + pynutil.insert('"') # a sign in front of zero carries no meaning, so "minus null" is not a cardinal accept_zero = pynini.project(zero, "input")