Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions api/src/services/text_processing/normalizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -410,6 +410,17 @@ def handle_time(t: re.Match[str]) -> str:
def normalize_text(text: str, normalization_options: NormalizationOptions) -> str:
"""Normalize text for TTS processing"""

# Expand the "'re" contractions that espeak mis-phonemizes with a spurious
# /ɹeɪ/ ("-ray") ending, e.g. "how're" -> /haʊɹeɪ/ which sounds like "harry".
# Only the wh-words and there/these/those break this way; "you're", "we're"
# and "they're" already phonemize correctly and are left untouched.
text = re.sub(
r"\b(how|what|where|who|when|why|there|these|those)['’]re\b",
r"\1 are",
text,
flags=re.IGNORECASE,
)

# Handle email addresses first if enabled
if normalization_options.email_normalization:
text = EMAIL_PATTERN.sub(handle_email, text)
Expand Down
27 changes: 27 additions & 0 deletions api/tests/test_normalizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -334,3 +334,30 @@ def test_remaining_symbol():
)
== "I love buying products at good store here and at other store"
)


def test_re_contraction_expansion():
"""Wh-word "'re" contractions are expanded so espeak does not voice a
spurious "-ray" ending (e.g. "how're" was phonemized like "harry")."""
assert (
normalize_text(
"Hello there, how're you doing this fine day?",
normalization_options=NormalizationOptions(),
)
== "Hello there, how are you doing this fine day?"
)
assert (
normalize_text(
"What're these and where're they going?",
normalization_options=NormalizationOptions(),
)
== "What are these and where are they going?"
)
# Contractions that already phonemize correctly must be left untouched.
assert (
normalize_text(
"You're sure we're not late and they're here?",
normalization_options=NormalizationOptions(),
)
== "You're sure we're not late and they're here?"
)
Loading