From e2daa550002a13f578d2b09cb7dcd07ccae6d751 Mon Sep 17 00:00:00 2001 From: tsukumi Date: Fri, 8 Mar 2024 08:43:54 +0000 Subject: [PATCH] Refactor: split style_bert_vits2.nlp.english package --- style_bert_vits2/nlp/__init__.py | 20 +- style_bert_vits2/nlp/english/__init__.py | 414 ------------------- style_bert_vits2/nlp/english/bert_feature.py | 6 +- style_bert_vits2/nlp/english/cmudict.py | 46 +++ style_bert_vits2/nlp/english/g2p.py | 240 +++++++++++ style_bert_vits2/nlp/english/normalizer.py | 130 ++++++ 6 files changed, 432 insertions(+), 424 deletions(-) create mode 100644 style_bert_vits2/nlp/english/cmudict.py create mode 100644 style_bert_vits2/nlp/english/g2p.py create mode 100644 style_bert_vits2/nlp/english/normalizer.py diff --git a/style_bert_vits2/nlp/__init__.py b/style_bert_vits2/nlp/__init__.py index 47786d1..99f56a9 100644 --- a/style_bert_vits2/nlp/__init__.py +++ b/style_bert_vits2/nlp/__init__.py @@ -1,5 +1,4 @@ -import torch -from typing import Optional +from typing import Optional, TYPE_CHECKING from style_bert_vits2.constants import Languages from style_bert_vits2.nlp.symbols import ( @@ -8,6 +7,11 @@ from style_bert_vits2.nlp.symbols import ( SYMBOLS, ) +# __init__.py は配下のモジュールをインポートした時点で実行される +# Pytorch のインポートは重いので、型チェック時以外はインポートしない +if TYPE_CHECKING: + import torch + __symbol_to_id = {s: i for i, s in enumerate(SYMBOLS)} @@ -16,10 +20,10 @@ def extract_bert_feature( text: str, word2ph: list[int], language: Languages, - device: torch.device | str, + device: str, assist_text: Optional[str] = None, assist_text_weight: float = 0.7, -) -> torch.Tensor: +) -> "torch.Tensor": """ テキストから BERT の特徴量を抽出する @@ -27,7 +31,7 @@ def extract_bert_feature( text (str): テキスト word2ph (list[int]): 元のテキストの各文字に音素が何個割り当てられるかを表すリスト language (Languages): テキストの言語 - device (torch.device | str): 推論に利用するデバイス + device (str): 推論に利用するデバイス assist_text (Optional[str], optional): 補助テキスト (デフォルト: None) assist_text_weight (float, optional): 補助テキストの重み (デフォルト: 0.7) @@ -68,11 +72,13 @@ def clean_text( # Changed to import inside if condition to avoid unnecessary import if language == Languages.JP: - from style_bert_vits2.nlp.japanese import g2p, normalize_text + from style_bert_vits2.nlp.japanese.g2p import g2p + from style_bert_vits2.nlp.japanese.normalizer import normalize_text norm_text = normalize_text(text) phones, tones, word2ph = g2p(norm_text, use_jp_extra, raise_yomi_error) elif language == Languages.EN: - from style_bert_vits2.nlp.english import g2p, normalize_text + from style_bert_vits2.nlp.english.g2p import g2p + from style_bert_vits2.nlp.english.normalizer import normalize_text norm_text = normalize_text(text) phones, tones, word2ph = g2p(norm_text) elif language == Languages.ZH: diff --git a/style_bert_vits2/nlp/english/__init__.py b/style_bert_vits2/nlp/english/__init__.py index b2067c3..e69de29 100644 --- a/style_bert_vits2/nlp/english/__init__.py +++ b/style_bert_vits2/nlp/english/__init__.py @@ -1,414 +0,0 @@ -import pickle -import os -import re -from pathlib import Path - -import inflect -from g2p_en import G2p - -from style_bert_vits2.constants import Languages -from style_bert_vits2.nlp import bert_models -from style_bert_vits2.nlp.symbols import PUNCTUATIONS, SYMBOLS - - -CMU_DICT_PATH = Path(__file__).parent / "cmudict.rep" -CACHE_PATH = Path(__file__).parent / "cmudict_cache.pickle" - - -def g2p(text: str) -> tuple[list[str], list[int], list[int]]: - - ARPA = { - "AH0", - "S", - "AH1", - "EY2", - "AE2", - "EH0", - "OW2", - "UH0", - "NG", - "B", - "G", - "AY0", - "M", - "AA0", - "F", - "AO0", - "ER2", - "UH1", - "IY1", - "AH2", - "DH", - "IY0", - "EY1", - "IH0", - "K", - "N", - "W", - "IY2", - "T", - "AA1", - "ER1", - "EH2", - "OY0", - "UH2", - "UW1", - "Z", - "AW2", - "AW1", - "V", - "UW2", - "AA2", - "ER", - "AW0", - "UW0", - "R", - "OW1", - "EH1", - "ZH", - "AE0", - "IH2", - "IH", - "Y", - "JH", - "P", - "AY1", - "EY0", - "OY2", - "TH", - "HH", - "D", - "ER0", - "CH", - "AO1", - "AE1", - "AO2", - "OY1", - "AY2", - "IH1", - "OW0", - "L", - "SH", - } - - _g2p = G2p() - - phones = [] - tones = [] - phone_len = [] - # tokens = [tokenizer.tokenize(i) for i in words] - words = __text_to_words(text) - eng_dict = __get_dict() - - for word in words: - temp_phones, temp_tones = [], [] - if len(word) > 1: - if "'" in word: - word = ["".join(word)] - for w in word: - if w in PUNCTUATIONS: - temp_phones.append(w) - temp_tones.append(0) - continue - if w.upper() in eng_dict: - phns, tns = __refine_syllables(eng_dict[w.upper()]) - temp_phones += [__post_replace_ph(i) for i in phns] - temp_tones += tns - # w2ph.append(len(phns)) - else: - phone_list = list(filter(lambda p: p != " ", _g2p(w))) # type: ignore - phns = [] - tns = [] - for ph in phone_list: - if ph in ARPA: - ph, tn = __refine_ph(ph) - phns.append(ph) - tns.append(tn) - else: - phns.append(ph) - tns.append(0) - temp_phones += [__post_replace_ph(i) for i in phns] - temp_tones += tns - phones += temp_phones - tones += temp_tones - phone_len.append(len(temp_phones)) - # phones = [post_replace_ph(i) for i in phones] - - word2ph = [] - for token, pl in zip(words, phone_len): - word_len = len(token) - - aaa = __distribute_phone(pl, word_len) - word2ph += aaa - - phones = ["_"] + phones + ["_"] - tones = [0] + tones + [0] - word2ph = [1] + word2ph + [1] - assert len(phones) == len(tones), text - assert len(phones) == sum(word2ph), text - - return phones, tones, word2ph - - -def normalize_text(text: str) -> str: - text = __normalize_numbers(text) - text = __replace_punctuation(text) - text = re.sub(r"([,;.\?\!])([\w])", r"\1 \2", text) - return text - - -def __normalize_numbers(text: str) -> str: - text = re.sub(__comma_number_re, __remove_commas, text) - text = re.sub(__pounds_re, r"\1 pounds", text) - text = re.sub(__dollars_re, __expand_dollars, text) - text = re.sub(__decimal_number_re, __expand_decimal_point, text) - text = re.sub(__ordinal_re, __expand_ordinal, text) - text = re.sub(__number_re, __expand_number, text) - return text - - -def __replace_punctuation(text: str) -> str: - REPLACE_MAP = { - "：": ",", - "；": ",", - "，": ",", - "。": ".", - "！": "!", - "？": "?", - "\n": ".", - "．": ".", - "…": "...", - "···": "...", - "・・・": "...", - "·": ",", - "・": ",", - "、": ",", - "$": ".", - "“": "'", - "”": "'", - '"': "'", - "‘": "'", - "’": "'", - "（": "'", - "）": "'", - "(": "'", - ")": "'", - "《": "'", - "》": "'", - "【": "'", - "】": "'", - "[": "'", - "]": "'", - "—": "-", - "−": "-", - "～": "-", - "~": "-", - "「": "'", - "」": "'", - } - pattern = re.compile("|".join(re.escape(p) for p in REPLACE_MAP.keys())) - replaced_text = pattern.sub(lambda x: REPLACE_MAP[x.group()], text) - # replaced_text = re.sub( - # r"[^\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\u3400-\u4DBF\u3005" - # + "".join(punctuation) - # + r"]+", - # "", - # replaced_text, - # ) - return replaced_text - - -def __post_replace_ph(ph: str) -> str: - REPLACE_MAP = { - "：": ",", - "；": ",", - "，": ",", - "。": ".", - "！": "!", - "？": "?", - "\n": ".", - "·": ",", - "、": ",", - "…": "...", - "···": "...", - "・・・": "...", - "v": "V", - } - if ph in REPLACE_MAP.keys(): - ph = REPLACE_MAP[ph] - if ph in SYMBOLS: - return ph - if ph not in SYMBOLS: - ph = "UNK" - return ph - - -def __read_dict() -> dict[str, list[list[str]]]: - g2p_dict = {} - start_line = 49 - with open(CMU_DICT_PATH) as f: - line = f.readline() - line_index = 1 - while line: - if line_index >= start_line: - line = line.strip() - word_split = line.split(" ") - word = word_split[0] - - syllable_split = word_split[1].split(" - ") - g2p_dict[word] = [] - for syllable in syllable_split: - phone_split = syllable.split(" ") - g2p_dict[word].append(phone_split) - - line_index = line_index + 1 - line = f.readline() - - return g2p_dict - - -def __cache_dict(g2p_dict: dict[str, list[list[str]]], file_path: Path) -> None: - with open(file_path, "wb") as pickle_file: - pickle.dump(g2p_dict, pickle_file) - - -def __get_dict() -> dict[str, list[list[str]]]: - if CACHE_PATH.exists(): - with open(CACHE_PATH, "rb") as pickle_file: - g2p_dict = pickle.load(pickle_file) - else: - g2p_dict = __read_dict() - __cache_dict(g2p_dict, CACHE_PATH) - - return g2p_dict - - -def __refine_ph(phn: str) -> tuple[str, int]: - tone = 0 - if re.search(r"\d$", phn): - tone = int(phn[-1]) + 1 - phn = phn[:-1] - else: - tone = 3 - return phn.lower(), tone - - -def __refine_syllables(syllables: list[list[str]]) -> tuple[list[str], list[int]]: - tones = [] - phonemes = [] - for phn_list in syllables: - for i in range(len(phn_list)): - phn = phn_list[i] - phn, tone = __refine_ph(phn) - phonemes.append(phn) - tones.append(tone) - return phonemes, tones - - -__inflect = inflect.engine() -__comma_number_re = re.compile(r"([0-9][0-9\,]+[0-9])") -__decimal_number_re = re.compile(r"([0-9]+\.[0-9]+)") -__pounds_re = re.compile(r"£([0-9\,]*[0-9]+)") -__dollars_re = re.compile(r"\$([0-9\.\,]*[0-9]+)") -__ordinal_re = re.compile(r"[0-9]+(st|nd|rd|th)") -__number_re = re.compile(r"[0-9]+") - - -def __expand_dollars(m: re.Match[str]) -> str: - match = m.group(1) - parts = match.split(".") - if len(parts) > 2: - return match + " dollars" # Unexpected format - dollars = int(parts[0]) if parts[0] else 0 - cents = int(parts[1]) if len(parts) > 1 and parts[1] else 0 - if dollars and cents: - dollar_unit = "dollar" if dollars == 1 else "dollars" - cent_unit = "cent" if cents == 1 else "cents" - return "%s %s, %s %s" % (dollars, dollar_unit, cents, cent_unit) - elif dollars: - dollar_unit = "dollar" if dollars == 1 else "dollars" - return "%s %s" % (dollars, dollar_unit) - elif cents: - cent_unit = "cent" if cents == 1 else "cents" - return "%s %s" % (cents, cent_unit) - else: - return "zero dollars" - - -def __remove_commas(m: re.Match[str]) -> str: - return m.group(1).replace(",", "") - - -def __expand_ordinal(m: re.Match[str]) -> str: - return __inflect.number_to_words(m.group(0)) # type: ignore - - -def __expand_number(m: re.Match[str]) -> str: - num = int(m.group(0)) - if num > 1000 and num < 3000: - if num == 2000: - return "two thousand" - elif num > 2000 and num < 2010: - return "two thousand " + __inflect.number_to_words(num % 100) # type: ignore - elif num % 100 == 0: - return __inflect.number_to_words(num // 100) + " hundred" # type: ignore - else: - return __inflect.number_to_words( - num, andword="", zero="oh", group=2 # type: ignore - ).replace(", ", " ") # type: ignore - else: - return __inflect.number_to_words(num, andword="") # type: ignore - - -def __expand_decimal_point(m: re.Match[str]) -> str: - return m.group(1).replace(".", " point ") - - -def __distribute_phone(n_phone: int, n_word: int) -> list[int]: - phones_per_word = [0] * n_word - for task in range(n_phone): - min_tasks = min(phones_per_word) - min_index = phones_per_word.index(min_tasks) - phones_per_word[min_index] += 1 - return phones_per_word - - -def __text_to_words(text: str) -> list[list[str]]: - tokenizer = bert_models.load_tokenizer(Languages.EN) - tokens = tokenizer.tokenize(text) - words = [] - for idx, t in enumerate(tokens): - if t.startswith("▁"): - words.append([t[1:]]) - else: - if t in PUNCTUATIONS: - if idx == len(tokens) - 1: - words.append([f"{t}"]) - else: - if ( - not tokens[idx + 1].startswith("▁") - and tokens[idx + 1] not in PUNCTUATIONS - ): - if idx == 0: - words.append([]) - words[-1].append(f"{t}") - else: - words.append([f"{t}"]) - else: - if idx == 0: - words.append([]) - words[-1].append(f"{t}") - return words - - -if __name__ == "__main__": - # print(get_dict()) - # print(eng_word_to_phoneme("hello")) - print(g2p("In this paper, we propose 1 DSPGAN, a GAN-based universal vocoder.")) - # all_phones = set() - # eng_dict = get_dict() - # for k, syllables in eng_dict.items(): - # for group in syllables: - # for ph in group: - # all_phones.add(ph) - # print(all_phones) diff --git a/style_bert_vits2/nlp/english/bert_feature.py b/style_bert_vits2/nlp/english/bert_feature.py index 647920d..27fd501 100644 --- a/style_bert_vits2/nlp/english/bert_feature.py +++ b/style_bert_vits2/nlp/english/bert_feature.py @@ -8,13 +8,13 @@ from style_bert_vits2.constants import Languages from style_bert_vits2.nlp import bert_models -__models: dict[torch.device | str, PreTrainedModel] = {} +__models: dict[str, PreTrainedModel] = {} def extract_bert_feature( text: str, word2ph: list[int], - device: torch.device | str, + device: str, assist_text: Optional[str] = None, assist_text_weight: float = 0.7, ) -> torch.Tensor: @@ -24,7 +24,7 @@ def extract_bert_feature( Args: text (str): 英語のテキスト word2ph (list[int]): 元のテキストの各文字に音素が何個割り当てられるかを表すリスト - device (torch.device | str): 推論に利用するデバイス + device (str): 推論に利用するデバイス assist_text (Optional[str], optional): 補助テキスト (デフォルト: None) assist_text_weight (float, optional): 補助テキストの重み (デフォルト: 0.7) diff --git a/style_bert_vits2/nlp/english/cmudict.py b/style_bert_vits2/nlp/english/cmudict.py new file mode 100644 index 0000000..e6afb89 --- /dev/null +++ b/style_bert_vits2/nlp/english/cmudict.py @@ -0,0 +1,46 @@ +import pickle +from pathlib import Path + + +CMU_DICT_PATH = Path(__file__).parent / "cmudict.rep" +CACHE_PATH = Path(__file__).parent / "cmudict_cache.pickle" + + +def get_dict() -> dict[str, list[list[str]]]: + if CACHE_PATH.exists(): + with open(CACHE_PATH, "rb") as pickle_file: + g2p_dict = pickle.load(pickle_file) + else: + g2p_dict = read_dict() + cache_dict(g2p_dict, CACHE_PATH) + + return g2p_dict + + +def read_dict() -> dict[str, list[list[str]]]: + g2p_dict = {} + start_line = 49 + with open(CMU_DICT_PATH) as f: + line = f.readline() + line_index = 1 + while line: + if line_index >= start_line: + line = line.strip() + word_split = line.split(" ") + word = word_split[0] + + syllable_split = word_split[1].split(" - ") + g2p_dict[word] = [] + for syllable in syllable_split: + phone_split = syllable.split(" ") + g2p_dict[word].append(phone_split) + + line_index = line_index + 1 + line = f.readline() + + return g2p_dict + + +def cache_dict(g2p_dict: dict[str, list[list[str]]], file_path: Path) -> None: + with open(file_path, "wb") as pickle_file: + pickle.dump(g2p_dict, pickle_file) diff --git a/style_bert_vits2/nlp/english/g2p.py b/style_bert_vits2/nlp/english/g2p.py new file mode 100644 index 0000000..db2a87f --- /dev/null +++ b/style_bert_vits2/nlp/english/g2p.py @@ -0,0 +1,240 @@ +import re + +from g2p_en import G2p + +from style_bert_vits2.constants import Languages +from style_bert_vits2.nlp import bert_models +from style_bert_vits2.nlp.english.cmudict import get_dict +from style_bert_vits2.nlp.symbols import PUNCTUATIONS, SYMBOLS + + +def g2p(text: str) -> tuple[list[str], list[int], list[int]]: + + ARPA = { + "AH0", + "S", + "AH1", + "EY2", + "AE2", + "EH0", + "OW2", + "UH0", + "NG", + "B", + "G", + "AY0", + "M", + "AA0", + "F", + "AO0", + "ER2", + "UH1", + "IY1", + "AH2", + "DH", + "IY0", + "EY1", + "IH0", + "K", + "N", + "W", + "IY2", + "T", + "AA1", + "ER1", + "EH2", + "OY0", + "UH2", + "UW1", + "Z", + "AW2", + "AW1", + "V", + "UW2", + "AA2", + "ER", + "AW0", + "UW0", + "R", + "OW1", + "EH1", + "ZH", + "AE0", + "IH2", + "IH", + "Y", + "JH", + "P", + "AY1", + "EY0", + "OY2", + "TH", + "HH", + "D", + "ER0", + "CH", + "AO1", + "AE1", + "AO2", + "OY1", + "AY2", + "IH1", + "OW0", + "L", + "SH", + } + + _g2p = G2p() + + phones = [] + tones = [] + phone_len = [] + # tokens = [tokenizer.tokenize(i) for i in words] + words = __text_to_words(text) + eng_dict = get_dict() + + for word in words: + temp_phones, temp_tones = [], [] + if len(word) > 1: + if "'" in word: + word = ["".join(word)] + for w in word: + if w in PUNCTUATIONS: + temp_phones.append(w) + temp_tones.append(0) + continue + if w.upper() in eng_dict: + phns, tns = __refine_syllables(eng_dict[w.upper()]) + temp_phones += [__post_replace_ph(i) for i in phns] + temp_tones += tns + # w2ph.append(len(phns)) + else: + phone_list = list(filter(lambda p: p != " ", _g2p(w))) # type: ignore + phns = [] + tns = [] + for ph in phone_list: + if ph in ARPA: + ph, tn = __refine_ph(ph) + phns.append(ph) + tns.append(tn) + else: + phns.append(ph) + tns.append(0) + temp_phones += [__post_replace_ph(i) for i in phns] + temp_tones += tns + phones += temp_phones + tones += temp_tones + phone_len.append(len(temp_phones)) + # phones = [post_replace_ph(i) for i in phones] + + word2ph = [] + for token, pl in zip(words, phone_len): + word_len = len(token) + + aaa = __distribute_phone(pl, word_len) + word2ph += aaa + + phones = ["_"] + phones + ["_"] + tones = [0] + tones + [0] + word2ph = [1] + word2ph + [1] + assert len(phones) == len(tones), text + assert len(phones) == sum(word2ph), text + + return phones, tones, word2ph + + +def __post_replace_ph(ph: str) -> str: + REPLACE_MAP = { + "：": ",", + "；": ",", + "，": ",", + "。": ".", + "！": "!", + "？": "?", + "\n": ".", + "·": ",", + "、": ",", + "…": "...", + "···": "...", + "・・・": "...", + "v": "V", + } + if ph in REPLACE_MAP.keys(): + ph = REPLACE_MAP[ph] + if ph in SYMBOLS: + return ph + if ph not in SYMBOLS: + ph = "UNK" + return ph + + +def __refine_ph(phn: str) -> tuple[str, int]: + tone = 0 + if re.search(r"\d$", phn): + tone = int(phn[-1]) + 1 + phn = phn[:-1] + else: + tone = 3 + return phn.lower(), tone + + +def __refine_syllables(syllables: list[list[str]]) -> tuple[list[str], list[int]]: + tones = [] + phonemes = [] + for phn_list in syllables: + for i in range(len(phn_list)): + phn = phn_list[i] + phn, tone = __refine_ph(phn) + phonemes.append(phn) + tones.append(tone) + return phonemes, tones + + +def __distribute_phone(n_phone: int, n_word: int) -> list[int]: + phones_per_word = [0] * n_word + for task in range(n_phone): + min_tasks = min(phones_per_word) + min_index = phones_per_word.index(min_tasks) + phones_per_word[min_index] += 1 + return phones_per_word + + +def __text_to_words(text: str) -> list[list[str]]: + tokenizer = bert_models.load_tokenizer(Languages.EN) + tokens = tokenizer.tokenize(text) + words = [] + for idx, t in enumerate(tokens): + if t.startswith("▁"): + words.append([t[1:]]) + else: + if t in PUNCTUATIONS: + if idx == len(tokens) - 1: + words.append([f"{t}"]) + else: + if ( + not tokens[idx + 1].startswith("▁") + and tokens[idx + 1] not in PUNCTUATIONS + ): + if idx == 0: + words.append([]) + words[-1].append(f"{t}") + else: + words.append([f"{t}"]) + else: + if idx == 0: + words.append([]) + words[-1].append(f"{t}") + return words + + +if __name__ == "__main__": + # print(get_dict()) + # print(eng_word_to_phoneme("hello")) + print(g2p("In this paper, we propose 1 DSPGAN, a GAN-based universal vocoder.")) + # all_phones = set() + # eng_dict = get_dict() + # for k, syllables in eng_dict.items(): + # for group in syllables: + # for ph in group: + # all_phones.add(ph) + # print(all_phones) diff --git a/style_bert_vits2/nlp/english/normalizer.py b/style_bert_vits2/nlp/english/normalizer.py new file mode 100644 index 0000000..0886581 --- /dev/null +++ b/style_bert_vits2/nlp/english/normalizer.py @@ -0,0 +1,130 @@ +import re + +import inflect + + +__INFLECT = inflect.engine() +__COMMA_NUMBER_PATTERN = re.compile(r"([0-9][0-9\,]+[0-9])") +__DECIMAL_NUMBER_PATTERN = re.compile(r"([0-9]+\.[0-9]+)") +__POUNDS_PATTERN = re.compile(r"£([0-9\,]*[0-9]+)") +__DOLLARS_PATTERN = re.compile(r"\$([0-9\.\,]*[0-9]+)") +__ORDINAL_PATTERN = re.compile(r"[0-9]+(st|nd|rd|th)") +__NUMBER_PATTERN = re.compile(r"[0-9]+") + + +def normalize_text(text: str) -> str: + text = __normalize_numbers(text) + text = __replace_punctuation(text) + text = re.sub(r"([,;.\?\!])([\w])", r"\1 \2", text) + return text + + +def __normalize_numbers(text: str) -> str: + text = re.sub(__COMMA_NUMBER_PATTERN, __remove_commas, text) + text = re.sub(__POUNDS_PATTERN, r"\1 pounds", text) + text = re.sub(__DOLLARS_PATTERN, __expand_dollars, text) + text = re.sub(__DECIMAL_NUMBER_PATTERN, __expand_decimal_point, text) + text = re.sub(__ORDINAL_PATTERN, __expand_ordinal, text) + text = re.sub(__NUMBER_PATTERN, __expand_number, text) + return text + + +def __replace_punctuation(text: str) -> str: + REPLACE_MAP = { + "：": ",", + "；": ",", + "，": ",", + "。": ".", + "！": "!", + "？": "?", + "\n": ".", + "．": ".", + "…": "...", + "···": "...", + "・・・": "...", + "·": ",", + "・": ",", + "、": ",", + "$": ".", + "“": "'", + "”": "'", + '"': "'", + "‘": "'", + "’": "'", + "（": "'", + "）": "'", + "(": "'", + ")": "'", + "《": "'", + "》": "'", + "【": "'", + "】": "'", + "[": "'", + "]": "'", + "—": "-", + "−": "-", + "～": "-", + "~": "-", + "「": "'", + "」": "'", + } + pattern = re.compile("|".join(re.escape(p) for p in REPLACE_MAP.keys())) + replaced_text = pattern.sub(lambda x: REPLACE_MAP[x.group()], text) + # replaced_text = re.sub( + # r"[^\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\u3400-\u4DBF\u3005" + # + "".join(punctuation) + # + r"]+", + # "", + # replaced_text, + # ) + return replaced_text + + +def __expand_dollars(m: re.Match[str]) -> str: + match = m.group(1) + parts = match.split(".") + if len(parts) > 2: + return match + " dollars" # Unexpected format + dollars = int(parts[0]) if parts[0] else 0 + cents = int(parts[1]) if len(parts) > 1 and parts[1] else 0 + if dollars and cents: + dollar_unit = "dollar" if dollars == 1 else "dollars" + cent_unit = "cent" if cents == 1 else "cents" + return "%s %s, %s %s" % (dollars, dollar_unit, cents, cent_unit) + elif dollars: + dollar_unit = "dollar" if dollars == 1 else "dollars" + return "%s %s" % (dollars, dollar_unit) + elif cents: + cent_unit = "cent" if cents == 1 else "cents" + return "%s %s" % (cents, cent_unit) + else: + return "zero dollars" + + +def __remove_commas(m: re.Match[str]) -> str: + return m.group(1).replace(",", "") + + +def __expand_ordinal(m: re.Match[str]) -> str: + return __INFLECT.number_to_words(m.group(0)) # type: ignore + + +def __expand_number(m: re.Match[str]) -> str: + num = int(m.group(0)) + if num > 1000 and num < 3000: + if num == 2000: + return "two thousand" + elif num > 2000 and num < 2010: + return "two thousand " + __INFLECT.number_to_words(num % 100) # type: ignore + elif num % 100 == 0: + return __INFLECT.number_to_words(num // 100) + " hundred" # type: ignore + else: + return __INFLECT.number_to_words( + num, andword="", zero="oh", group=2 # type: ignore + ).replace(", ", " ") # type: ignore + else: + return __INFLECT.number_to_words(num, andword="") # type: ignore + + +def __expand_decimal_point(m: re.Match[str]) -> str: + return m.group(1).replace(".", " point ")