df68771651
Configured so that the same public function is exported from the module with the same name for each language.
131 lines
3.8 KiB
Python
131 lines
3.8 KiB
Python
import re
|
||
|
||
import inflect
|
||
|
||
|
||
__INFLECT = inflect.engine()
|
||
__COMMA_NUMBER_PATTERN = re.compile(r"([0-9][0-9\,]+[0-9])")
|
||
__DECIMAL_NUMBER_PATTERN = re.compile(r"([0-9]+\.[0-9]+)")
|
||
__POUNDS_PATTERN = re.compile(r"£([0-9\,]*[0-9]+)")
|
||
__DOLLARS_PATTERN = re.compile(r"\$([0-9\.\,]*[0-9]+)")
|
||
__ORDINAL_PATTERN = re.compile(r"[0-9]+(st|nd|rd|th)")
|
||
__NUMBER_PATTERN = re.compile(r"[0-9]+")
|
||
|
||
|
||
def normalize_text(text: str) -> str:
|
||
text = __normalize_numbers(text)
|
||
text = replace_punctuation(text)
|
||
text = re.sub(r"([,;.\?\!])([\w])", r"\1 \2", text)
|
||
return text
|
||
|
||
|
||
def replace_punctuation(text: str) -> str:
|
||
REPLACE_MAP = {
|
||
"：": ",",
|
||
"；": ",",
|
||
"，": ",",
|
||
"。": ".",
|
||
"！": "!",
|
||
"？": "?",
|
||
"\n": ".",
|
||
"．": ".",
|
||
"…": "...",
|
||
"···": "...",
|
||
"・・・": "...",
|
||
"·": ",",
|
||
"・": ",",
|
||
"、": ",",
|
||
"$": ".",
|
||
"“": "'",
|
||
"”": "'",
|
||
'"': "'",
|
||
"‘": "'",
|
||
"’": "'",
|
||
"（": "'",
|
||
"）": "'",
|
||
"(": "'",
|
||
")": "'",
|
||
"《": "'",
|
||
"》": "'",
|
||
"【": "'",
|
||
"】": "'",
|
||
"[": "'",
|
||
"]": "'",
|
||
"—": "-",
|
||
"−": "-",
|
||
"～": "-",
|
||
"~": "-",
|
||
"「": "'",
|
||
"」": "'",
|
||
}
|
||
pattern = re.compile("|".join(re.escape(p) for p in REPLACE_MAP.keys()))
|
||
replaced_text = pattern.sub(lambda x: REPLACE_MAP[x.group()], text)
|
||
# replaced_text = re.sub(
|
||
# r"[^\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\u3400-\u4DBF\u3005"
|
||
# + "".join(punctuation)
|
||
# + r"]+",
|
||
# "",
|
||
# replaced_text,
|
||
# )
|
||
return replaced_text
|
||
|
||
|
||
def __normalize_numbers(text: str) -> str:
|
||
text = re.sub(__COMMA_NUMBER_PATTERN, __remove_commas, text)
|
||
text = re.sub(__POUNDS_PATTERN, r"\1 pounds", text)
|
||
text = re.sub(__DOLLARS_PATTERN, __expand_dollars, text)
|
||
text = re.sub(__DECIMAL_NUMBER_PATTERN, __expand_decimal_point, text)
|
||
text = re.sub(__ORDINAL_PATTERN, __expand_ordinal, text)
|
||
text = re.sub(__NUMBER_PATTERN, __expand_number, text)
|
||
return text
|
||
|
||
|
||
def __expand_dollars(m: re.Match[str]) -> str:
|
||
match = m.group(1)
|
||
parts = match.split(".")
|
||
if len(parts) > 2:
|
||
return match + " dollars" # Unexpected format
|
||
dollars = int(parts[0]) if parts[0] else 0
|
||
cents = int(parts[1]) if len(parts) > 1 and parts[1] else 0
|
||
if dollars and cents:
|
||
dollar_unit = "dollar" if dollars == 1 else "dollars"
|
||
cent_unit = "cent" if cents == 1 else "cents"
|
||
return "%s %s, %s %s" % (dollars, dollar_unit, cents, cent_unit)
|
||
elif dollars:
|
||
dollar_unit = "dollar" if dollars == 1 else "dollars"
|
||
return "%s %s" % (dollars, dollar_unit)
|
||
elif cents:
|
||
cent_unit = "cent" if cents == 1 else "cents"
|
||
return "%s %s" % (cents, cent_unit)
|
||
else:
|
||
return "zero dollars"
|
||
|
||
|
||
def __remove_commas(m: re.Match[str]) -> str:
|
||
return m.group(1).replace(",", "")
|
||
|
||
|
||
def __expand_ordinal(m: re.Match[str]) -> str:
|
||
return __INFLECT.number_to_words(m.group(0)) # type: ignore
|
||
|
||
|
||
def __expand_number(m: re.Match[str]) -> str:
|
||
num = int(m.group(0))
|
||
if num > 1000 and num < 3000:
|
||
if num == 2000:
|
||
return "two thousand"
|
||
elif num > 2000 and num < 2010:
|
||
return "two thousand " + __INFLECT.number_to_words(num % 100) # type: ignore
|
||
elif num % 100 == 0:
|
||
return __INFLECT.number_to_words(num // 100) + " hundred" # type: ignore
|
||
else:
|
||
return __INFLECT.number_to_words(
|
||
num, andword="", zero="oh", group=2 # type: ignore
|
||
).replace(", ", " ") # type: ignore
|
||
else:
|
||
return __INFLECT.number_to_words(num, andword="") # type: ignore
|
||
|
||
|
||
def __expand_decimal_point(m: re.Match[str]) -> str:
|
||
return m.group(1).replace(".", " point ")
|