tuna2134/sbv2-v2
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Refactor: improve regular expression performance during NLP

Browse Source 
Improve performance by pre-compiling regular expressions that are executed many times.
This commit is contained in:
tsukumi
2024-05-12 00:26:46 +09:00
parent 2d6baa3928
commit 44693e7d6b
5 changed files with 171 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
style_bert_vits2/nlp/chinese/normalizer.py
View File

@@ -5,6 +5,41 @@ import cn2an
from style_bert_vits2.nlp.symbols import PUNCTUATIONS
__REPLACE_MAP = {
"：": ",",
"；": ",",
"，": ",",
"。": ".",
"！": "!",
"？": "?",
"\n": ".",
"·": ",",
"、": ",",
"...": "…",
"$": ".",
"“": "'",
"”": "'",
'"': "'",
"‘": "'",
"’": "'",
"（": "'",
"）": "'",
"(": "'",
")": "'",
"《": "'",
"》": "'",
"【": "'",
"】": "'",
"[": "'",
"]": "'",
"—": "-",
"～": "-",
"~": "-",
"「": "'",
"」": "'",
}
def normalize_text(text: str) -> str:
numbers = re.findall(r"\d+(?:\.?\d+)?", text)
for number in numbers:
@@ -15,44 +50,10 @@ def normalize_text(text: str) -> str:
def replace_punctuation(text: str) -> str:
REPLACE_MAP = {
"：": ",",
"；": ",",
"，": ",",
"。": ".",
"！": "!",
"？": "?",
"\n": ".",
"·": ",",
"、": ",",
"...": "…",
"$": ".",
"“": "'",
"”": "'",
'"': "'",
"‘": "'",
"’": "'",
"（": "'",
"）": "'",
"(": "'",
")": "'",
"《": "'",
"》": "'",
"【": "'",
"】": "'",
"[": "'",
"]": "'",
"—": "-",
"～": "-",
"~": "-",
"「": "'",
"」": "'",
}
text = text.replace("嗯", "恩").replace("呣", "母")
pattern = re.compile("|".join(re.escape(p) for p in REPLACE_MAP.keys()))
pattern = re.compile("|".join(re.escape(p) for p in __REPLACE_MAP.keys()))
replaced_text = pattern.sub(lambda x: REPLACE_MAP[x.group()], text)
replaced_text = pattern.sub(lambda x: __REPLACE_MAP[x.group()], text)
replaced_text = re.sub(
r"[^\u4e00-\u9fa5" + "".join(PUNCTUATIONS) + r"]+", "", replaced_text