tuna2134/sbv2-v2
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix mismatch when there is space between alpha and non-alpha words (#222)

Browse Source 
* fix mismatch when there is space between alpha and non-alpha words

* revert blank line

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Sihan Chen
2023-12-08 11:34:51 +08:00
committed by GitHub
parent 698d2bfa5b
commit f1a1b4c73d
2 changed files with 14 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
tools/classify_language.py
View File

@@ -162,9 +162,9 @@ def classify_zh_ja(text: str) -> str:
def split_alpha_nonalpha(text, mode=1): def split_alpha_nonalpha(text, mode=1):
if mode == 1: if mode == 1:
pattern = r"(?<=[\u4e00-\u9fff\u3040-\u30FF\d])(?=[\p{Latin}])|(?<=[\p{Latin}])(?=[\u4e00-\u9fff\u3040-\u30FF\d])" pattern = r"(?<=[\u4e00-\u9fff\u3040-\u30FF\d\s])(?=[\p{Latin}])|(?<=[\p{Latin}\s])(?=[\u4e00-\u9fff\u3040-\u30FF\d])"
elif mode == 2: elif mode == 2:
pattern = r"(?<=[\u4e00-\u9fff\u3040-\u30FF])(?=[\p{Latin}\d])|(?<=[\p{Latin}\d])(?=[\u4e00-\u9fff\u3040-\u30FF])" pattern = r"(?<=[\u4e00-\u9fff\u3040-\u30FF\s])(?=[\p{Latin}\d])|(?<=[\p{Latin}\d\s])(?=[\u4e00-\u9fff\u3040-\u30FF])"
else: else:
raise ValueError("Invalid mode. Supported modes are 1 and 2.") raise ValueError("Invalid mode. Supported modes are 1 and 2.")
@@ -187,3 +187,11 @@ if __name__ == "__main__":
print(split_alpha_nonalpha(text, mode=2)) print(split_alpha_nonalpha(text, mode=2))
# output: ['vits', '和', 'Bert-VITS2', '是', 'tts', '模型。花费', '3days.花费', '3', '天。Take 3 days'] # output: ['vits', '和', 'Bert-VITS2', '是', 'tts', '模型。花费', '3days.花费', '3', '天。Take 3 days']
text = "vits 和 Bert-VITS2 是 tts 模型。花费3days.花费3天。Take 3 days"
print(split_alpha_nonalpha(text, mode=1))
# output: ['vits ', '和 ', 'Bert-VITS', '2 ', '是 ', 'tts ', '模型。花费3', 'days.花费3天。Take ', '3 ', 'days']
text = "vits 和 Bert-VITS2 是 tts 模型。花费3days.花费3天。Take 3 days"
print(split_alpha_nonalpha(text, mode=2))
# output: ['vits ', '和 ', 'Bert-VITS2 ', '是 ', 'tts ', '模型。花费', '3days.花费', '3', '天。Take ', '3 ', 'days']

4
tools/sentence.py
View File

@@ -167,3 +167,7 @@ if __name__ == "__main__":
print(split_by_language(text, ["zh", "en"])) print(split_by_language(text, ["zh", "en"]))
# output: [('vits', 'en'), ('和', 'zh'), ('Bert-VITS', 'en'), ('2是', 'zh'), ('tts', 'en'), ('模型。花费3', 'zh'), ('days.', 'en'), ('花费3天。', 'zh'), ('Take 3 days', 'en')] # output: [('vits', 'en'), ('和', 'zh'), ('Bert-VITS', 'en'), ('2是', 'zh'), ('tts', 'en'), ('模型。花费3', 'zh'), ('days.', 'en'), ('花费3天。', 'zh'), ('Take 3 days', 'en')]
text = "vits 和 Bert-VITS2 是 tts 模型。花费 3 days. 花费 3天。Take 3 days"
print(split_by_language(text, ["zh", "en"]))
# output: [('vits ', 'en'), ('和 ', 'zh'), ('Bert-VITS2 ', 'en'), ('是 ', 'zh'), ('tts ', 'en'), ('模型。花费 ', 'zh'), ('3 days. ', 'en'), ('花费 3天。', 'zh'), ('Take 3 days', 'en')]