From f1a1b4c73d92e16542edd4394359f8a587dab531 Mon Sep 17 00:00:00 2001
From: Sihan Chen <39623753+Spycsh@users.noreply.github.com>
Date: Fri, 8 Dec 2023 11:34:51 +0800
Subject: [PATCH] fix mismatch when there is space between alpha and non-alpha
 words (#222)

* fix mismatch when there is space between alpha and non-alpha words

* revert blank line

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
---
 tools/classify_language.py | 12 ++++++++++--
 tools/sentence.py          |  4 ++++
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/tools/classify_language.py b/tools/classify_language.py
index 185bf2b..2b8a7ab 100644
--- a/tools/classify_language.py
+++ b/tools/classify_language.py
@@ -162,9 +162,9 @@ def classify_zh_ja(text: str) -> str:
 
 def split_alpha_nonalpha(text, mode=1):
     if mode == 1:
-        pattern = r"(?<=[\u4e00-\u9fff\u3040-\u30FF\d])(?=[\p{Latin}])|(?<=[\p{Latin}])(?=[\u4e00-\u9fff\u3040-\u30FF\d])"
+        pattern = r"(?<=[\u4e00-\u9fff\u3040-\u30FF\d\s])(?=[\p{Latin}])|(?<=[\p{Latin}\s])(?=[\u4e00-\u9fff\u3040-\u30FF\d])"
     elif mode == 2:
-        pattern = r"(?<=[\u4e00-\u9fff\u3040-\u30FF])(?=[\p{Latin}\d])|(?<=[\p{Latin}\d])(?=[\u4e00-\u9fff\u3040-\u30FF])"
+        pattern = r"(?<=[\u4e00-\u9fff\u3040-\u30FF\s])(?=[\p{Latin}\d])|(?<=[\p{Latin}\d\s])(?=[\u4e00-\u9fff\u3040-\u30FF])"
     else:
         raise ValueError("Invalid mode. Supported modes are 1 and 2.")
 
@@ -187,3 +187,11 @@ if __name__ == "__main__":
 
     print(split_alpha_nonalpha(text, mode=2))
     # output: ['vits', '和', 'Bert-VITS2', '是', 'tts', '模型。花费', '3days.花费', '3', '天。Take 3 days']
+
+    text = "vits 和 Bert-VITS2 是 tts 模型。花费3days.花费3天。Take 3 days"
+    print(split_alpha_nonalpha(text, mode=1))
+    # output: ['vits ', '和 ', 'Bert-VITS', '2 ', '是 ', 'tts ', '模型。花费3', 'days.花费3天。Take ', '3 ', 'days']
+
+    text = "vits 和 Bert-VITS2 是 tts 模型。花费3days.花费3天。Take 3 days"
+    print(split_alpha_nonalpha(text, mode=2))
+    # output: ['vits ', '和 ', 'Bert-VITS2 ', '是 ', 'tts ', '模型。花费', '3days.花费', '3', '天。Take ', '3 ', 'days']
diff --git a/tools/sentence.py b/tools/sentence.py
index a4bd48d..b66864c 100644
--- a/tools/sentence.py
+++ b/tools/sentence.py
@@ -167,3 +167,7 @@ if __name__ == "__main__":
 
     print(split_by_language(text, ["zh", "en"]))
     # output: [('vits', 'en'), ('和', 'zh'), ('Bert-VITS', 'en'), ('2是', 'zh'), ('tts', 'en'), ('模型。花费3', 'zh'), ('days.', 'en'), ('花费3天。', 'zh'), ('Take 3 days', 'en')]
+
+    text = "vits 和 Bert-VITS2 是 tts 模型。花费 3 days. 花费 3天。Take 3 days"
+    print(split_by_language(text, ["zh", "en"]))
+    # output: [('vits ', 'en'), ('和 ', 'zh'), ('Bert-VITS2 ', 'en'), ('是 ', 'zh'), ('tts ', 'en'), ('模型。花费 ', 'zh'), ('3 days. ', 'en'), ('花费 3天。', 'zh'), ('Take 3 days', 'en')]