Optimize the recognition of mixed Chinese and English characters in numbers. (#212)

* Optimize the recognition of mixed Chinese and English characters in numbers.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

tools/classify_language.py

@@ -1,8 +1,13 @@
-import re
+import regex as re
 
-from config import config
+try:
+    from config import config
 
-LANGUAGE_IDENTIFICATION_LIBRARY = config.webui_config.language_identification_library
+    LANGUAGE_IDENTIFICATION_LIBRARY = (
+        config.webui_config.language_identification_library
+    )
+except:
+    LANGUAGE_IDENTIFICATION_LIBRARY = "langid"
 
 module = LANGUAGE_IDENTIFICATION_LIBRARY.lower()
 
@@ -155,11 +160,15 @@ def classify_zh_ja(text: str) -> str:
     return "zh"
 
 
-def split_alpha_nonalpha(text):
+def split_alpha_nonalpha(text, mode=1):
-    return re.split(
+    if mode == 1:
-        r"(?:(?<=[\u4e00-\u9fff])|(?<=[\u3040-\u30FF]))(?=[a-zA-Z])|(?<=[a-zA-Z])(?:(?=[\u4e00-\u9fff])|(?=[\u3040-\u30FF]))",
+        pattern = r"(?<=[\u4e00-\u9fff\u3040-\u30FF\d])(?=[\p{Latin}])|(?<=[\p{Latin}])(?=[\u4e00-\u9fff\u3040-\u30FF\d])"
-        text,
+    elif mode == 2:
-    )
+        pattern = r"(?<=[\u4e00-\u9fff\u3040-\u30FF])(?=[\p{Latin}\d])|(?<=[\p{Latin}\d])(?=[\u4e00-\u9fff\u3040-\u30FF])"
+    else:
+        raise ValueError("Invalid mode. Supported modes are 1 and 2.")
+
+    return re.split(pattern, text)
 
 
 if __name__ == "__main__":
@@ -170,3 +179,11 @@ if __name__ == "__main__":
     text = "これはテストテキストです"
     print(classify_language(text))
     print(classify_zh_ja(text))  # "ja"
+
+    text = "vits和Bert-VITS2是tts模型。花费3days.花费3天。Take 3 days"
+
+    print(split_alpha_nonalpha(text, mode=1))
+    # output: ['vits', '和', 'Bert-VITS', '2是', 'tts', '模型。花费3', 'days.花费3天。Take 3 days']
+
+    print(split_alpha_nonalpha(text, mode=2))
+    # output: ['vits', '和', 'Bert-VITS2', '是', 'tts', '模型。花费', '3days.花费', '3', '天。Take 3 days']

tools/sentence.py

@@ -25,12 +25,13 @@ def markup_language(text: str, target_languages: list = None) -> str:
     pre_lang = ""
     p = 0
 
-    sorted_target_languages = sorted(target_languages)
+    if target_languages is not None:
-    if sorted_target_languages in [["en", "zh"], ["en", "ja"], ["en", "ja", "zh"]]:
+        sorted_target_languages = sorted(target_languages)
-        new_sentences = []
+        if sorted_target_languages in [["en", "zh"], ["en", "ja"], ["en", "ja", "zh"]]:
-        for sentence in sentences:
+            new_sentences = []
-            new_sentences.extend(split_alpha_nonalpha(sentence))
+            for sentence in sentences:
-        sentences = new_sentences
+                new_sentences.extend(split_alpha_nonalpha(sentence))
+            sentences = new_sentences
 
     for sentence in sentences:
         if check_is_none(sentence):
@@ -68,12 +69,13 @@ def split_by_language(text: str, target_languages: list = None) -> list:
     end = 0
     sentences_list = []
 
-    sorted_target_languages = sorted(target_languages)
+    if target_languages is not None:
-    if sorted_target_languages in [["en", "zh"], ["en", "ja"], ["en", "ja", "zh"]]:
+        sorted_target_languages = sorted(target_languages)
-        new_sentences = []
+        if sorted_target_languages in [["en", "zh"], ["en", "ja"], ["en", "ja", "zh"]]:
-        for sentence in sentences:
+            new_sentences = []
-            new_sentences.extend(split_alpha_nonalpha(sentence))
+            for sentence in sentences:
-        sentences = new_sentences
+                new_sentences.extend(split_alpha_nonalpha(sentence))
+            sentences = new_sentences
 
     for sentence in sentences:
         if check_is_none(sentence):
@@ -154,5 +156,14 @@ if __name__ == "__main__":
     print(markup_language(text, target_languages=None))
     print(sentence_split(text, max=50))
     print(sentence_split_and_markup(text, max=50, lang="auto", speaker_lang=None))
+
     text = "你好，这是一段用来测试自动标注的文本。こんにちは,これは自動ラベリングのテスト用テキストです.Hello, this is a piece of text to test autotagging.你好！今天我们要介绍VITS项目，其重点是使用了GAN Duration predictor和transformer flow,并且接入了Bert模型来提升韵律。Bert embedding会在稍后介绍。"
     print(split_by_language(text, ["zh", "ja", "en"]))
+
+    text = "vits和Bert-VITS2是tts模型。花费3days.花费3天。Take 3 days"
+
+    print(split_by_language(text, ["zh", "ja", "en"]))
+    # output: [('vits', 'en'), ('和', 'ja'), ('Bert-VITS', 'en'), ('2是', 'zh'), ('tts', 'en'), ('模型。花费3', 'zh'), ('days.', 'en'), ('花费3天。', 'zh'), ('Take 3 days', 'en')]
+
+    print(split_by_language(text, ["zh", "en"]))
+    # output: [('vits', 'en'), ('和', 'zh'), ('Bert-VITS', 'en'), ('2是', 'zh'), ('tts', 'en'), ('模型。花费3', 'zh'), ('days.', 'en'), ('花费3天。', 'zh'), ('Take 3 days', 'en')]