From a752307e1edd6c3460e0da1de7c73deef5d54305 Mon Sep 17 00:00:00 2001
From: Artrajz <969242373@qq.com>
Date: Tue, 7 Nov 2023 10:43:06 +0800
Subject: [PATCH] Update zje language identification. (#155)

* Update zje language identification.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
---
 tools/classify_language.py |  9 +++++++++
 tools/sentence.py          | 18 ++++++++++++++++--
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/tools/classify_language.py b/tools/classify_language.py
index 25641f6..5f13c50 100644
--- a/tools/classify_language.py
+++ b/tools/classify_language.py
@@ -1,3 +1,5 @@
+import re
+
 from config import config
 
 LANGUAGE_IDENTIFICATION_LIBRARY = config.webui_config.language_identification_library
@@ -153,6 +155,13 @@ def classify_zh_ja(text: str) -> str:
     return "zh"
 
 
+def split_alpha_nonalpha(text):
+    return re.split(
+        r"(?:(?<=[\u4e00-\u9fff])|(?<=[\u3040-\u30FF]))(?=[a-zA-Z])|(?<=[a-zA-Z])(?:(?=[\u4e00-\u9fff])|(?=[\u3040-\u30FF]))",
+        text,
+    )
+
+
 if __name__ == "__main__":
     text = "这是一个测试文本"
     print(classify_language(text))
diff --git a/tools/sentence.py b/tools/sentence.py
index b6721b7..e033010 100644
--- a/tools/sentence.py
+++ b/tools/sentence.py
@@ -2,7 +2,7 @@ import logging
 
 import regex as re
 
-from tools.classify_language import classify_language
+from tools.classify_language import classify_language, split_alpha_nonalpha
 
 
 def check_is_none(item) -> bool:
@@ -25,6 +25,13 @@ def markup_language(text: str, target_languages: list = None) -> str:
     pre_lang = ""
     p = 0
 
+    sorted_target_languages = sorted(target_languages)
+    if sorted_target_languages in [["en", "zh"], ["en", "ja"], ["en", "ja", "zh"]]:
+        new_sentences = []
+        for sentence in sentences:
+            new_sentences.extend(split_alpha_nonalpha(sentence))
+        sentences = new_sentences
+
     for sentence in sentences:
         if check_is_none(sentence):
             continue
@@ -61,6 +68,13 @@ def split_by_language(text: str, target_languages: list = None) -> list:
     end = 0
     sentences_list = []
 
+    sorted_target_languages = sorted(target_languages)
+    if sorted_target_languages in [["en", "zh"], ["en", "ja"], ["en", "ja", "zh"]]:
+        new_sentences = []
+        for sentence in sentences:
+            new_sentences.extend(split_alpha_nonalpha(sentence))
+        sentences = new_sentences
+
     for sentence in sentences:
         if check_is_none(sentence):
             continue
@@ -140,5 +154,5 @@ if __name__ == "__main__":
     print(markup_language(text, target_languages=None))
     print(sentence_split(text, max=50))
     print(sentence_split_and_markup(text, max=50, lang="auto", speaker_lang=None))
-    text = "你好，这是一段用来测试自动标注的文本。こんにちは,これは自動ラベリングのテスト用テキストです.Hello, this is a piece of text to test autotagging."
+    text = "你好，这是一段用来测试自动标注的文本。こんにちは,これは自動ラベリングのテスト用テキストです.Hello, this is a piece of text to test autotagging.你好！今天我们要介绍VITS项目，其重点是使用了GAN Duration predictor和transformer flow,并且接入了Bert模型来提升韵律。Bert embedding会在稍后介绍。"
     print(split_by_language(text, ["zh", "ja", "en"]))