Update zje language identification. (#155)

* Update zje language identification.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

tools/classify_language.py

@@ -1,3 +1,5 @@
+import re
+
 from config import config
 
 LANGUAGE_IDENTIFICATION_LIBRARY = config.webui_config.language_identification_library
@@ -153,6 +155,13 @@ def classify_zh_ja(text: str) -> str:
     return "zh"
 
 
+def split_alpha_nonalpha(text):
+    return re.split(
+        r"(?:(?<=[\u4e00-\u9fff])|(?<=[\u3040-\u30FF]))(?=[a-zA-Z])|(?<=[a-zA-Z])(?:(?=[\u4e00-\u9fff])|(?=[\u3040-\u30FF]))",
+        text,
+    )
+
+
 if __name__ == "__main__":
     text = "这是一个测试文本"
     print(classify_language(text))