Add language identification. (#153)

* Add language identification.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

config.py

@@ -142,6 +142,7 @@ class Webui_config:
         device: str,
         model: str,
         config_path: str,
+        language_identification_library: str,
         port: int = 7860,
         share: bool = False,
         debug: bool = False,
@@ -152,6 +153,9 @@ class Webui_config:
         self.port: int = port  # 是否开启debug模式
         self.share: bool = share  # 模型路径
         self.debug: bool = debug  # 配置文件路径
+        self.language_identification_library: str = (
+            language_identification_library  # 语种识别库
+        )
 
     @classmethod
     def from_dict(cls, dataset_path: str, data: Dict[str, any]):

default_config.yml

@@ -95,6 +95,8 @@ webui:
   share: false
   # 是否开启debug模式
   debug: false
+  # 语种识别库，可选langid, fastlid
+  language_identification_library: "langid"
 
 
 # server api配置

requirements.txt

@@ -30,3 +30,4 @@ vector_quantize_pytorch
 g2p_en
 sentencepiece
 pykakasi
+langid

tools/classify_language.py

@@ -0,0 +1,163 @@
+from config import config
+
+LANGUAGE_IDENTIFICATION_LIBRARY = config.webui_config.language_identification_library
+
+module = LANGUAGE_IDENTIFICATION_LIBRARY.lower()
+
+langid_languages = [
+    "af",
+    "am",
+    "an",
+    "ar",
+    "as",
+    "az",
+    "be",
+    "bg",
+    "bn",
+    "br",
+    "bs",
+    "ca",
+    "cs",
+    "cy",
+    "da",
+    "de",
+    "dz",
+    "el",
+    "en",
+    "eo",
+    "es",
+    "et",
+    "eu",
+    "fa",
+    "fi",
+    "fo",
+    "fr",
+    "ga",
+    "gl",
+    "gu",
+    "he",
+    "hi",
+    "hr",
+    "ht",
+    "hu",
+    "hy",
+    "id",
+    "is",
+    "it",
+    "ja",
+    "jv",
+    "ka",
+    "kk",
+    "km",
+    "kn",
+    "ko",
+    "ku",
+    "ky",
+    "la",
+    "lb",
+    "lo",
+    "lt",
+    "lv",
+    "mg",
+    "mk",
+    "ml",
+    "mn",
+    "mr",
+    "ms",
+    "mt",
+    "nb",
+    "ne",
+    "nl",
+    "nn",
+    "no",
+    "oc",
+    "or",
+    "pa",
+    "pl",
+    "ps",
+    "pt",
+    "qu",
+    "ro",
+    "ru",
+    "rw",
+    "se",
+    "si",
+    "sk",
+    "sl",
+    "sq",
+    "sr",
+    "sv",
+    "sw",
+    "ta",
+    "te",
+    "th",
+    "tl",
+    "tr",
+    "ug",
+    "uk",
+    "ur",
+    "vi",
+    "vo",
+    "wa",
+    "xh",
+    "zh",
+    "zu",
+]
+
+
+def classify_language(text: str, target_languages: list = None) -> str:
+    if module == "fastlid" or module == "fasttext":
+        from fastlid import fastlid, supported_langs
+
+        classifier = fastlid
+        if target_languages != None:
+            target_languages = [
+                lang for lang in target_languages if lang in supported_langs
+            ]
+            fastlid.set_languages = target_languages
+    elif module == "langid":
+        import langid
+
+        classifier = langid.classify
+        if target_languages != None:
+            target_languages = [
+                lang for lang in target_languages if lang in langid_languages
+            ]
+            langid.set_languages(target_languages)
+    else:
+        raise ValueError(f"Wrong module {module}")
+
+    lang = classifier(text)[0]
+
+    return lang
+
+
+def classify_zh_ja(text: str) -> str:
+    for idx, char in enumerate(text):
+        unicode_val = ord(char)
+
+        # 检测日语字符
+        if 0x3040 <= unicode_val <= 0x309F or 0x30A0 <= unicode_val <= 0x30FF:
+            return "ja"
+
+        # 检测汉字字符
+        if 0x4E00 <= unicode_val <= 0x9FFF:
+            # 检查周围的字符
+            next_char = text[idx + 1] if idx + 1 < len(text) else None
+
+            if next_char and (
+                0x3040 <= ord(next_char) <= 0x309F or 0x30A0 <= ord(next_char) <= 0x30FF
+            ):
+                return "ja"
+
+    return "zh"
+
+
+if __name__ == "__main__":
+    text = "这是一个测试文本"
+    print(classify_language(text))
+    print(classify_zh_ja(text))  # "zh"
+
+    text = "これはテストテキストです"
+    print(classify_language(text))
+    print(classify_zh_ja(text))  # "ja"

tools/sentence.py

@@ -0,0 +1,144 @@
+import logging
+
+import regex as re
+
+from tools.classify_language import classify_language
+
+
+def check_is_none(item) -> bool:
+    """none -> True, not none -> False"""
+    return (
+        item is None
+        or (isinstance(item, str) and str(item).isspace())
+        or str(item) == ""
+    )
+
+
+def markup_language(text: str, target_languages: list = None) -> str:
+    pattern = (
+        r"[\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\>\=\?\@\[\]\{\}\\\\\^\_\`"
+        r"\！？。＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」"
+        r"『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘\'\‛\“\”\„\‟…‧﹏.]+"
+    )
+    sentences = re.split(pattern, text)
+
+    pre_lang = ""
+    p = 0
+
+    for sentence in sentences:
+        if check_is_none(sentence):
+            continue
+
+        lang = classify_language(sentence, target_languages)
+
+        if pre_lang == "":
+            text = text[:p] + text[p:].replace(
+                sentence, f"[{lang.upper()}]{sentence}", 1
+            )
+            p += len(f"[{lang.upper()}]")
+        elif pre_lang != lang:
+            text = text[:p] + text[p:].replace(
+                sentence, f"[{pre_lang.upper()}][{lang.upper()}]{sentence}", 1
+            )
+            p += len(f"[{pre_lang.upper()}][{lang.upper()}]")
+        pre_lang = lang
+        p += text[p:].index(sentence) + len(sentence)
+    text += f"[{pre_lang.upper()}]"
+
+    return text
+
+
+def split_by_language(text: str, target_languages: list = None) -> list:
+    pattern = (
+        r"[\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\>\=\?\@\[\]\{\}\\\\\^\_\`"
+        r"\！？\。＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」"
+        r"『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘\'\‛\“\”\„\‟…‧﹏.]+"
+    )
+    sentences = re.split(pattern, text)
+
+    pre_lang = ""
+    start = 0
+    end = 0
+    sentences_list = []
+
+    for sentence in sentences:
+        if check_is_none(sentence):
+            continue
+
+        lang = classify_language(sentence, target_languages)
+
+        end += text[end:].index(sentence)
+        if pre_lang != "" and pre_lang != lang:
+            sentences_list.append((text[start:end], pre_lang))
+            start = end
+        end += len(sentence)
+        pre_lang = lang
+    sentences_list.append((text[start:], pre_lang))
+
+    return sentences_list
+
+
+def sentence_split(text: str, max: int) -> list:
+    pattern = r"[!(),—+\-.:;?？。，、；：]+"
+    sentences = re.split(pattern, text)
+    discarded_chars = re.findall(pattern, text)
+
+    sentences_list, count, p = [], 0, 0
+
+    # 按被分割的符号遍历
+    for i, discarded_chars in enumerate(discarded_chars):
+        count += len(sentences[i]) + len(discarded_chars)
+        if count >= max:
+            sentences_list.append(text[p : p + count].strip())
+            p += count
+            count = 0
+
+    # 加入最后剩余的文本
+    if p < len(text):
+        sentences_list.append(text[p:])
+
+    return sentences_list
+
+
+def sentence_split_and_markup(text, max=50, lang="auto", speaker_lang=None):
+    # 如果该speaker只支持一种语言
+    if speaker_lang is not None and len(speaker_lang) == 1:
+        if lang.upper() not in ["AUTO", "MIX"] and lang.lower() != speaker_lang[0]:
+            logging.debug(
+                f'lang "{lang}" is not in speaker_lang {speaker_lang},automatically set lang={speaker_lang[0]}'
+            )
+        lang = speaker_lang[0]
+
+    sentences_list = []
+    if lang.upper() != "MIX":
+        if max <= 0:
+            sentences_list.append(
+                markup_language(text, speaker_lang)
+                if lang.upper() == "AUTO"
+                else f"[{lang.upper()}]{text}[{lang.upper()}]"
+            )
+        else:
+            for i in sentence_split(text, max):
+                if check_is_none(i):
+                    continue
+                sentences_list.append(
+                    markup_language(i, speaker_lang)
+                    if lang.upper() == "AUTO"
+                    else f"[{lang.upper()}]{i}[{lang.upper()}]"
+                )
+    else:
+        sentences_list.append(text)
+
+    for i in sentences_list:
+        logging.debug(i)
+
+    return sentences_list
+
+
+if __name__ == "__main__":
+    text = "这几天心里颇不宁静。今晚在院子里坐着乘凉，忽然想起日日走过的荷塘，在这满月的光里，总该另有一番样子吧。月亮渐渐地升高了，墙外马路上孩子们的欢笑，已经听不见了；妻在屋里拍着闰儿，迷迷糊糊地哼着眠歌。我悄悄地披了大衫，带上门出去。"
+    print(markup_language(text, target_languages=None))
+    print(sentence_split(text, max=50))
+    print(sentence_split_and_markup(text, max=50, lang="auto", speaker_lang=None))
+    text = "你好，这是一段用来测试自动标注的文本。こんにちは,これは自動ラベリングのテスト用テキストです.Hello, this is a piece of text to test autotagging."
+    print(split_by_language(text, ["zh", "ja", "en"]))

webui.py

@@ -1,7 +1,9 @@
 # flake8: noqa: E402
 import os
 import logging
+
 import re_matching
+from tools.sentence import split_by_language, sentence_split
 
 logging.getLogger("numba").setLevel(logging.WARNING)
 logging.getLogger("markdown_it").setLevel(logging.WARNING)
@@ -162,6 +164,25 @@ def tts_fn(
                         lang,
                     )
                 )
+    elif language.lower() == "auto":
+        sentences_list = split_by_language(text, target_languages=["zh", "ja", "en"])
+        for sentences, lang in sentences_list:
+            lang = lang.upper()
+            if lang == "JA":
+                lang = "JP"
+            sentences = sentence_split(sentences, max=250)
+            for content in sentences:
+                audio_list.extend(
+                    generate_audio(
+                        content.split("|"),
+                        sdp_ratio,
+                        noise_scale,
+                        noise_scale_w,
+                        length_scale,
+                        speaker,
+                        lang,
+                    )
+                )
     else:
         audio_list.extend(
             generate_audio(
@@ -191,7 +212,7 @@ if __name__ == "__main__":
     )
     speaker_ids = hps.data.spk2id
     speakers = list(speaker_ids.keys())
-    languages = ["ZH", "JP", "EN", "mix"]
+    languages = ["ZH", "JP", "EN", "mix", "auto"]
     with gr.Blocks() as app:
         with gr.Row():
             with gr.Column():