From c86da6eeeb32216cf7b3cf91523171211ce2f8b4 Mon Sep 17 00:00:00 2001 From: Artrajz <969242373@qq.com> Date: Mon, 6 Nov 2023 21:56:09 +0800 Subject: [PATCH] Add language identification. (#153) * Add language identification. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- config.py | 4 + default_config.yml | 2 + requirements.txt | 1 + tools/classify_language.py | 163 +++++++++++++++++++++++++++++++++++++ tools/sentence.py | 144 ++++++++++++++++++++++++++++++++ webui.py | 23 +++++- 6 files changed, 336 insertions(+), 1 deletion(-) create mode 100644 tools/classify_language.py create mode 100644 tools/sentence.py diff --git a/config.py b/config.py index 155fd32..dbbd980 100644 --- a/config.py +++ b/config.py @@ -142,6 +142,7 @@ class Webui_config: device: str, model: str, config_path: str, + language_identification_library: str, port: int = 7860, share: bool = False, debug: bool = False, @@ -152,6 +153,9 @@ class Webui_config: self.port: int = port # 是否开启debug模式 self.share: bool = share # 模型路径 self.debug: bool = debug # 配置文件路径 + self.language_identification_library: str = ( + language_identification_library # 语种识别库 + ) @classmethod def from_dict(cls, dataset_path: str, data: Dict[str, any]): diff --git a/default_config.yml b/default_config.yml index e8a6379..a8e078d 100644 --- a/default_config.yml +++ b/default_config.yml @@ -95,6 +95,8 @@ webui: share: false # 是否开启debug模式 debug: false + # 语种识别库，可选langid, fastlid + language_identification_library: "langid" # server api配置 diff --git a/requirements.txt b/requirements.txt index 24e6a45..46a50c7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,3 +30,4 @@ vector_quantize_pytorch g2p_en sentencepiece pykakasi +langid diff --git a/tools/classify_language.py b/tools/classify_language.py new file mode 100644 index 0000000..25641f6 --- /dev/null +++ b/tools/classify_language.py @@ -0,0 +1,163 @@ +from config import config + +LANGUAGE_IDENTIFICATION_LIBRARY = config.webui_config.language_identification_library + +module = LANGUAGE_IDENTIFICATION_LIBRARY.lower() + +langid_languages = [ + "af", + "am", + "an", + "ar", + "as", + "az", + "be", + "bg", + "bn", + "br", + "bs", + "ca", + "cs", + "cy", + "da", + "de", + "dz", + "el", + "en", + "eo", + "es", + "et", + "eu", + "fa", + "fi", + "fo", + "fr", + "ga", + "gl", + "gu", + "he", + "hi", + "hr", + "ht", + "hu", + "hy", + "id", + "is", + "it", + "ja", + "jv", + "ka", + "kk", + "km", + "kn", + "ko", + "ku", + "ky", + "la", + "lb", + "lo", + "lt", + "lv", + "mg", + "mk", + "ml", + "mn", + "mr", + "ms", + "mt", + "nb", + "ne", + "nl", + "nn", + "no", + "oc", + "or", + "pa", + "pl", + "ps", + "pt", + "qu", + "ro", + "ru", + "rw", + "se", + "si", + "sk", + "sl", + "sq", + "sr", + "sv", + "sw", + "ta", + "te", + "th", + "tl", + "tr", + "ug", + "uk", + "ur", + "vi", + "vo", + "wa", + "xh", + "zh", + "zu", +] + + +def classify_language(text: str, target_languages: list = None) -> str: + if module == "fastlid" or module == "fasttext": + from fastlid import fastlid, supported_langs + + classifier = fastlid + if target_languages != None: + target_languages = [ + lang for lang in target_languages if lang in supported_langs + ] + fastlid.set_languages = target_languages + elif module == "langid": + import langid + + classifier = langid.classify + if target_languages != None: + target_languages = [ + lang for lang in target_languages if lang in langid_languages + ] + langid.set_languages(target_languages) + else: + raise ValueError(f"Wrong module {module}") + + lang = classifier(text)[0] + + return lang + + +def classify_zh_ja(text: str) -> str: + for idx, char in enumerate(text): + unicode_val = ord(char) + + # 检测日语字符 + if 0x3040 <= unicode_val <= 0x309F or 0x30A0 <= unicode_val <= 0x30FF: + return "ja" + + # 检测汉字字符 + if 0x4E00 <= unicode_val <= 0x9FFF: + # 检查周围的字符 + next_char = text[idx + 1] if idx + 1 < len(text) else None + + if next_char and ( + 0x3040 <= ord(next_char) <= 0x309F or 0x30A0 <= ord(next_char) <= 0x30FF + ): + return "ja" + + return "zh" + + +if __name__ == "__main__": + text = "这是一个测试文本" + print(classify_language(text)) + print(classify_zh_ja(text)) # "zh" + + text = "これはテストテキストです" + print(classify_language(text)) + print(classify_zh_ja(text)) # "ja" diff --git a/tools/sentence.py b/tools/sentence.py new file mode 100644 index 0000000..b6721b7 --- /dev/null +++ b/tools/sentence.py @@ -0,0 +1,144 @@ +import logging + +import regex as re + +from tools.classify_language import classify_language + + +def check_is_none(item) -> bool: + """none -> True, not none -> False""" + return ( + item is None + or (isinstance(item, str) and str(item).isspace()) + or str(item) == "" + ) + + +def markup_language(text: str, target_languages: list = None) -> str: + pattern = ( + r"[\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\>\=\?\@\[\]\{\}\\\\\^\_\`" + r"\！？。＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」" + r"『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘\'\‛\“\”\„\‟…‧﹏.]+" + ) + sentences = re.split(pattern, text) + + pre_lang = "" + p = 0 + + for sentence in sentences: + if check_is_none(sentence): + continue + + lang = classify_language(sentence, target_languages) + + if pre_lang == "": + text = text[:p] + text[p:].replace( + sentence, f"[{lang.upper()}]{sentence}", 1 + ) + p += len(f"[{lang.upper()}]") + elif pre_lang != lang: + text = text[:p] + text[p:].replace( + sentence, f"[{pre_lang.upper()}][{lang.upper()}]{sentence}", 1 + ) + p += len(f"[{pre_lang.upper()}][{lang.upper()}]") + pre_lang = lang + p += text[p:].index(sentence) + len(sentence) + text += f"[{pre_lang.upper()}]" + + return text + + +def split_by_language(text: str, target_languages: list = None) -> list: + pattern = ( + r"[\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\>\=\?\@\[\]\{\}\\\\\^\_\`" + r"\！？\。＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」" + r"『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘\'\‛\“\”\„\‟…‧﹏.]+" + ) + sentences = re.split(pattern, text) + + pre_lang = "" + start = 0 + end = 0 + sentences_list = [] + + for sentence in sentences: + if check_is_none(sentence): + continue + + lang = classify_language(sentence, target_languages) + + end += text[end:].index(sentence) + if pre_lang != "" and pre_lang != lang: + sentences_list.append((text[start:end], pre_lang)) + start = end + end += len(sentence) + pre_lang = lang + sentences_list.append((text[start:], pre_lang)) + + return sentences_list + + +def sentence_split(text: str, max: int) -> list: + pattern = r"[!(),—+\-.:;?？。，、；：]+" + sentences = re.split(pattern, text) + discarded_chars = re.findall(pattern, text) + + sentences_list, count, p = [], 0, 0 + + # 按被分割的符号遍历 + for i, discarded_chars in enumerate(discarded_chars): + count += len(sentences[i]) + len(discarded_chars) + if count >= max: + sentences_list.append(text[p : p + count].strip()) + p += count + count = 0 + + # 加入最后剩余的文本 + if p < len(text): + sentences_list.append(text[p:]) + + return sentences_list + + +def sentence_split_and_markup(text, max=50, lang="auto", speaker_lang=None): + # 如果该speaker只支持一种语言 + if speaker_lang is not None and len(speaker_lang) == 1: + if lang.upper() not in ["AUTO", "MIX"] and lang.lower() != speaker_lang[0]: + logging.debug( + f'lang "{lang}" is not in speaker_lang {speaker_lang},automatically set lang={speaker_lang[0]}' + ) + lang = speaker_lang[0] + + sentences_list = [] + if lang.upper() != "MIX": + if max <= 0: + sentences_list.append( + markup_language(text, speaker_lang) + if lang.upper() == "AUTO" + else f"[{lang.upper()}]{text}[{lang.upper()}]" + ) + else: + for i in sentence_split(text, max): + if check_is_none(i): + continue + sentences_list.append( + markup_language(i, speaker_lang) + if lang.upper() == "AUTO" + else f"[{lang.upper()}]{i}[{lang.upper()}]" + ) + else: + sentences_list.append(text) + + for i in sentences_list: + logging.debug(i) + + return sentences_list + + +if __name__ == "__main__": + text = "这几天心里颇不宁静。今晚在院子里坐着乘凉，忽然想起日日走过的荷塘，在这满月的光里，总该另有一番样子吧。月亮渐渐地升高了，墙外马路上孩子们的欢笑，已经听不见了；妻在屋里拍着闰儿，迷迷糊糊地哼着眠歌。我悄悄地披了大衫，带上门出去。" + print(markup_language(text, target_languages=None)) + print(sentence_split(text, max=50)) + print(sentence_split_and_markup(text, max=50, lang="auto", speaker_lang=None)) + text = "你好，这是一段用来测试自动标注的文本。こんにちは,これは自動ラベリングのテスト用テキストです.Hello, this is a piece of text to test autotagging." + print(split_by_language(text, ["zh", "ja", "en"])) diff --git a/webui.py b/webui.py index ed85af3..1f74d89 100644 --- a/webui.py +++ b/webui.py @@ -1,7 +1,9 @@ # flake8: noqa: E402 import os import logging + import re_matching +from tools.sentence import split_by_language, sentence_split logging.getLogger("numba").setLevel(logging.WARNING) logging.getLogger("markdown_it").setLevel(logging.WARNING) @@ -162,6 +164,25 @@ def tts_fn( lang, ) ) + elif language.lower() == "auto": + sentences_list = split_by_language(text, target_languages=["zh", "ja", "en"]) + for sentences, lang in sentences_list: + lang = lang.upper() + if lang == "JA": + lang = "JP" + sentences = sentence_split(sentences, max=250) + for content in sentences: + audio_list.extend( + generate_audio( + content.split("|"), + sdp_ratio, + noise_scale, + noise_scale_w, + length_scale, + speaker, + lang, + ) + ) else: audio_list.extend( generate_audio( @@ -191,7 +212,7 @@ if __name__ == "__main__": ) speaker_ids = hps.data.spk2id speakers = list(speaker_ids.keys()) - languages = ["ZH", "JP", "EN", "mix"] + languages = ["ZH", "JP", "EN", "mix", "auto"] with gr.Blocks() as app: with gr.Row(): with gr.Column():