From 2c50b7816b3ce769dbe70e625a883ea3b5d7caec Mon Sep 17 00:00:00 2001 From: Artrajz <969242373@qq.com> Date: Fri, 1 Dec 2023 19:20:31 +0800 Subject: [PATCH] Optimize the recognition of mixed Chinese and English characters in numbers. (#212) * Optimize the recognition of mixed Chinese and English characters in numbers. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- tools/classify_language.py | 33 +++++++++++++++++++++++++-------- tools/sentence.py | 35 +++++++++++++++++++++++------------ 2 files changed, 48 insertions(+), 20 deletions(-) diff --git a/tools/classify_language.py b/tools/classify_language.py index 5f13c50..185bf2b 100644 --- a/tools/classify_language.py +++ b/tools/classify_language.py @@ -1,8 +1,13 @@ -import re +import regex as re -from config import config +try: + from config import config -LANGUAGE_IDENTIFICATION_LIBRARY = config.webui_config.language_identification_library + LANGUAGE_IDENTIFICATION_LIBRARY = ( + config.webui_config.language_identification_library + ) +except: + LANGUAGE_IDENTIFICATION_LIBRARY = "langid" module = LANGUAGE_IDENTIFICATION_LIBRARY.lower() @@ -155,11 +160,15 @@ def classify_zh_ja(text: str) -> str: return "zh" -def split_alpha_nonalpha(text): - return re.split( - r"(?:(?<=[\u4e00-\u9fff])|(?<=[\u3040-\u30FF]))(?=[a-zA-Z])|(?<=[a-zA-Z])(?:(?=[\u4e00-\u9fff])|(?=[\u3040-\u30FF]))", - text, - ) +def split_alpha_nonalpha(text, mode=1): + if mode == 1: + pattern = r"(?<=[\u4e00-\u9fff\u3040-\u30FF\d])(?=[\p{Latin}])|(?<=[\p{Latin}])(?=[\u4e00-\u9fff\u3040-\u30FF\d])" + elif mode == 2: + pattern = r"(?<=[\u4e00-\u9fff\u3040-\u30FF])(?=[\p{Latin}\d])|(?<=[\p{Latin}\d])(?=[\u4e00-\u9fff\u3040-\u30FF])" + else: + raise ValueError("Invalid mode. Supported modes are 1 and 2.") + + return re.split(pattern, text) if __name__ == "__main__": @@ -170,3 +179,11 @@ if __name__ == "__main__": text = "これはテストテキストです" print(classify_language(text)) print(classify_zh_ja(text)) # "ja" + + text = "vits和Bert-VITS2是tts模型。花费3days.花费3天。Take 3 days" + + print(split_alpha_nonalpha(text, mode=1)) + # output: ['vits', '和', 'Bert-VITS', '2是', 'tts', '模型。花费3', 'days.花费3天。Take 3 days'] + + print(split_alpha_nonalpha(text, mode=2)) + # output: ['vits', '和', 'Bert-VITS2', '是', 'tts', '模型。花费', '3days.花费', '3', '天。Take 3 days'] diff --git a/tools/sentence.py b/tools/sentence.py index e033010..a4bd48d 100644 --- a/tools/sentence.py +++ b/tools/sentence.py @@ -25,12 +25,13 @@ def markup_language(text: str, target_languages: list = None) -> str: pre_lang = "" p = 0 - sorted_target_languages = sorted(target_languages) - if sorted_target_languages in [["en", "zh"], ["en", "ja"], ["en", "ja", "zh"]]: - new_sentences = [] - for sentence in sentences: - new_sentences.extend(split_alpha_nonalpha(sentence)) - sentences = new_sentences + if target_languages is not None: + sorted_target_languages = sorted(target_languages) + if sorted_target_languages in [["en", "zh"], ["en", "ja"], ["en", "ja", "zh"]]: + new_sentences = [] + for sentence in sentences: + new_sentences.extend(split_alpha_nonalpha(sentence)) + sentences = new_sentences for sentence in sentences: if check_is_none(sentence): @@ -68,12 +69,13 @@ def split_by_language(text: str, target_languages: list = None) -> list: end = 0 sentences_list = [] - sorted_target_languages = sorted(target_languages) - if sorted_target_languages in [["en", "zh"], ["en", "ja"], ["en", "ja", "zh"]]: - new_sentences = [] - for sentence in sentences: - new_sentences.extend(split_alpha_nonalpha(sentence)) - sentences = new_sentences + if target_languages is not None: + sorted_target_languages = sorted(target_languages) + if sorted_target_languages in [["en", "zh"], ["en", "ja"], ["en", "ja", "zh"]]: + new_sentences = [] + for sentence in sentences: + new_sentences.extend(split_alpha_nonalpha(sentence)) + sentences = new_sentences for sentence in sentences: if check_is_none(sentence): @@ -154,5 +156,14 @@ if __name__ == "__main__": print(markup_language(text, target_languages=None)) print(sentence_split(text, max=50)) print(sentence_split_and_markup(text, max=50, lang="auto", speaker_lang=None)) + text = "你好，这是一段用来测试自动标注的文本。こんにちは,これは自動ラベリングのテスト用テキストです.Hello, this is a piece of text to test autotagging.你好！今天我们要介绍VITS项目，其重点是使用了GAN Duration predictor和transformer flow,并且接入了Bert模型来提升韵律。Bert embedding会在稍后介绍。" print(split_by_language(text, ["zh", "ja", "en"])) + + text = "vits和Bert-VITS2是tts模型。花费3days.花费3天。Take 3 days" + + print(split_by_language(text, ["zh", "ja", "en"])) + # output: [('vits', 'en'), ('和', 'ja'), ('Bert-VITS', 'en'), ('2是', 'zh'), ('tts', 'en'), ('模型。花费3', 'zh'), ('days.', 'en'), ('花费3天。', 'zh'), ('Take 3 days', 'en')] + + print(split_by_language(text, ["zh", "en"])) + # output: [('vits', 'en'), ('和', 'zh'), ('Bert-VITS', 'en'), ('2是', 'zh'), ('tts', 'en'), ('模型。花费3', 'zh'), ('days.', 'en'), ('花费3天。', 'zh'), ('Take 3 days', 'en')]