tuna2134/sbv2-v2
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Update zje language identification. (#155)

Browse Source 
* Update zje language identification.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Artrajz
2023-11-07 10:43:06 +08:00
committed by GitHub
parent a233fb44fc
commit a752307e1e
2 changed files with 25 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
tools/sentence.py
View File

@@ -2,7 +2,7 @@ import logging
import regex as re
from tools.classify_language import classify_language
from tools.classify_language import classify_language, split_alpha_nonalpha
def check_is_none(item) -> bool:
@@ -25,6 +25,13 @@ def markup_language(text: str, target_languages: list = None) -> str:
pre_lang = ""
p = 0
sorted_target_languages = sorted(target_languages)
if sorted_target_languages in [["en", "zh"], ["en", "ja"], ["en", "ja", "zh"]]:
new_sentences = []
for sentence in sentences:
new_sentences.extend(split_alpha_nonalpha(sentence))
sentences = new_sentences
for sentence in sentences:
if check_is_none(sentence):
continue
@@ -61,6 +68,13 @@ def split_by_language(text: str, target_languages: list = None) -> list:
end = 0
sentences_list = []
sorted_target_languages = sorted(target_languages)
if sorted_target_languages in [["en", "zh"], ["en", "ja"], ["en", "ja", "zh"]]:
new_sentences = []
for sentence in sentences:
new_sentences.extend(split_alpha_nonalpha(sentence))
sentences = new_sentences
for sentence in sentences:
if check_is_none(sentence):
continue
@@ -140,5 +154,5 @@ if __name__ == "__main__":
print(markup_language(text, target_languages=None))
print(sentence_split(text, max=50))
print(sentence_split_and_markup(text, max=50, lang="auto", speaker_lang=None))
text = "你好，这是一段用来测试自动标注的文本。こんにちは,これは自動ラベリングのテスト用テキストです.Hello, this is a piece of text to test autotagging."
text = "你好，这是一段用来测试自动标注的文本。こんにちは,これは自動ラベリングのテスト用テキストです.Hello, this is a piece of text to test autotagging.你好！今天我们要介绍VITS项目，其重点是使用了GAN Duration predictor和transformer flow,并且接入了Bert模型来提升韵律。Bert embedding会在稍后介绍。"
print(split_by_language(text, ["zh", "ja", "en"]))