Add language identification. (#153)
* Add language identification. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Artrajz
163
tools/classify_language.py
Normal file
163
tools/classify_language.py
Normal file
@@ -0,0 +1,163 @@
|
||||
from config import config
|
||||
|
||||
LANGUAGE_IDENTIFICATION_LIBRARY = config.webui_config.language_identification_library
|
||||
|
||||
module = LANGUAGE_IDENTIFICATION_LIBRARY.lower()
|
||||
|
||||
langid_languages = [
|
||||
"af",
|
||||
"am",
|
||||
"an",
|
||||
"ar",
|
||||
"as",
|
||||
"az",
|
||||
"be",
|
||||
"bg",
|
||||
"bn",
|
||||
"br",
|
||||
"bs",
|
||||
"ca",
|
||||
"cs",
|
||||
"cy",
|
||||
"da",
|
||||
"de",
|
||||
"dz",
|
||||
"el",
|
||||
"en",
|
||||
"eo",
|
||||
"es",
|
||||
"et",
|
||||
"eu",
|
||||
"fa",
|
||||
"fi",
|
||||
"fo",
|
||||
"fr",
|
||||
"ga",
|
||||
"gl",
|
||||
"gu",
|
||||
"he",
|
||||
"hi",
|
||||
"hr",
|
||||
"ht",
|
||||
"hu",
|
||||
"hy",
|
||||
"id",
|
||||
"is",
|
||||
"it",
|
||||
"ja",
|
||||
"jv",
|
||||
"ka",
|
||||
"kk",
|
||||
"km",
|
||||
"kn",
|
||||
"ko",
|
||||
"ku",
|
||||
"ky",
|
||||
"la",
|
||||
"lb",
|
||||
"lo",
|
||||
"lt",
|
||||
"lv",
|
||||
"mg",
|
||||
"mk",
|
||||
"ml",
|
||||
"mn",
|
||||
"mr",
|
||||
"ms",
|
||||
"mt",
|
||||
"nb",
|
||||
"ne",
|
||||
"nl",
|
||||
"nn",
|
||||
"no",
|
||||
"oc",
|
||||
"or",
|
||||
"pa",
|
||||
"pl",
|
||||
"ps",
|
||||
"pt",
|
||||
"qu",
|
||||
"ro",
|
||||
"ru",
|
||||
"rw",
|
||||
"se",
|
||||
"si",
|
||||
"sk",
|
||||
"sl",
|
||||
"sq",
|
||||
"sr",
|
||||
"sv",
|
||||
"sw",
|
||||
"ta",
|
||||
"te",
|
||||
"th",
|
||||
"tl",
|
||||
"tr",
|
||||
"ug",
|
||||
"uk",
|
||||
"ur",
|
||||
"vi",
|
||||
"vo",
|
||||
"wa",
|
||||
"xh",
|
||||
"zh",
|
||||
"zu",
|
||||
]
|
||||
|
||||
|
||||
def classify_language(text: str, target_languages: list = None) -> str:
|
||||
if module == "fastlid" or module == "fasttext":
|
||||
from fastlid import fastlid, supported_langs
|
||||
|
||||
classifier = fastlid
|
||||
if target_languages != None:
|
||||
target_languages = [
|
||||
lang for lang in target_languages if lang in supported_langs
|
||||
]
|
||||
fastlid.set_languages = target_languages
|
||||
elif module == "langid":
|
||||
import langid
|
||||
|
||||
classifier = langid.classify
|
||||
if target_languages != None:
|
||||
target_languages = [
|
||||
lang for lang in target_languages if lang in langid_languages
|
||||
]
|
||||
langid.set_languages(target_languages)
|
||||
else:
|
||||
raise ValueError(f"Wrong module {module}")
|
||||
|
||||
lang = classifier(text)[0]
|
||||
|
||||
return lang
|
||||
|
||||
|
||||
def classify_zh_ja(text: str) -> str:
|
||||
for idx, char in enumerate(text):
|
||||
unicode_val = ord(char)
|
||||
|
||||
# 检测日语字符
|
||||
if 0x3040 <= unicode_val <= 0x309F or 0x30A0 <= unicode_val <= 0x30FF:
|
||||
return "ja"
|
||||
|
||||
# 检测汉字字符
|
||||
if 0x4E00 <= unicode_val <= 0x9FFF:
|
||||
# 检查周围的字符
|
||||
next_char = text[idx + 1] if idx + 1 < len(text) else None
|
||||
|
||||
if next_char and (
|
||||
0x3040 <= ord(next_char) <= 0x309F or 0x30A0 <= ord(next_char) <= 0x30FF
|
||||
):
|
||||
return "ja"
|
||||
|
||||
return "zh"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
text = "这是一个测试文本"
|
||||
print(classify_language(text))
|
||||
print(classify_zh_ja(text)) # "zh"
|
||||
|
||||
text = "これはテストテキストです"
|
||||
print(classify_language(text))
|
||||
print(classify_zh_ja(text)) # "ja"
|
||||
144
tools/sentence.py
Normal file
144
tools/sentence.py
Normal file
@@ -0,0 +1,144 @@
|
||||
import logging
|
||||
|
||||
import regex as re
|
||||
|
||||
from tools.classify_language import classify_language
|
||||
|
||||
|
||||
def check_is_none(item) -> bool:
|
||||
"""none -> True, not none -> False"""
|
||||
return (
|
||||
item is None
|
||||
or (isinstance(item, str) and str(item).isspace())
|
||||
or str(item) == ""
|
||||
)
|
||||
|
||||
|
||||
def markup_language(text: str, target_languages: list = None) -> str:
|
||||
pattern = (
|
||||
r"[\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\>\=\?\@\[\]\{\}\\\\\^\_\`"
|
||||
r"\！？。＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」"
|
||||
r"『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘\'\‛\“\”\„\‟…‧﹏.]+"
|
||||
)
|
||||
sentences = re.split(pattern, text)
|
||||
|
||||
pre_lang = ""
|
||||
p = 0
|
||||
|
||||
for sentence in sentences:
|
||||
if check_is_none(sentence):
|
||||
continue
|
||||
|
||||
lang = classify_language(sentence, target_languages)
|
||||
|
||||
if pre_lang == "":
|
||||
text = text[:p] + text[p:].replace(
|
||||
sentence, f"[{lang.upper()}]{sentence}", 1
|
||||
)
|
||||
p += len(f"[{lang.upper()}]")
|
||||
elif pre_lang != lang:
|
||||
text = text[:p] + text[p:].replace(
|
||||
sentence, f"[{pre_lang.upper()}][{lang.upper()}]{sentence}", 1
|
||||
)
|
||||
p += len(f"[{pre_lang.upper()}][{lang.upper()}]")
|
||||
pre_lang = lang
|
||||
p += text[p:].index(sentence) + len(sentence)
|
||||
text += f"[{pre_lang.upper()}]"
|
||||
|
||||
return text
|
||||
|
||||
|
||||
def split_by_language(text: str, target_languages: list = None) -> list:
|
||||
pattern = (
|
||||
r"[\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\>\=\?\@\[\]\{\}\\\\\^\_\`"
|
||||
r"\！？\。＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」"
|
||||
r"『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘\'\‛\“\”\„\‟…‧﹏.]+"
|
||||
)
|
||||
sentences = re.split(pattern, text)
|
||||
|
||||
pre_lang = ""
|
||||
start = 0
|
||||
end = 0
|
||||
sentences_list = []
|
||||
|
||||
for sentence in sentences:
|
||||
if check_is_none(sentence):
|
||||
continue
|
||||
|
||||
lang = classify_language(sentence, target_languages)
|
||||
|
||||
end += text[end:].index(sentence)
|
||||
if pre_lang != "" and pre_lang != lang:
|
||||
sentences_list.append((text[start:end], pre_lang))
|
||||
start = end
|
||||
end += len(sentence)
|
||||
pre_lang = lang
|
||||
sentences_list.append((text[start:], pre_lang))
|
||||
|
||||
return sentences_list
|
||||
|
||||
|
||||
def sentence_split(text: str, max: int) -> list:
|
||||
pattern = r"[!(),—+\-.:;?？。，、；：]+"
|
||||
sentences = re.split(pattern, text)
|
||||
discarded_chars = re.findall(pattern, text)
|
||||
|
||||
sentences_list, count, p = [], 0, 0
|
||||
|
||||
# 按被分割的符号遍历
|
||||
for i, discarded_chars in enumerate(discarded_chars):
|
||||
count += len(sentences[i]) + len(discarded_chars)
|
||||
if count >= max:
|
||||
sentences_list.append(text[p : p + count].strip())
|
||||
p += count
|
||||
count = 0
|
||||
|
||||
# 加入最后剩余的文本
|
||||
if p < len(text):
|
||||
sentences_list.append(text[p:])
|
||||
|
||||
return sentences_list
|
||||
|
||||
|
||||
def sentence_split_and_markup(text, max=50, lang="auto", speaker_lang=None):
|
||||
# 如果该speaker只支持一种语言
|
||||
if speaker_lang is not None and len(speaker_lang) == 1:
|
||||
if lang.upper() not in ["AUTO", "MIX"] and lang.lower() != speaker_lang[0]:
|
||||
logging.debug(
|
||||
f'lang "{lang}" is not in speaker_lang {speaker_lang},automatically set lang={speaker_lang[0]}'
|
||||
)
|
||||
lang = speaker_lang[0]
|
||||
|
||||
sentences_list = []
|
||||
if lang.upper() != "MIX":
|
||||
if max <= 0:
|
||||
sentences_list.append(
|
||||
markup_language(text, speaker_lang)
|
||||
if lang.upper() == "AUTO"
|
||||
else f"[{lang.upper()}]{text}[{lang.upper()}]"
|
||||
)
|
||||
else:
|
||||
for i in sentence_split(text, max):
|
||||
if check_is_none(i):
|
||||
continue
|
||||
sentences_list.append(
|
||||
markup_language(i, speaker_lang)
|
||||
if lang.upper() == "AUTO"
|
||||
else f"[{lang.upper()}]{i}[{lang.upper()}]"
|
||||
)
|
||||
else:
|
||||
sentences_list.append(text)
|
||||
|
||||
for i in sentences_list:
|
||||
logging.debug(i)
|
||||
|
||||
return sentences_list
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
text = "这几天心里颇不宁静。今晚在院子里坐着乘凉，忽然想起日日走过的荷塘，在这满月的光里，总该另有一番样子吧。月亮渐渐地升高了，墙外马路上孩子们的欢笑，已经听不见了；妻在屋里拍着闰儿，迷迷糊糊地哼着眠歌。我悄悄地披了大衫，带上门出去。"
|
||||
print(markup_language(text, target_languages=None))
|
||||
print(sentence_split(text, max=50))
|
||||
print(sentence_split_and_markup(text, max=50, lang="auto", speaker_lang=None))
|
||||
text = "你好，这是一段用来测试自动标注的文本。こんにちは,これは自動ラベリングのテスト用テキストです.Hello, this is a piece of text to test autotagging."
|
||||
print(split_by_language(text, ["zh", "ja", "en"]))
|
||||
Reference in New Issue
Block a user