tuna2134/sbv2-v2
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Refactor and improve: handling with yomi error, add preprocess options

Browse Source
This commit is contained in:
litagin02
2024-02-27 13:37:24 +09:00
parent dcad61be3a
commit ebea1be519
10 changed files with 101 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app.py
View File

@@ -96,6 +96,8 @@ def tts_fn(
start_time = datetime.datetime.now()
assert model_holder.current_model is not None
try:
sr, audio = model_holder.current_model.infer(
text=text,

3
common/tts_model.py
View File

@@ -136,7 +136,6 @@ class Model:
given_tone: Optional[list[int]] = None,
pitch_scale: float = 1.0,
intonation_scale: float = 1.0,
ignore_unknown: bool = False,
) -> tuple[int, np.ndarray]:
logger.info(f"Start generating audio data from text:\n{text}")
if language != "JP" and self.hps.version.endswith("JP-Extra"):
@@ -174,7 +173,6 @@ class Model:
assist_text_weight=assist_text_weight,
style_vec=style_vector,
given_tone=given_tone,
ignore_unknown=ignore_unknown,
)
else:
texts = text.split("\n")
@@ -197,7 +195,6 @@ class Model:
assist_text=assist_text,
assist_text_weight=assist_text_weight,
style_vec=style_vector,
ignore_unknown=ignore_unknown,
)
)
if i != len(texts) - 1:

7
infer.py
View File

@@ -52,11 +52,11 @@ def get_text(
assist_text=None,
assist_text_weight=0.7,
given_tone=None,
ignore_unknown=False,
):
use_jp_extra = hps.version.endswith("JP-Extra")
# 推論のときにのみ呼び出されるので、raise_yomi_errorはFalseに設定
norm_text, phone, tone, word2ph = clean_text(
text, language_str, use_jp_extra, ignore_unknown=ignore_unknown
text, language_str, use_jp_extra, raise_yomi_error=False
)
if given_tone is not None:
if len(given_tone) != len(phone):
@@ -80,7 +80,6 @@ def get_text(
device,
assist_text,
assist_text_weight,
ignore_unknown,
)
del word2ph
assert bert_ori.shape[-1] == len(phone), phone
@@ -127,7 +126,6 @@ def infer(
assist_text=None,
assist_text_weight=0.7,
given_tone=None,
ignore_unknown=False,
):
is_jp_extra = hps.version.endswith("JP-Extra")
bert, ja_bert, en_bert, phones, tones, lang_ids = get_text(
@@ -138,7 +136,6 @@ def infer(
assist_text=assist_text,
assist_text_weight=assist_text_weight,
given_tone=given_tone,
ignore_unknown=ignore_unknown,
)
if skip_start:
phones = phones[3:]

23
preprocess_text.py
View File

@@ -40,6 +40,7 @@ def count_lines(file_path: str):
@click.option("--clean/--no-clean", default=preprocess_text_config.clean)
@click.option("-y", "--yml_config")
@click.option("--use_jp_extra", is_flag=True)
@click.option("--yomi_error", default="raise")
def preprocess(
transcription_path: str,
cleaned_path: Optional[str],
@@ -51,11 +52,15 @@ def preprocess(
clean: bool,
yml_config: str, # 这个不要删
use_jp_extra: bool,
yomi_error: str,
):
assert yomi_error in ["raise", "skip", "use"]
if cleaned_path == "" or cleaned_path is None:
cleaned_path = transcription_path + ".cleaned"
error_log_path = os.path.join(os.path.dirname(cleaned_path), "text_error.log")
if os.path.exists(error_log_path):
os.remove(error_log_path)
error_count = 0
if clean:
@@ -66,8 +71,12 @@ def preprocess(
try:
utt, spk, language, text = line.strip().split("|")
norm_text, phones, tones, word2ph = clean_text(
text, language, use_jp_extra
text=text,
language=language,
use_jp_extra=use_jp_extra,
raise_yomi_error=(yomi_error != "use"),
)
out_file.write(
"{}|{}|{}|{}|{}|{}|{}\n".format(
utt,
@@ -151,11 +160,19 @@ def preprocess(
with open(config_path, "w", encoding="utf-8") as f:
json.dump(json_config, f, indent=2, ensure_ascii=False)
if error_count > 0:
if yomi_error == "skip":
logger.warning(
f"An error occurred in {error_count} lines. Proceed with lines without errors. Please check {error_log_path} for details."
)
else:
# yom_error == "raise"と"use"の場合。
# "use"の場合は、そもそもyomi_error = Falseで処理しているので、
# ここが実行されるのは他の例外のときなので、エラーをraiseする。
logger.error(
f"An error occurred in {error_count} lines. Please check {error_log_path} for details. You can proceed with lines that do not have errors."
f"An error occurred in {error_count} lines. Please check {error_log_path} for details."
)
raise Exception(
f"An error occurred in {error_count} lines. Please check {error_log_path} for details. You can proceed with lines that do not have errors."
f"An error occurred in {error_count} lines. Please check {error_log_path} for details."
)
else:
logger.info(

4
server_editor.py
View File

@@ -214,7 +214,7 @@ async def read_item(item: TextRequest):
try:
# 最初に正規化しないと整合性がとれない
text = text_normalize(item.text)
kata_tone_list = g2kata_tone(text, ignore_unknown=True)
kata_tone_list = g2kata_tone(text)
except Exception as e:
raise HTTPException(
status_code=400,
@@ -289,7 +289,6 @@ def synthesis(request: SynthesisRequest):
assist_text_weight=request.assistTextWeight,
use_assist_text=bool(request.assistText),
line_split=False,
ignore_unknown=True,
pitch_scale=request.pitchScale,
intonation_scale=request.intonationScale,
)
@@ -348,7 +347,6 @@ def multi_synthesis(request: MultiSynthesisRequest):
assist_text_weight=req.assistTextWeight,
use_assist_text=bool(req.assistText),
line_split=False,
ignore_unknown=True,
pitch_scale=req.pitchScale,
intonation_scale=req.intonationScale,
)

26
text/__init__.py
View File

@@ -18,28 +18,14 @@ def cleaned_text_to_sequence(cleaned_text, tones, language):
return phones, tones, lang_ids
def get_bert(
text,
word2ph,
language,
device,
assist_text=None,
assist_text_weight=0.7,
ignore_unknown=False,
):
def get_bert(text, word2ph, language, device, assist_text=None, assist_text_weight=0.7):
if language == "ZH":
from .chinese_bert import get_bert_feature as zh_bert
return zh_bert(text, word2ph, device, assist_text, assist_text_weight)
from .chinese_bert import get_bert_feature
elif language == "EN":
from .english_bert_mock import get_bert_feature as en_bert
return en_bert(text, word2ph, device, assist_text, assist_text_weight)
from .english_bert_mock import get_bert_feature
elif language == "JP":
from .japanese_bert import get_bert_feature as jp_bert
return jp_bert(
text, word2ph, device, assist_text, assist_text_weight, ignore_unknown
)
from .japanese_bert import get_bert_feature
else:
raise ValueError(f"Language {language} not supported")
return get_bert_feature(text, word2ph, device, assist_text, assist_text_weight)

4
text/cleaner.py
View File

@@ -1,4 +1,4 @@
def clean_text(text, language, use_jp_extra=True, ignore_unknown=False):
def clean_text(text, language, use_jp_extra=True, raise_yomi_error=False):
# Changed to import inside if condition to avoid unnecessary import
if language == "ZH":
from . import chinese as language_module
@@ -15,7 +15,7 @@ def clean_text(text, language, use_jp_extra=True, ignore_unknown=False):
norm_text = language_module.text_normalize(text)
phones, tones, word2ph = language_module.g2p(
norm_text, use_jp_extra, ignore_unknown=ignore_unknown
norm_text, use_jp_extra, raise_yomi_error=raise_yomi_error
)
else:
raise ValueError(f"Language {language} not supported")

38
text/japanese.py
View File

@@ -33,6 +33,16 @@ COSONANTS = set(
VOWELS = {"a", "i", "u", "e", "o", "N"}
class YomiError(Exception):
"""
OpenJTalkで、読みが正しく取得できない箇所があるときに発生する例外。
基本的に「学習の前処理のテキスト処理時」には発生させ、そうでない場合は、
ignore_yomi_error=Trueにしておいて、この例外を発生させないようにする。
"""
pass
# 正規化で記号を変換するための辞書
rep_map = {
"：": ",",
@@ -166,7 +176,7 @@ def japanese_convert_numbers_to_words(text: str) -> str:
def g2p(
norm_text: str, use_jp_extra: bool = True, ignore_unknown: bool = False
norm_text: str, use_jp_extra: bool = True, raise_yomi_error: bool = False
) -> tuple[list[str], list[int], list[int]]:
"""
他で使われるメインの関数。`text_normalize()`で正規化された`norm_text`を受け取り、
@@ -175,7 +185,10 @@ def g2p(
- word2ph: 元のテキストの各文字に音素が何個割り当てられるかを表すリスト
のタプルを返す。
ただし`phones`と`tones`の最初と終わりに`_`が入り、応じて`word2ph`の最初と最後に1が追加される。
use_jp_extra: Falseの場合、「ん」の音素を「N」ではなく「n」とする。
raise_yomi_error: Trueの場合、読めない文字があるときに例外を発生させる。
Falseの場合は読めない文字が消えたような扱いとして処理される。
"""
# pyopenjtalkのフルコンテキストラベルを使ってアクセントを取り出すと、punctuationの位置が消えてしまい情報が失われてしまう：
# 「こんにちは、世界。」と「こんにちは！世界。」と「こんにちは！！！？？？世界……。」は全て同じになる。
@@ -186,9 +199,9 @@ def g2p(
# punctuationがすべて消えた、音素とアクセントのタプルのリスト（「ん」は「N」）
phone_tone_list_wo_punct = g2phone_tone_wo_punct(norm_text)
# sep_text: 単語単位の単語のリスト
# sep_text: 単語単位の単語のリスト、読めない文字があったらraise_yomi_errorなら例外、そうでないなら読めない文字が消えて返ってくる
# sep_kata: 単語単位の単語のカタカナ読みのリスト
sep_text, sep_kata = text2sep_kata(norm_text, ignore_unknown=ignore_unknown)
sep_text, sep_kata = text2sep_kata(norm_text, raise_yomi_error=raise_yomi_error)
# sep_phonemes: 各単語ごとの音素のリストのリスト
sep_phonemes = handle_long([kata2phoneme_list(i) for i in sep_kata])
@@ -237,8 +250,12 @@ def g2p(
return phones, tones, word2ph
def g2kata_tone(norm_text: str, ignore_unknown: bool = False) -> list[tuple[str, int]]:
phones, tones, _ = g2p(norm_text, use_jp_extra=True, ignore_unknown=ignore_unknown)
def g2kata_tone(norm_text: str) -> list[tuple[str, int]]:
"""
テキストからカタカナとアクセントのペアのリストを返す。
推論時のみに使われるので、常に`raise_yomi_error=False`でg2pを呼ぶ。
"""
phones, tones, _ = g2p(norm_text, use_jp_extra=True, raise_yomi_error=False)
return phone_tone2kata_tone(list(zip(phones, tones)))
@@ -332,7 +349,7 @@ def g2phone_tone_wo_punct(text: str) -> list[tuple[str, int]]:
def text2sep_kata(
norm_text: str, ignore_unknown: bool = False
norm_text: str, raise_yomi_error: bool = False
) -> tuple[list[str], list[str]]:
"""
`text_normalize`で正規化済みの`norm_text`を受け取り、それを単語分割し、
@@ -341,6 +358,9 @@ def text2sep_kata(
例:
`私はそう思う!って感じ?` →
["私", "は", "そう", "思う", "!", "って", "感じ", "?"], ["ワタシ", "ワ", "ソー", "オモウ", "!", "ッテ", "カンジ", "?"]
raise_yomi_error: Trueの場合、読めない文字があるときに例外を発生させる。
Falseの場合は読めない文字が消えたような扱いとして処理される。
"""
# parsed: OpenJTalkの解析結果
parsed = pyopenjtalk.run_frontend(norm_text)
@@ -369,10 +389,10 @@ def text2sep_kata(
# wordは正規化されているので、`.`, `,`, `!`, `'`, `-`, `--` のいずれか
if not set(word).issubset(set(punctuation)): # 記号繰り返しか判定
# ここはpyopenjtalkが読めない文字等のときに起こる
if ignore_unknown:
logger.error(f"Ignoring unknown: {word} in:\n{norm_text}")
if raise_yomi_error:
raise YomiError(f"Cannot read: {word} in:\n{norm_text}")
logger.warning(f"Ignoring unknown: {word} in:\n{norm_text}")
continue
raise ValueError(f"Cannot read: {word} in:\n{norm_text}")
# yomiは元の記号のままに変更
yomi = word
elif yomi == "？":

9
text/japanese_bert.py
View File

@@ -19,12 +19,13 @@ def get_bert_feature(
device=config.bert_gen_config.device,
assist_text=None,
assist_text_weight=0.7,
ignore_unknown=False,
):
text = "".join(text2sep_kata(text, ignore_unknown=ignore_unknown)[0])
# text = text_normalize(text)
# 各単語が何文字かを作る`word2ph`を使う必要があるので、読めない文字は必ず無視する
# でないと`word2ph`の結果とテキストの文字数結果が整合性が取れない
text = "".join(text2sep_kata(text, raise_yomi_error=False)[0])
if assist_text:
assist_text = "".join(text2sep_kata(assist_text)[0])
assist_text = "".join(text2sep_kata(assist_text, raise_yomi_error=False)[0])
if (
sys.platform == "darwin"
and torch.backends.mps.is_available()

31
webui_train.py
View File

@@ -155,7 +155,7 @@ def resample(model_name, normalize, trim, num_processes):
return True, "Step 2, Success: 音声ファイルの前処理が完了しました"
def preprocess_text(model_name, use_jp_extra, val_per_lang):
def preprocess_text(model_name, use_jp_extra, val_per_lang, yomi_error):
logger.info("Step 3: start preprocessing text...")
dataset_path, lbl_path, train_path, val_path, config_path = get_path(model_name)
try:
@@ -189,6 +189,8 @@ def preprocess_text(model_name, use_jp_extra, val_per_lang):
val_path,
"--val-per-lang",
str(val_per_lang),
"--yomi_error",
yomi_error,
]
if use_jp_extra:
cmd.append("--use_jp_extra")
@@ -278,6 +280,7 @@ def preprocess_all(
use_jp_extra,
val_per_lang,
log_interval,
yomi_error,
):
if model_name == "":
return False, "Error: モデル名を入力してください"
@@ -304,8 +307,12 @@ def preprocess_all(
)
if not success:
return False, message
success, message = preprocess_text(
model_name=model_name, use_jp_extra=use_jp_extra, val_per_lang=val_per_lang
model_name=model_name,
use_jp_extra=use_jp_extra,
val_per_lang=val_per_lang,
yomi_error=yomi_error,
)
if not success:
return False, message
@@ -488,6 +495,15 @@ if __name__ == "__main__":
label="音声の最初と最後の無音を取り除く",
value=False,
)
yomi_error = gr.Radio(
label="読みエラーの扱い",
choices=[
("エラー出たらテキスト前処理が終わった時点で中断", "raise"),
("エラーファイルは使わず続行", "skip"),
("読みを強引に埋めて使い続行", "use"),
],
value="raise",
)
with gr.Accordion("詳細設定", open=False):
num_processes = gr.Slider(
label="プロセス数",
@@ -630,6 +646,15 @@ if __name__ == "__main__":
maximum=100,
step=1,
)
yomi_error_manual = gr.Radio(
label="読みエラーの扱い",
choices=[
("エラー出たらテキスト前処理が終わった時点で中断", "raise"),
("エラーファイルは使わず続行", "skip"),
("読みを強引に埋めて使い続行", "use"),
],
value="raise",
)
with gr.Column():
preprocess_text_btn = gr.Button(value="実行", variant="primary")
info_preprocess_text = gr.Textbox(label="状況")
@@ -693,6 +718,7 @@ if __name__ == "__main__":
use_jp_extra,
val_per_lang,
log_interval,
yomi_error,
],
outputs=[info_all],
)
@@ -731,6 +757,7 @@ if __name__ == "__main__":
model_name,
use_jp_extra_manual,
val_per_lang_manual,
yomi_error_manual,
],
outputs=[info_preprocess_text],
)