Dev 2.3. (#242)
* Fix inputs of duration discriminator * Add LSTM * Update models.py * Update tensorboard scalar * Noise injection for minimizing modality gap * Update infer.py * support bf16 run * del unused_para flag * support bf16 config * add grad clip * fix(logger and grad):add dur grad,fix grad clip * Update webui_preprocess.py * Fix English G2P * fix(bert_gen):add pass * Pass SDP to DD * Update webui_preprocess.py * Update config.json * Update webui.py * Update chinese_bert.py * Upload webui for deploy * Update webui.py * torch.save as pt not npy * Update config.json * add freeze emo vq * Update webui_preprocess.py * Fix tone_sandhi.py * Comment up grad clip * Fix in-place addition * Add SLM discriminator * Add DDP for WD * Feat: Style text: make emotions and style similar to the style text by mixing bert (#240) (#241) * fix:(oldVersion210) Load on demand Emotion model * feat: update fastapi.py. 添加更多错误日志信息 * Switch pyopenjtalk to pyopenjtalk-prebuilt * fix: update fastapi.py. 2.2 reference适配 * Update resample.py * 修复Onnx导出的BUG (#237) * Add files via upload * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add files via upload * Add files via upload * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Delete attentions_onnx.py * Delete models_onnx.py * Add files via upload * Add files via upload * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update __init__.py * Update __init__.py * Update __init__.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- * Fix onnx * Format export * Feat: style-text and bert mixing (JA only) * Ensure the same tensor shape * Update * update gradio version * Fix * Style text for chinese and english (ver 2.2) * Style text for chinese and english (ver 2.1) * Style text in FastAPI * Translate style text desc in chinese --------- Co-authored-by: litagin02 <139731664+litagin02@users.noreply.github.com> Co-authored-by: Sora <654163754@qq.com> Co-authored-by: Sihan Wang <wangsihan1995@gmail.com> Co-authored-by: Ναρουσέ·μ·γιουμεμί·Χινακάννα <40709280+NaruseMioShirakana@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Remove CLAP * Revert "Remove CLAP" This reverts commit 62fd59bc837c580239840a2bc84b15e0663730fc. Revert * Remove CLAP * bf16 audo grad cilp * Update webui and infer utils * Update webui.py * Update webui.py * Update webui-preprocess.py * Update webui_preprocess.py --------- Co-authored-by: Sihan Wang <wangsihan1995@gmail.com> Co-authored-by: OedoSoldier <31711261+OedoSoldier@users.noreply.github.com> Co-authored-by: litagin02 <139731664+litagin02@users.noreply.github.com> Co-authored-by: Sora <654163754@qq.com> Co-authored-by: Ναρουσέ·μ·γιουμεμί·Χινακάννα <40709280+NaruseMioShirakana@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Stardust·减
402
webui.py
402
webui.py
@@ -42,6 +42,8 @@ def generate_audio(
|
||||
language,
|
||||
reference_audio,
|
||||
emotion,
|
||||
style_text,
|
||||
style_weight,
|
||||
skip_start=False,
|
||||
skip_end=False,
|
||||
):
|
||||
@@ -49,8 +51,8 @@ def generate_audio(
|
||||
# silence = np.zeros(hps.data.sampling_rate // 2, dtype=np.int16)
|
||||
with torch.no_grad():
|
||||
for idx, piece in enumerate(slices):
|
||||
skip_start = (idx != 0) and skip_start
|
||||
skip_end = (idx != len(slices) - 1) and skip_end
|
||||
skip_start = idx != 0
|
||||
skip_end = idx != len(slices) - 1
|
||||
audio = infer(
|
||||
piece,
|
||||
reference_audio=reference_audio,
|
||||
@@ -66,10 +68,11 @@ def generate_audio(
|
||||
device=device,
|
||||
skip_start=skip_start,
|
||||
skip_end=skip_end,
|
||||
style_text=style_text,
|
||||
style_weight=style_weight,
|
||||
)
|
||||
audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
|
||||
audio_list.append(audio16bit)
|
||||
# audio_list.append(silence) # 将静音添加到列表中
|
||||
return audio_list
|
||||
|
||||
|
||||
@@ -90,8 +93,8 @@ def generate_audio_multilang(
|
||||
# silence = np.zeros(hps.data.sampling_rate // 2, dtype=np.int16)
|
||||
with torch.no_grad():
|
||||
for idx, piece in enumerate(slices):
|
||||
skip_start = (idx != 0) and skip_start
|
||||
skip_end = (idx != len(slices) - 1) and skip_end
|
||||
skip_start = idx != 0
|
||||
skip_end = idx != len(slices) - 1
|
||||
audio = infer_multilang(
|
||||
piece,
|
||||
reference_audio=reference_audio,
|
||||
@@ -110,7 +113,6 @@ def generate_audio_multilang(
|
||||
)
|
||||
audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
|
||||
audio_list.append(audio16bit)
|
||||
# audio_list.append(silence) # 将静音添加到列表中
|
||||
return audio_list
|
||||
|
||||
|
||||
@@ -127,63 +129,50 @@ def tts_split(
|
||||
interval_between_sent,
|
||||
reference_audio,
|
||||
emotion,
|
||||
style_text,
|
||||
style_weight,
|
||||
):
|
||||
if language == "mix":
|
||||
return ("invalid", None)
|
||||
while text.find("\n\n") != -1:
|
||||
text = text.replace("\n\n", "\n")
|
||||
text = text.replace("|", "")
|
||||
para_list = re_matching.cut_para(text)
|
||||
para_list = [p for p in para_list if p != ""]
|
||||
audio_list = []
|
||||
if not cut_by_sent:
|
||||
for idx, p in enumerate(para_list):
|
||||
skip_start = idx != 0
|
||||
skip_end = idx != len(para_list) - 1
|
||||
audio = infer(
|
||||
for p in para_list:
|
||||
if not cut_by_sent:
|
||||
audio_list += process_text(
|
||||
p,
|
||||
reference_audio=reference_audio,
|
||||
emotion=emotion,
|
||||
sdp_ratio=sdp_ratio,
|
||||
noise_scale=noise_scale,
|
||||
noise_scale_w=noise_scale_w,
|
||||
length_scale=length_scale,
|
||||
sid=speaker,
|
||||
language=language,
|
||||
hps=hps,
|
||||
net_g=net_g,
|
||||
device=device,
|
||||
skip_start=skip_start,
|
||||
skip_end=skip_end,
|
||||
speaker,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
language,
|
||||
reference_audio,
|
||||
emotion,
|
||||
style_text,
|
||||
style_weight,
|
||||
)
|
||||
audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
|
||||
audio_list.append(audio16bit)
|
||||
silence = np.zeros((int)(44100 * interval_between_para), dtype=np.int16)
|
||||
audio_list.append(silence)
|
||||
else:
|
||||
for idx, p in enumerate(para_list):
|
||||
skip_start = idx != 0
|
||||
skip_end = idx != len(para_list) - 1
|
||||
else:
|
||||
audio_list_sent = []
|
||||
sent_list = re_matching.cut_sent(p)
|
||||
for idx, s in enumerate(sent_list):
|
||||
skip_start = (idx != 0) and skip_start
|
||||
skip_end = (idx != len(sent_list) - 1) and skip_end
|
||||
audio = infer(
|
||||
sent_list = [s for s in sent_list if s != ""]
|
||||
for s in sent_list:
|
||||
audio_list_sent += process_text(
|
||||
s,
|
||||
reference_audio=reference_audio,
|
||||
emotion=emotion,
|
||||
sdp_ratio=sdp_ratio,
|
||||
noise_scale=noise_scale,
|
||||
noise_scale_w=noise_scale_w,
|
||||
length_scale=length_scale,
|
||||
sid=speaker,
|
||||
language=language,
|
||||
hps=hps,
|
||||
net_g=net_g,
|
||||
device=device,
|
||||
skip_start=skip_start,
|
||||
skip_end=skip_end,
|
||||
speaker,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
language,
|
||||
reference_audio,
|
||||
emotion,
|
||||
style_text,
|
||||
style_weight,
|
||||
)
|
||||
audio_list_sent.append(audio)
|
||||
silence = np.zeros((int)(44100 * interval_between_sent))
|
||||
audio_list_sent.append(silence)
|
||||
if (interval_between_para - interval_between_sent) > 0:
|
||||
@@ -196,7 +185,118 @@ def tts_split(
|
||||
) # 对完整句子做音量归一
|
||||
audio_list.append(audio16bit)
|
||||
audio_concat = np.concatenate(audio_list)
|
||||
return ("Success", (44100, audio_concat))
|
||||
return ("Success", (hps.data.sampling_rate, audio_concat))
|
||||
|
||||
|
||||
def process_mix(slice):
|
||||
_speaker = slice.pop()
|
||||
_text, _lang = [], []
|
||||
for lang, content in slice:
|
||||
content = content.split("|")
|
||||
content = [part for part in content if part != ""]
|
||||
if len(content) == 0:
|
||||
continue
|
||||
if len(_text) == 0:
|
||||
_text = [[part] for part in content]
|
||||
_lang = [[lang] for part in content]
|
||||
else:
|
||||
_text[-1].append(content[0])
|
||||
_lang[-1].append(lang)
|
||||
if len(content) > 1:
|
||||
_text += [[part] for part in content[1:]]
|
||||
_lang += [[lang] for part in content[1:]]
|
||||
return _text, _lang, _speaker
|
||||
|
||||
|
||||
def process_auto(text):
|
||||
_text, _lang = [], []
|
||||
for slice in text.split("|"):
|
||||
if slice == "":
|
||||
continue
|
||||
temp_text, temp_lang = [], []
|
||||
sentences_list = split_by_language(slice, target_languages=["zh", "ja", "en"])
|
||||
for sentence, lang in sentences_list:
|
||||
if sentence == "":
|
||||
continue
|
||||
temp_text.append(sentence)
|
||||
temp_lang.append(lang.upper())
|
||||
_text.append(temp_text)
|
||||
_lang.append(temp_lang)
|
||||
return _text, _lang
|
||||
|
||||
|
||||
def process_text(
|
||||
text: str,
|
||||
speaker,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
language,
|
||||
reference_audio,
|
||||
emotion,
|
||||
style_text=None,
|
||||
style_weight=0,
|
||||
):
|
||||
audio_list = []
|
||||
if language == "mix":
|
||||
bool_valid, str_valid = re_matching.validate_text(text)
|
||||
if not bool_valid:
|
||||
return str_valid, (
|
||||
hps.data.sampling_rate,
|
||||
np.concatenate([np.zeros(hps.data.sampling_rate // 2)]),
|
||||
)
|
||||
for slice in re_matching.text_matching(text):
|
||||
_text, _lang, _speaker = process_mix(slice)
|
||||
if _speaker is None:
|
||||
continue
|
||||
print(f"Text: {_text}\nLang: {_lang}")
|
||||
audio_list.extend(
|
||||
generate_audio_multilang(
|
||||
_text,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
_speaker,
|
||||
_lang,
|
||||
reference_audio,
|
||||
emotion,
|
||||
)
|
||||
)
|
||||
elif language.lower() == "auto":
|
||||
_text, _lang = process_auto(text)
|
||||
print(f"Text: {_text}\nLang: {_lang}")
|
||||
audio_list.extend(
|
||||
generate_audio_multilang(
|
||||
_text,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
speaker,
|
||||
_lang,
|
||||
reference_audio,
|
||||
emotion,
|
||||
)
|
||||
)
|
||||
else:
|
||||
audio_list.extend(
|
||||
generate_audio(
|
||||
text.split("|"),
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
speaker,
|
||||
language,
|
||||
reference_audio,
|
||||
emotion,
|
||||
style_text,
|
||||
style_weight,
|
||||
)
|
||||
)
|
||||
return audio_list
|
||||
|
||||
|
||||
def tts_fn(
|
||||
@@ -210,7 +310,11 @@ def tts_fn(
|
||||
reference_audio,
|
||||
emotion,
|
||||
prompt_mode,
|
||||
style_text=None,
|
||||
style_weight=0,
|
||||
):
|
||||
if style_text == "":
|
||||
style_text = None
|
||||
if prompt_mode == "Audio prompt":
|
||||
if reference_audio == None:
|
||||
return ("Invalid audio prompt", None)
|
||||
@@ -218,147 +322,35 @@ def tts_fn(
|
||||
reference_audio = load_audio(reference_audio)[1]
|
||||
else:
|
||||
reference_audio = None
|
||||
audio_list = []
|
||||
if language == "mix":
|
||||
bool_valid, str_valid = re_matching.validate_text(text)
|
||||
if not bool_valid:
|
||||
return str_valid, (
|
||||
hps.data.sampling_rate,
|
||||
np.concatenate([np.zeros(hps.data.sampling_rate // 2)]),
|
||||
)
|
||||
result = []
|
||||
for slice in re_matching.text_matching(text):
|
||||
_speaker = slice.pop()
|
||||
temp_contant = []
|
||||
temp_lang = []
|
||||
for lang, content in slice:
|
||||
if "|" in content:
|
||||
temp = []
|
||||
temp_ = []
|
||||
for i in content.split("|"):
|
||||
if i != "":
|
||||
temp.append([i])
|
||||
temp_.append([lang])
|
||||
else:
|
||||
temp.append([])
|
||||
temp_.append([])
|
||||
temp_contant += temp
|
||||
temp_lang += temp_
|
||||
else:
|
||||
if len(temp_contant) == 0:
|
||||
temp_contant.append([])
|
||||
temp_lang.append([])
|
||||
temp_contant[-1].append(content)
|
||||
temp_lang[-1].append(lang)
|
||||
for i, j in zip(temp_lang, temp_contant):
|
||||
result.append([*zip(i, j), _speaker])
|
||||
for i, one in enumerate(result):
|
||||
skip_start = i != 0
|
||||
skip_end = i != len(result) - 1
|
||||
_speaker = one.pop()
|
||||
idx = 0
|
||||
while idx < len(one):
|
||||
text_to_generate = []
|
||||
lang_to_generate = []
|
||||
while True:
|
||||
lang, content = one[idx]
|
||||
temp_text = [content]
|
||||
if len(text_to_generate) > 0:
|
||||
text_to_generate[-1] += [temp_text.pop(0)]
|
||||
lang_to_generate[-1] += [lang]
|
||||
if len(temp_text) > 0:
|
||||
text_to_generate += [[i] for i in temp_text]
|
||||
lang_to_generate += [[lang]] * len(temp_text)
|
||||
if idx + 1 < len(one):
|
||||
idx += 1
|
||||
else:
|
||||
break
|
||||
skip_start = (idx != 0) and skip_start
|
||||
skip_end = (idx != len(one) - 1) and skip_end
|
||||
print(text_to_generate, lang_to_generate)
|
||||
audio_list.extend(
|
||||
generate_audio_multilang(
|
||||
text_to_generate,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
_speaker,
|
||||
lang_to_generate,
|
||||
reference_audio,
|
||||
emotion,
|
||||
skip_start,
|
||||
skip_end,
|
||||
)
|
||||
)
|
||||
idx += 1
|
||||
elif language.lower() == "auto":
|
||||
for idx, slice in enumerate(text.split("|")):
|
||||
if slice == "":
|
||||
continue
|
||||
skip_start = idx != 0
|
||||
skip_end = idx != len(text.split("|")) - 1
|
||||
sentences_list = split_by_language(
|
||||
slice, target_languages=["zh", "ja", "en"]
|
||||
)
|
||||
idx = 0
|
||||
while idx < len(sentences_list):
|
||||
text_to_generate = []
|
||||
lang_to_generate = []
|
||||
while True:
|
||||
content, lang = sentences_list[idx]
|
||||
temp_text = [content]
|
||||
lang = lang.upper()
|
||||
if lang == "JA":
|
||||
lang = "JP"
|
||||
if len(text_to_generate) > 0:
|
||||
text_to_generate[-1] += [temp_text.pop(0)]
|
||||
lang_to_generate[-1] += [lang]
|
||||
if len(temp_text) > 0:
|
||||
text_to_generate += [[i] for i in temp_text]
|
||||
lang_to_generate += [[lang]] * len(temp_text)
|
||||
if idx + 1 < len(sentences_list):
|
||||
idx += 1
|
||||
else:
|
||||
break
|
||||
skip_start = (idx != 0) and skip_start
|
||||
skip_end = (idx != len(sentences_list) - 1) and skip_end
|
||||
print(text_to_generate, lang_to_generate)
|
||||
audio_list.extend(
|
||||
generate_audio_multilang(
|
||||
text_to_generate,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
speaker,
|
||||
lang_to_generate,
|
||||
reference_audio,
|
||||
emotion,
|
||||
skip_start,
|
||||
skip_end,
|
||||
)
|
||||
)
|
||||
idx += 1
|
||||
else:
|
||||
audio_list.extend(
|
||||
generate_audio(
|
||||
text.split("|"),
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
speaker,
|
||||
language,
|
||||
reference_audio,
|
||||
emotion,
|
||||
)
|
||||
)
|
||||
|
||||
audio_list = process_text(
|
||||
text,
|
||||
speaker,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
language,
|
||||
reference_audio,
|
||||
emotion,
|
||||
style_text,
|
||||
style_weight,
|
||||
)
|
||||
|
||||
audio_concat = np.concatenate(audio_list)
|
||||
return "Success", (hps.data.sampling_rate, audio_concat)
|
||||
|
||||
|
||||
def format_utils(text, speaker):
|
||||
_text, _lang = process_auto(text)
|
||||
res = f"[{speaker}]"
|
||||
for lang_s, content_s in zip(_lang, _text):
|
||||
for lang, content in zip(lang_s, content_s):
|
||||
res += f"<{lang.lower()}>{content}"
|
||||
res += "|"
|
||||
return "mix", res[:-1]
|
||||
|
||||
|
||||
def load_audio(path):
|
||||
audio, sr = librosa.load(path, 48000)
|
||||
# audio = librosa.resample(audio, 44100, 48000)
|
||||
@@ -408,34 +400,37 @@ if __name__ == "__main__":
|
||||
)
|
||||
trans = gr.Button("中翻日", variant="primary")
|
||||
slicer = gr.Button("快速切分", variant="primary")
|
||||
formatter = gr.Button("检测语言，并整理为 MIX 格式", variant="primary")
|
||||
speaker = gr.Dropdown(
|
||||
choices=speakers, value=speakers[0], label="Speaker"
|
||||
)
|
||||
_ = gr.Markdown(
|
||||
value="提示模式（Prompt mode）：可选文字提示或音频提示，用于生成文字或音频指定风格的声音。\n"
|
||||
value="提示模式（Prompt mode）：可选文字提示或音频提示，用于生成文字或音频指定风格的声音。\n",
|
||||
visible=False,
|
||||
)
|
||||
prompt_mode = gr.Radio(
|
||||
["Text prompt", "Audio prompt"],
|
||||
label="Prompt Mode",
|
||||
value="Text prompt",
|
||||
visible=False,
|
||||
)
|
||||
text_prompt = gr.Textbox(
|
||||
label="Text prompt",
|
||||
placeholder="用文字描述生成风格。如：Happy",
|
||||
value="Happy",
|
||||
visible=True,
|
||||
visible=False,
|
||||
)
|
||||
audio_prompt = gr.Audio(
|
||||
label="Audio prompt", type="filepath", visible=False
|
||||
)
|
||||
sdp_ratio = gr.Slider(
|
||||
minimum=0, maximum=1, value=0.2, step=0.1, label="SDP Ratio"
|
||||
minimum=0, maximum=1, value=0.5, step=0.1, label="SDP Ratio"
|
||||
)
|
||||
noise_scale = gr.Slider(
|
||||
minimum=0.1, maximum=2, value=0.6, step=0.1, label="Noise"
|
||||
)
|
||||
noise_scale_w = gr.Slider(
|
||||
minimum=0.1, maximum=2, value=0.8, step=0.1, label="Noise_W"
|
||||
minimum=0.1, maximum=2, value=0.9, step=0.1, label="Noise_W"
|
||||
)
|
||||
length_scale = gr.Slider(
|
||||
minimum=0.1, maximum=2, value=1.0, step=0.1, label="Length"
|
||||
@@ -445,6 +440,21 @@ if __name__ == "__main__":
|
||||
)
|
||||
btn = gr.Button("生成音频！", variant="primary")
|
||||
with gr.Column():
|
||||
with gr.Accordion("融合文本语义", open=False):
|
||||
gr.Markdown(
|
||||
value="使用辅助文本的语意来辅助生成对话（语言保持与主文本相同）\n\n"
|
||||
"**注意**：不要使用**指令式文本**（如：开心），要使用**带有强烈情感的文本**（如：我好快乐！！！）\n\n"
|
||||
"效果较不明确，留空即为不使用该功能"
|
||||
)
|
||||
style_text = gr.Textbox(label="辅助文本")
|
||||
style_weight = gr.Slider(
|
||||
minimum=0,
|
||||
maximum=1,
|
||||
value=0.7,
|
||||
step=0.1,
|
||||
label="Weight",
|
||||
info="主文本和辅助文本的bert混合比率，0表示仅主文本，1表示仅辅助文本",
|
||||
)
|
||||
with gr.Row():
|
||||
with gr.Column():
|
||||
interval_between_sent = gr.Slider(
|
||||
@@ -487,6 +497,8 @@ if __name__ == "__main__":
|
||||
audio_prompt,
|
||||
text_prompt,
|
||||
prompt_mode,
|
||||
style_text,
|
||||
style_weight,
|
||||
],
|
||||
outputs=[text_output, audio_output],
|
||||
)
|
||||
@@ -511,6 +523,8 @@ if __name__ == "__main__":
|
||||
interval_between_sent,
|
||||
audio_prompt,
|
||||
text_prompt,
|
||||
style_text,
|
||||
style_weight,
|
||||
],
|
||||
outputs=[text_output, audio_output],
|
||||
)
|
||||
@@ -527,6 +541,12 @@ if __name__ == "__main__":
|
||||
outputs=[audio_prompt],
|
||||
)
|
||||
|
||||
formatter.click(
|
||||
format_utils,
|
||||
inputs=[text, speaker],
|
||||
outputs=[language, text],
|
||||
)
|
||||
|
||||
print("推理页面已开启!")
|
||||
webbrowser.open(f"http://127.0.0.1:{config.webui_config.port}")
|
||||
app.launch(share=config.webui_config.share, server_port=config.webui_config.port)
|
||||
|
||||
Reference in New Issue
Block a user