tuna2134/sbv2-v2
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Dev 2.3. (#242)

Browse Source 
* Fix inputs of duration discriminator

* Add LSTM

* Update models.py

* Update tensorboard scalar

* Noise injection for minimizing modality gap

* Update infer.py

* support bf16 run

* del unused_para flag

* support bf16 config

* add grad clip

* fix(logger and grad):add dur grad,fix grad clip

* Update webui_preprocess.py

* Fix English G2P

* fix(bert_gen):add pass

* Pass SDP to DD

* Update webui_preprocess.py

* Update config.json

* Update webui.py

* Update chinese_bert.py

* Upload webui for deploy

* Update webui.py

* torch.save as pt not npy

* Update config.json

* add freeze emo vq

* Update webui_preprocess.py

* Fix tone_sandhi.py

* Comment up grad clip

* Fix in-place addition

* Add SLM discriminator

* Add DDP for WD

* Feat: Style text: make emotions and style similar to the style text by mixing bert (#240) (#241)

* fix:(oldVersion210) Load on demand Emotion model

* feat: update fastapi.py. 添加更多错误日志信息

* Switch pyopenjtalk to pyopenjtalk-prebuilt

* fix: update fastapi.py. 2.2 reference适配

* Update resample.py

* 修复Onnx导出的BUG (#237)

* Add files via upload

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Add files via upload

* Add files via upload

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Delete attentions_onnx.py

* Delete models_onnx.py

* Add files via upload

* Add files via upload

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Update __init__.py

* Update __init__.py

* Update __init__.py

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------



* Fix onnx

* Format export

* Feat: style-text and bert mixing (JA only)

* Ensure the same tensor shape

* Update

* update gradio version

* Fix

* Style text for chinese and english (ver 2.2)

* Style text for chinese and english (ver 2.1)

* Style text in FastAPI

* Translate style text desc in chinese

---------

Co-authored-by: litagin02 <139731664+litagin02@users.noreply.github.com>
Co-authored-by: Sora <654163754@qq.com>
Co-authored-by: Sihan Wang <wangsihan1995@gmail.com>
Co-authored-by: Ναρουσέ·μ·γιουμεμί·Χινακάννα <40709280+NaruseMioShirakana@users.noreply.github.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

* Remove CLAP

* Revert "Remove CLAP"

This reverts commit 62fd59bc837c580239840a2bc84b15e0663730fc.

Revert

* Remove CLAP

* bf16 audo grad cilp

* Update webui and infer utils

* Update webui.py

* Update webui.py

* Update webui-preprocess.py

* Update webui_preprocess.py

---------

Co-authored-by: Sihan Wang <wangsihan1995@gmail.com>
Co-authored-by: OedoSoldier <31711261+OedoSoldier@users.noreply.github.com>
Co-authored-by: litagin02 <139731664+litagin02@users.noreply.github.com>
Co-authored-by: Sora <654163754@qq.com>
Co-authored-by: Ναρουσέ·μ·γιουμεμί·Χινακάννα <40709280+NaruseMioShirakana@users.noreply.github.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Stardust·减
2023-12-19 19:12:26 +08:00
committed by GitHub
parent 5479e9039d
commit 76653b5b6d
34 changed files with 2978 additions and 1226 deletions
Download Patch File Download Diff File Expand all files Collapse all files

386
for_deploy/infer.py Normal file
View File

@@ -0,0 +1,386 @@
"""
版本管理、兼容推理及模型加载实现。
版本说明：
1. 版本号与github的release版本号对应，使用哪个release版本训练的模型即对应其版本号
2. 请在模型的config.json中显示声明版本号，添加一个字段"version" : "你的版本号"
特殊版本说明：
1.1.1-fix： 1.1.1版本训练的模型，但是在推理时使用dev的日语修复
2.2：当前版本
"""
import torch
import commons
from text import cleaned_text_to_sequence, get_bert
from clap_wrapper import get_clap_audio_feature, get_clap_text_feature
from text.cleaner import clean_text
import utils
import numpy as np
from models import SynthesizerTrn
from text.symbols import symbols
from oldVersion.V210.models import SynthesizerTrn as V210SynthesizerTrn
from oldVersion.V210.text import symbols as V210symbols
from oldVersion.V200.models import SynthesizerTrn as V200SynthesizerTrn
from oldVersion.V200.text import symbols as V200symbols
from oldVersion.V111.models import SynthesizerTrn as V111SynthesizerTrn
from oldVersion.V111.text import symbols as V111symbols
from oldVersion.V110.models import SynthesizerTrn as V110SynthesizerTrn
from oldVersion.V110.text import symbols as V110symbols
from oldVersion.V101.models import SynthesizerTrn as V101SynthesizerTrn
from oldVersion.V101.text import symbols as V101symbols
from oldVersion import V111, V110, V101, V200, V210
# 当前版本信息
latest_version = "2.2"
# 版本兼容
SynthesizerTrnMap = {
"2.1": V210SynthesizerTrn,
"2.0.2-fix": V200SynthesizerTrn,
"2.0.1": V200SynthesizerTrn,
"2.0": V200SynthesizerTrn,
"1.1.1-fix": V111SynthesizerTrn,
"1.1.1": V111SynthesizerTrn,
"1.1": V110SynthesizerTrn,
"1.1.0": V110SynthesizerTrn,
"1.0.1": V101SynthesizerTrn,
"1.0": V101SynthesizerTrn,
"1.0.0": V101SynthesizerTrn,
}
symbolsMap = {
"2.1": V210symbols,
"2.0.2-fix": V200symbols,
"2.0.1": V200symbols,
"2.0": V200symbols,
"1.1.1-fix": V111symbols,
"1.1.1": V111symbols,
"1.1": V110symbols,
"1.1.0": V110symbols,
"1.0.1": V101symbols,
"1.0": V101symbols,
"1.0.0": V101symbols,
}
# def get_emo_(reference_audio, emotion, sid):
# emo = (
# torch.from_numpy(get_emo(reference_audio))
# if reference_audio and emotion == -1
# else torch.FloatTensor(
# np.load(f"emo_clustering/{sid}/cluster_center_{emotion}.npy")
# )
# )
# return emo
def get_net_g(model_path: str, version: str, device: str, hps):
if version != latest_version:
net_g = SynthesizerTrnMap[version](
len(symbolsMap[version]),
hps.data.filter_length // 2 + 1,
hps.train.segment_size // hps.data.hop_length,
n_speakers=hps.data.n_speakers,
**hps.model,
).to(device)
else:
# 当前版本模型 net_g
net_g = SynthesizerTrn(
len(symbols),
hps.data.filter_length // 2 + 1,
hps.train.segment_size // hps.data.hop_length,
n_speakers=hps.data.n_speakers,
**hps.model,
).to(device)
_ = net_g.eval()
_ = utils.load_checkpoint(model_path, net_g, None, skip_optimizer=True)
return net_g
def get_text(text, language_str, bert, hps, device):
# 在此处实现当前版本的get_text
norm_text, phone, tone, word2ph = clean_text(text, language_str)
phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
if hps.data.add_blank:
phone = commons.intersperse(phone, 0)
tone = commons.intersperse(tone, 0)
language = commons.intersperse(language, 0)
for i in range(len(word2ph)):
word2ph[i] = word2ph[i] * 2
word2ph[0] += 1
# bert_ori = get_bert(norm_text, word2ph, language_str, device)
bert_ori = bert[language_str].get_bert_feature(norm_text, word2ph, device)
del word2ph
assert bert_ori.shape[-1] == len(phone), phone
if language_str == "ZH":
bert = bert_ori
ja_bert = torch.randn(1024, len(phone))
en_bert = torch.randn(1024, len(phone))
elif language_str == "JP":
bert = torch.randn(1024, len(phone))
ja_bert = bert_ori
en_bert = torch.randn(1024, len(phone))
elif language_str == "EN":
bert = torch.randn(1024, len(phone))
ja_bert = torch.randn(1024, len(phone))
en_bert = bert_ori
else:
raise ValueError("language_str should be ZH, JP or EN")
assert bert.shape[-1] == len(
phone
), f"Bert seq len {bert.shape[-1]} != {len(phone)}"
phone = torch.LongTensor(phone)
tone = torch.LongTensor(tone)
language = torch.LongTensor(language)
return bert, ja_bert, en_bert, phone, tone, language
def infer(
text,
emotion,
sdp_ratio,
noise_scale,
noise_scale_w,
length_scale,
sid,
language,
hps,
net_g,
device,
bert=None,
clap=None,
reference_audio=None,
skip_start=False,
skip_end=False,
):
# 2.2版本参数位置变了
# 2.1 参数新增 emotion reference_audio skip_start skip_end
inferMap_V3 = {
"2.1": V210.infer,
}
# 支持中日英三语版本
inferMap_V2 = {
"2.0.2-fix": V200.infer,
"2.0.1": V200.infer,
"2.0": V200.infer,
"1.1.1-fix": V111.infer_fix,
"1.1.1": V111.infer,
"1.1": V110.infer,
"1.1.0": V110.infer,
}
# 仅支持中文版本
# 在测试中，并未发现两个版本的模型不能互相通用
inferMap_V1 = {
"1.0.1": V101.infer,
"1.0": V101.infer,
"1.0.0": V101.infer,
}
version = hps.version if hasattr(hps, "version") else latest_version
# 非当前版本，根据版本号选择合适的infer
if version != latest_version:
if version in inferMap_V3.keys():
return inferMap_V3[version](
text,
sdp_ratio,
noise_scale,
noise_scale_w,
length_scale,
sid,
language,
hps,
net_g,
device,
reference_audio,
emotion,
skip_start,
skip_end,
)
if version in inferMap_V2.keys():
return inferMap_V2[version](
text,
sdp_ratio,
noise_scale,
noise_scale_w,
length_scale,
sid,
language,
hps,
net_g,
device,
)
if version in inferMap_V1.keys():
return inferMap_V1[version](
text,
sdp_ratio,
noise_scale,
noise_scale_w,
length_scale,
sid,
hps,
net_g,
device,
)
# 在此处实现当前版本的推理
# emo = get_emo_(reference_audio, emotion, sid)
if isinstance(reference_audio, np.ndarray):
emo = clap.get_clap_audio_feature(reference_audio, device)
else:
emo = clap.get_clap_text_feature(emotion, device)
emo = torch.squeeze(emo, dim=1)
bert, ja_bert, en_bert, phones, tones, lang_ids = get_text(
text, language, bert, hps, device
)
if skip_start:
phones = phones[3:]
tones = tones[3:]
lang_ids = lang_ids[3:]
bert = bert[:, 3:]
ja_bert = ja_bert[:, 3:]
en_bert = en_bert[:, 3:]
if skip_end:
phones = phones[:-2]
tones = tones[:-2]
lang_ids = lang_ids[:-2]
bert = bert[:, :-2]
ja_bert = ja_bert[:, :-2]
en_bert = en_bert[:, :-2]
with torch.no_grad():
x_tst = phones.to(device).unsqueeze(0)
tones = tones.to(device).unsqueeze(0)
lang_ids = lang_ids.to(device).unsqueeze(0)
bert = bert.to(device).unsqueeze(0)
ja_bert = ja_bert.to(device).unsqueeze(0)
en_bert = en_bert.to(device).unsqueeze(0)
x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
emo = emo.to(device).unsqueeze(0)
del phones
speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
audio = (
net_g.infer(
x_tst,
x_tst_lengths,
speakers,
tones,
lang_ids,
bert,
ja_bert,
en_bert,
emo,
sdp_ratio=sdp_ratio,
noise_scale=noise_scale,
noise_scale_w=noise_scale_w,
length_scale=length_scale,
)[0][0, 0]
.data.cpu()
.float()
.numpy()
)
del x_tst, tones, lang_ids, bert, x_tst_lengths, speakers, ja_bert, en_bert, emo
if torch.cuda.is_available():
torch.cuda.empty_cache()
return audio
def infer_multilang(
text,
sdp_ratio,
noise_scale,
noise_scale_w,
length_scale,
sid,
language,
hps,
net_g,
device,
bert=None,
clap=None,
reference_audio=None,
emotion=None,
skip_start=False,
skip_end=False,
):
bert, ja_bert, en_bert, phones, tones, lang_ids = [], [], [], [], [], []
# emo = get_emo_(reference_audio, emotion, sid)
if isinstance(reference_audio, np.ndarray):
emo = clap.get_clap_audio_feature(reference_audio, device)
else:
emo = clap.get_clap_text_feature(emotion, device)
emo = torch.squeeze(emo, dim=1)
for idx, (txt, lang) in enumerate(zip(text, language)):
skip_start = (idx != 0) or (skip_start and idx == 0)
skip_end = (idx != len(text) - 1) or (skip_end and idx == len(text) - 1)
(
temp_bert,
temp_ja_bert,
temp_en_bert,
temp_phones,
temp_tones,
temp_lang_ids,
) = get_text(txt, lang, bert, hps, device)
if skip_start:
temp_bert = temp_bert[:, 3:]
temp_ja_bert = temp_ja_bert[:, 3:]
temp_en_bert = temp_en_bert[:, 3:]
temp_phones = temp_phones[3:]
temp_tones = temp_tones[3:]
temp_lang_ids = temp_lang_ids[3:]
if skip_end:
temp_bert = temp_bert[:, :-2]
temp_ja_bert = temp_ja_bert[:, :-2]
temp_en_bert = temp_en_bert[:, :-2]
temp_phones = temp_phones[:-2]
temp_tones = temp_tones[:-2]
temp_lang_ids = temp_lang_ids[:-2]
bert.append(temp_bert)
ja_bert.append(temp_ja_bert)
en_bert.append(temp_en_bert)
phones.append(temp_phones)
tones.append(temp_tones)
lang_ids.append(temp_lang_ids)
bert = torch.concatenate(bert, dim=1)
ja_bert = torch.concatenate(ja_bert, dim=1)
en_bert = torch.concatenate(en_bert, dim=1)
phones = torch.concatenate(phones, dim=0)
tones = torch.concatenate(tones, dim=0)
lang_ids = torch.concatenate(lang_ids, dim=0)
with torch.no_grad():
x_tst = phones.to(device).unsqueeze(0)
tones = tones.to(device).unsqueeze(0)
lang_ids = lang_ids.to(device).unsqueeze(0)
bert = bert.to(device).unsqueeze(0)
ja_bert = ja_bert.to(device).unsqueeze(0)
en_bert = en_bert.to(device).unsqueeze(0)
emo = emo.to(device).unsqueeze(0)
x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
del phones
speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
audio = (
net_g.infer(
x_tst,
x_tst_lengths,
speakers,
tones,
lang_ids,
bert,
ja_bert,
en_bert,
emo,
sdp_ratio=sdp_ratio,
noise_scale=noise_scale,
noise_scale_w=noise_scale_w,
length_scale=length_scale,
)[0][0, 0]
.data.cpu()
.float()
.numpy()
)
del x_tst, tones, lang_ids, bert, x_tst_lengths, speakers, ja_bert, en_bert, emo
if torch.cuda.is_available():
torch.cuda.empty_cache()
return audio

111
for_deploy/infer_utils.py Normal file
View File

@@ -0,0 +1,111 @@
import sys
import torch
from transformers import (
AutoModelForMaskedLM,
AutoTokenizer,
DebertaV2Model,
DebertaV2Tokenizer,
ClapModel,
ClapProcessor,
)
from config import config
from text.japanese import text2sep_kata
class BertFeature:
def __init__(self, model_path, language="ZH"):
self.model_path = model_path
self.language = language
self.tokenizer = None
self.model = None
self.device = None
self._prepare()
def _get_device(self, device=config.bert_gen_config.device):
if (
sys.platform == "darwin"
and torch.backends.mps.is_available()
and device == "cpu"
):
device = "mps"
if not device:
device = "cuda"
return device
def _prepare(self):
self.device = self._get_device()
if self.language == "EN":
self.tokenizer = DebertaV2Tokenizer.from_pretrained(self.model_path)
self.model = DebertaV2Model.from_pretrained(self.model_path).to(self.device)
else:
self.tokenizer = AutoTokenizer.from_pretrained(self.model_path)
self.model = AutoModelForMaskedLM.from_pretrained(self.model_path).to(
self.device
)
self.model.eval()
def get_bert_feature(self, text, word2ph):
if self.language == "JP":
text = "".join(text2sep_kata(text)[0])
with torch.no_grad():
inputs = self.tokenizer(text, return_tensors="pt")
for i in inputs:
inputs[i] = inputs[i].to(self.device)
res = self.model(**inputs, output_hidden_states=True)
res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
word2phone = word2ph
phone_level_feature = []
for i in range(len(word2phone)):
repeat_feature = res[i].repeat(word2phone[i], 1)
phone_level_feature.append(repeat_feature)
phone_level_feature = torch.cat(phone_level_feature, dim=0)
return phone_level_feature.T
class ClapFeature:
def __init__(self, model_path):
self.model_path = model_path
self.processor = None
self.model = None
self.device = None
self._prepare()
def _get_device(self, device=config.bert_gen_config.device):
if (
sys.platform == "darwin"
and torch.backends.mps.is_available()
and device == "cpu"
):
device = "mps"
if not device:
device = "cuda"
return device
def _prepare(self):
self.device = self._get_device()
self.processor = ClapProcessor.from_pretrained(self.model_path)
self.model = ClapModel.from_pretrained(self.model_path).to(self.device)
self.model.eval()
def get_clap_audio_feature(self, audio_data):
with torch.no_grad():
inputs = self.processor(
audios=audio_data, return_tensors="pt", sampling_rate=48000
).to(self.device)
emb = self.model.get_audio_features(**inputs)
return emb.T
def get_clap_text_feature(self, text):
with torch.no_grad():
inputs = self.processor(text=text, return_tensors="pt").to(self.device)
emb = self.model.get_text_features(**inputs)
return emb.T

556
for_deploy/webui.py Normal file
View File

@@ -0,0 +1,556 @@
# flake8: noqa: E402
import os
import logging
import re_matching
from tools.sentence import split_by_language
logging.getLogger("numba").setLevel(logging.WARNING)
logging.getLogger("markdown_it").setLevel(logging.WARNING)
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("matplotlib").setLevel(logging.WARNING)
logging.basicConfig(
level=logging.INFO, format="| %(name)s | %(levelname)s | %(message)s"
)
logger = logging.getLogger(__name__)
import torch
import utils
from infer import infer, latest_version, get_net_g, infer_multilang
import gradio as gr
import webbrowser
import numpy as np
from config import config
from tools.translate import translate
import librosa
from infer_utils import BertFeature, ClapFeature
net_g = None
device = config.webui_config.device
if device == "mps":
os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
os.environ["OMP_NUM_THREADS"] = "1"
os.environ["MKL_NUM_THREADS"] = "1"
bert_feature_map = {
"ZH": BertFeature(
"./bert/chinese-roberta-wwm-ext-large",
language="ZH",
),
"JP": BertFeature(
"./bert/deberta-v2-large-japanese-char-wwm",
language="JP",
),
"EN": BertFeature(
"./bert/deberta-v3-large",
language="EN",
),
}
clap_feature = ClapFeature("./emotional/clap-htsat-fused")
def generate_audio(
slices,
sdp_ratio,
noise_scale,
noise_scale_w,
length_scale,
speaker,
language,
reference_audio,
emotion,
skip_start=False,
skip_end=False,
):
audio_list = []
# silence = np.zeros(hps.data.sampling_rate // 2, dtype=np.int16)
with torch.no_grad():
for idx, piece in enumerate(slices):
skip_start = (idx != 0) and skip_start
skip_end = (idx != len(slices) - 1) and skip_end
audio = infer(
piece,
reference_audio=reference_audio,
emotion=emotion,
sdp_ratio=sdp_ratio,
noise_scale=noise_scale,
noise_scale_w=noise_scale_w,
length_scale=length_scale,
sid=speaker,
language=language,
hps=hps,
net_g=net_g,
device=device,
skip_start=skip_start,
skip_end=skip_end,
bert=bert_feature_map,
clap=clap_feature,
)
audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
audio_list.append(audio16bit)
# audio_list.append(silence) # 将静音添加到列表中
return audio_list
def generate_audio_multilang(
slices,
sdp_ratio,
noise_scale,
noise_scale_w,
length_scale,
speaker,
language,
reference_audio,
emotion,
skip_start=False,
skip_end=False,
):
audio_list = []
# silence = np.zeros(hps.data.sampling_rate // 2, dtype=np.int16)
with torch.no_grad():
for idx, piece in enumerate(slices):
skip_start = (idx != 0) and skip_start
skip_end = (idx != len(slices) - 1) and skip_end
audio = infer_multilang(
piece,
reference_audio=reference_audio,
emotion=emotion,
sdp_ratio=sdp_ratio,
noise_scale=noise_scale,
noise_scale_w=noise_scale_w,
length_scale=length_scale,
sid=speaker,
language=language[idx],
hps=hps,
net_g=net_g,
device=device,
skip_start=skip_start,
skip_end=skip_end,
)
audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
audio_list.append(audio16bit)
# audio_list.append(silence) # 将静音添加到列表中
return audio_list
def tts_split(
text: str,
speaker,
sdp_ratio,
noise_scale,
noise_scale_w,
length_scale,
language,
cut_by_sent,
interval_between_para,
interval_between_sent,
reference_audio,
emotion,
):
if language == "mix":
return ("invalid", None)
while text.find("\n\n") != -1:
text = text.replace("\n\n", "\n")
para_list = re_matching.cut_para(text)
audio_list = []
if not cut_by_sent:
for idx, p in enumerate(para_list):
skip_start = idx != 0
skip_end = idx != len(para_list) - 1
audio = infer(
p,
reference_audio=reference_audio,
emotion=emotion,
sdp_ratio=sdp_ratio,
noise_scale=noise_scale,
noise_scale_w=noise_scale_w,
length_scale=length_scale,
sid=speaker,
language=language,
hps=hps,
net_g=net_g,
device=device,
skip_start=skip_start,
skip_end=skip_end,
)
audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
audio_list.append(audio16bit)
silence = np.zeros((int)(44100 * interval_between_para), dtype=np.int16)
audio_list.append(silence)
else:
for idx, p in enumerate(para_list):
skip_start = idx != 0
skip_end = idx != len(para_list) - 1
audio_list_sent = []
sent_list = re_matching.cut_sent(p)
for idx, s in enumerate(sent_list):
skip_start = (idx != 0) and skip_start
skip_end = (idx != len(sent_list) - 1) and skip_end
audio = infer(
s,
reference_audio=reference_audio,
emotion=emotion,
sdp_ratio=sdp_ratio,
noise_scale=noise_scale,
noise_scale_w=noise_scale_w,
length_scale=length_scale,
sid=speaker,
language=language,
hps=hps,
net_g=net_g,
device=device,
skip_start=skip_start,
skip_end=skip_end,
)
audio_list_sent.append(audio)
silence = np.zeros((int)(44100 * interval_between_sent))
audio_list_sent.append(silence)
if (interval_between_para - interval_between_sent) > 0:
silence = np.zeros(
(int)(44100 * (interval_between_para - interval_between_sent))
)
audio_list_sent.append(silence)
audio16bit = gr.processing_utils.convert_to_16_bit_wav(
np.concatenate(audio_list_sent)
) # 对完整句子做音量归一
audio_list.append(audio16bit)
audio_concat = np.concatenate(audio_list)
return ("Success", (44100, audio_concat))
def tts_fn(
text: str,
speaker,
sdp_ratio,
noise_scale,
noise_scale_w,
length_scale,
language,
reference_audio,
emotion,
prompt_mode,
):
if prompt_mode == "Audio prompt":
if reference_audio == None:
return ("Invalid audio prompt", None)
else:
reference_audio = load_audio(reference_audio)[1]
else:
reference_audio = None
audio_list = []
if language == "mix":
bool_valid, str_valid = re_matching.validate_text(text)
if not bool_valid:
return str_valid, (
hps.data.sampling_rate,
np.concatenate([np.zeros(hps.data.sampling_rate // 2)]),
)
result = []
for slice in re_matching.text_matching(text):
_speaker = slice.pop()
temp_contant = []
temp_lang = []
for lang, content in slice:
if "|" in content:
temp = []
temp_ = []
for i in content.split("|"):
if i != "":
temp.append([i])
temp_.append([lang])
else:
temp.append([])
temp_.append([])
temp_contant += temp
temp_lang += temp_
else:
if len(temp_contant) == 0:
temp_contant.append([])
temp_lang.append([])
temp_contant[-1].append(content)
temp_lang[-1].append(lang)
for i, j in zip(temp_lang, temp_contant):
result.append([*zip(i, j), _speaker])
for i, one in enumerate(result):
skip_start = i != 0
skip_end = i != len(result) - 1
_speaker = one.pop()
idx = 0
while idx < len(one):
text_to_generate = []
lang_to_generate = []
while True:
lang, content = one[idx]
temp_text = [content]
if len(text_to_generate) > 0:
text_to_generate[-1] += [temp_text.pop(0)]
lang_to_generate[-1] += [lang]
if len(temp_text) > 0:
text_to_generate += [[i] for i in temp_text]
lang_to_generate += [[lang]] * len(temp_text)
if idx + 1 < len(one):
idx += 1
else:
break
skip_start = (idx != 0) and skip_start
skip_end = (idx != len(one) - 1) and skip_end
print(text_to_generate, lang_to_generate)
audio_list.extend(
generate_audio_multilang(
text_to_generate,
sdp_ratio,
noise_scale,
noise_scale_w,
length_scale,
_speaker,
lang_to_generate,
reference_audio,
emotion,
skip_start,
skip_end,
)
)
idx += 1
elif language.lower() == "auto":
for idx, slice in enumerate(text.split("|")):
if slice == "":
continue
skip_start = idx != 0
skip_end = idx != len(text.split("|")) - 1
sentences_list = split_by_language(
slice, target_languages=["zh", "ja", "en"]
)
idx = 0
while idx < len(sentences_list):
text_to_generate = []
lang_to_generate = []
while True:
content, lang = sentences_list[idx]
temp_text = [content]
lang = lang.upper()
if lang == "JA":
lang = "JP"
if len(text_to_generate) > 0:
text_to_generate[-1] += [temp_text.pop(0)]
lang_to_generate[-1] += [lang]
if len(temp_text) > 0:
text_to_generate += [[i] for i in temp_text]
lang_to_generate += [[lang]] * len(temp_text)
if idx + 1 < len(sentences_list):
idx += 1
else:
break
skip_start = (idx != 0) and skip_start
skip_end = (idx != len(sentences_list) - 1) and skip_end
print(text_to_generate, lang_to_generate)
audio_list.extend(
generate_audio_multilang(
text_to_generate,
sdp_ratio,
noise_scale,
noise_scale_w,
length_scale,
speaker,
lang_to_generate,
reference_audio,
emotion,
skip_start,
skip_end,
)
)
idx += 1
else:
audio_list.extend(
generate_audio(
text.split("|"),
sdp_ratio,
noise_scale,
noise_scale_w,
length_scale,
speaker,
language,
reference_audio,
emotion,
)
)
audio_concat = np.concatenate(audio_list)
return "Success", (hps.data.sampling_rate, audio_concat)
def load_audio(path):
audio, sr = librosa.load(path, 48000)
# audio = librosa.resample(audio, 44100, 48000)
return sr, audio
def gr_util(item):
if item == "Text prompt":
return {"visible": True, "__type__": "update"}, {
"visible": False,
"__type__": "update",
}
else:
return {"visible": False, "__type__": "update"}, {
"visible": True,
"__type__": "update",
}
if __name__ == "__main__":
if config.webui_config.debug:
logger.info("Enable DEBUG-LEVEL log")
logging.basicConfig(level=logging.DEBUG)
hps = utils.get_hparams_from_file(config.webui_config.config_path)
# 若config.json中未指定版本则默认为最新版本
version = hps.version if hasattr(hps, "version") else latest_version
net_g = get_net_g(
model_path=config.webui_config.model, version=version, device=device, hps=hps
)
speaker_ids = hps.data.spk2id
speakers = list(speaker_ids.keys())
languages = ["ZH", "JP", "EN", "mix", "auto"]
with gr.Blocks() as app:
with gr.Row():
with gr.Column():
text = gr.TextArea(
label="输入文本内容",
placeholder="""
如果你选择语言为\'mix\'，必须按照格式输入，否则报错:
格式举例(zh是中文，jp是日语，不区分大小写；说话人举例:gongzi):
[说话人1]<zh>你好，こんにちは！ <jp>こんにちは，世界。
[说话人2]<zh>你好吗？<jp>元気ですか？
[说话人3]<zh>谢谢。<jp>どういたしまして。
...
另外，所有的语言选项都可以用'|'分割长段实现分句生成。
""",
)
trans = gr.Button("中翻日", variant="primary")
slicer = gr.Button("快速切分", variant="primary")
speaker = gr.Dropdown(
choices=speakers, value=speakers[0], label="Speaker"
)
_ = gr.Markdown(
value="提示模式（Prompt mode）：可选文字提示或音频提示，用于生成文字或音频指定风格的声音。\n"
)
prompt_mode = gr.Radio(
["Text prompt", "Audio prompt"],
label="Prompt Mode",
value="Text prompt",
)
text_prompt = gr.Textbox(
label="Text prompt",
placeholder="用文字描述生成风格。如：Happy",
value="Happy",
visible=True,
)
audio_prompt = gr.Audio(
label="Audio prompt", type="filepath", visible=False
)
sdp_ratio = gr.Slider(
minimum=0, maximum=1, value=0.2, step=0.1, label="SDP Ratio"
)
noise_scale = gr.Slider(
minimum=0.1, maximum=2, value=0.6, step=0.1, label="Noise"
)
noise_scale_w = gr.Slider(
minimum=0.1, maximum=2, value=0.8, step=0.1, label="Noise_W"
)
length_scale = gr.Slider(
minimum=0.1, maximum=2, value=1.0, step=0.1, label="Length"
)
language = gr.Dropdown(
choices=languages, value=languages[0], label="Language"
)
btn = gr.Button("生成音频！", variant="primary")
with gr.Column():
with gr.Row():
with gr.Column():
interval_between_sent = gr.Slider(
minimum=0,
maximum=5,
value=0.2,
step=0.1,
label="句间停顿(秒)，勾选按句切分才生效",
)
interval_between_para = gr.Slider(
minimum=0,
maximum=10,
value=1,
step=0.1,
label="段间停顿(秒)，需要大于句间停顿才有效",
)
opt_cut_by_sent = gr.Checkbox(
label="按句切分 在按段落切分的基础上再按句子切分文本"
)
slicer = gr.Button("切分生成", variant="primary")
text_output = gr.Textbox(label="状态信息")
audio_output = gr.Audio(label="输出音频")
# explain_image = gr.Image(
# label="参数解释信息",
# show_label=True,
# show_share_button=False,
# show_download_button=False,
# value=os.path.abspath("./img/参数说明.png"),
# )
btn.click(
tts_fn,
inputs=[
text,
speaker,
sdp_ratio,
noise_scale,
noise_scale_w,
length_scale,
language,
audio_prompt,
text_prompt,
prompt_mode,
],
outputs=[text_output, audio_output],
)
trans.click(
translate,
inputs=[text],
outputs=[text],
)
slicer.click(
tts_split,
inputs=[
text,
speaker,
sdp_ratio,
noise_scale,
noise_scale_w,
length_scale,
language,
opt_cut_by_sent,
interval_between_para,
interval_between_sent,
audio_prompt,
text_prompt,
],
outputs=[text_output, audio_output],
)
prompt_mode.change(
lambda x: gr_util(x),
inputs=[prompt_mode],
outputs=[text_prompt, audio_prompt],
)
audio_prompt.upload(
lambda x: load_audio(x),
inputs=[audio_prompt],
outputs=[audio_prompt],
)
print("推理页面已开启!")
webbrowser.open(f"http://127.0.0.1:{config.webui_config.port}")
app.launch(share=config.webui_config.share, server_port=config.webui_config.port)