tuna2134/sbv2-v2
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Add: Preparation for ONNX inference support ②

Browse Source
This commit is contained in:
tsukumi
2024-09-17 09:02:14 +09:00
parent eede796c4e
commit b9fdeb4492
5 changed files with 439 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
style_bert_vits2/models/infer.py
View File

@@ -12,14 +12,16 @@ from style_bert_vits2.models.models_jp_extra import (
SynthesizerTrn as SynthesizerTrnJPExtra,
)
from style_bert_vits2.nlp import (
clean_text,
clean_text_with_given_phone_tone,
cleaned_text_to_sequence,
extract_bert_feature,
)
from style_bert_vits2.nlp.symbols import SYMBOLS
def get_net_g(model_path: str, version: str, device: str, hps: HyperParameters):
def get_net_g(
model_path: str, version: str, device: str, hps: HyperParameters
) -> Union[SynthesizerTrn, SynthesizerTrnJPExtra]:
if version.endswith("JP-Extra"):
logger.info("Using JP-Extra model")
net_g = SynthesizerTrnJPExtra(
@@ -104,59 +106,19 @@ def get_text(
assist_text_weight: float = 0.7,
given_phone: Optional[list[str]] = None,
given_tone: Optional[list[int]] = None,
):
) -> tuple[
torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor
]:
use_jp_extra = hps.version.endswith("JP-Extra")
# 推論時のみ呼び出されるので、raise_yomi_error は False に設定
norm_text, phone, tone, word2ph = clean_text(
norm_text, phone, tone, word2ph = clean_text_with_given_phone_tone(
text,
language_str,
given_phone=given_phone,
given_tone=given_tone,
use_jp_extra=use_jp_extra,
# 推論時のみ呼び出されるので、raise_yomi_error は False に設定
raise_yomi_error=False,
)
# phone と tone の両方が与えられた場合はそれを使う
if given_phone is not None and given_tone is not None:
# 指定された phone と指定された tone 両方の長さが一致していなければならない
if len(given_phone) != len(given_tone):
raise InvalidPhoneError(
f"Length of given_phone ({len(given_phone)}) != length of given_tone ({len(given_tone)})"
)
# 与えられた音素数と pyopenjtalk で生成した読みの音素数が一致しない
if len(given_phone) != sum(word2ph):
# 日本語の場合、len(given_phone) と sum(word2ph) が一致するように word2ph を適切に調整する
# 他の言語は word2ph の調整方法が思いつかないのでエラー
if language_str == Languages.JP:
from style_bert_vits2.nlp.japanese.g2p import adjust_word2ph
# use_jp_extra でない場合は given_phone 内の「N」を「n」に変換
if not use_jp_extra:
given_phone = [p if p != "N" else "n" for p in given_phone]
# clean_text() から取得した word2ph を調整結果で上書き
word2ph = adjust_word2ph(word2ph, phone, given_phone)
# 上記処理により word2ph の合計が given_phone の長さと一致するはず
# それでも一致しない場合、大半は読み上げテキストと given_phone が著しく乖離していて調整し切れなかったことを意味する
if len(given_phone) != sum(word2ph):
raise InvalidPhoneError(
f"Length of given_phone ({len(given_phone)}) != sum of word2ph ({sum(word2ph)})"
)
else:
raise InvalidPhoneError(
f"Length of given_phone ({len(given_phone)}) != sum of word2ph ({sum(word2ph)})"
)
phone = given_phone
# 生成あるいは指定された phone と指定された tone 両方の長さが一致していなければならない
if len(phone) != len(given_tone):
raise InvalidToneError(
f"Length of phone ({len(phone)}) != length of given_tone ({len(given_tone)})"
)
tone = given_tone
# tone だけが与えられた場合は clean_text() で生成した phone と合わせて使う
elif given_tone is not None:
# 生成した phone と指定された tone 両方の長さが一致していなければならない
if len(phone) != len(given_tone):
raise InvalidToneError(
f"Length of phone ({len(phone)}) != length of given_tone ({len(given_tone)})"
)
tone = given_tone
phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
if hps.data.add_blank:
@@ -220,7 +182,7 @@ def infer(
assist_text_weight: float = 0.7,
given_phone: Optional[list[str]] = None,
given_tone: Optional[list[int]] = None,
):
) -> NDArray[Any]:
is_jp_extra = hps.version.endswith("JP-Extra")
bert, ja_bert, en_bert, phones, tones, lang_ids = get_text(
text,
@@ -246,6 +208,7 @@ def infer(
bert = bert[:, :-2]
ja_bert = ja_bert[:, :-2]
en_bert = en_bert[:, :-2]
with torch.no_grad():
x_tst = phones.to(device).unsqueeze(0)
tones = tones.to(device).unsqueeze(0)
@@ -257,6 +220,7 @@ def infer(
style_vec_tensor = torch.from_numpy(style_vec).to(device).unsqueeze(0)
del phones
sid_tensor = torch.LongTensor([sid]).to(device)
if is_jp_extra:
output = cast(SynthesizerTrnJPExtra, net_g).infer(
x_tst,
@@ -287,7 +251,9 @@ def infer(
noise_scale_w=noise_scale_w,
length_scale=length_scale,
)
audio = output[0][0, 0].data.cpu().float().numpy()
del (
x_tst,
tones,
@@ -301,12 +267,5 @@ def infer(
) # , emo
if torch.cuda.is_available():
torch.cuda.empty_cache()
return audio
class InvalidPhoneError(ValueError):
pass
class InvalidToneError(ValueError):
pass

178
style_bert_vits2/models/infer_onnx.py Normal file
View File

@@ -0,0 +1,178 @@
from typing import Any, Optional, Sequence
import numpy as np
import onnxruntime
from numpy.typing import NDArray
from style_bert_vits2.constants import Languages
from style_bert_vits2.models import commons
from style_bert_vits2.models.hyper_parameters import HyperParameters
from style_bert_vits2.nlp import (
clean_text_with_given_phone_tone,
cleaned_text_to_sequence,
extract_bert_feature_onnx,
)
def get_text_onnx(
text: str,
language_str: Languages,
hps: HyperParameters,
onnx_providers: list[str],
onnx_provider_options: Optional[Sequence[dict[str, Any]]],
assist_text: Optional[str] = None,
assist_text_weight: float = 0.7,
given_phone: Optional[list[str]] = None,
given_tone: Optional[list[int]] = None,
) -> tuple[
NDArray[Any], NDArray[Any], NDArray[Any], NDArray[Any], NDArray[Any], NDArray[Any]
]:
use_jp_extra = hps.version.endswith("JP-Extra")
norm_text, phone, tone, word2ph = clean_text_with_given_phone_tone(
text,
language_str,
given_phone=given_phone,
given_tone=given_tone,
use_jp_extra=use_jp_extra,
# 推論時のみ呼び出されるので、raise_yomi_error は False に設定
raise_yomi_error=False,
)
phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
if hps.data.add_blank:
phone = commons.intersperse(phone, 0)
tone = commons.intersperse(tone, 0)
language = commons.intersperse(language, 0)
for i in range(len(word2ph)):
word2ph[i] = word2ph[i] * 2
word2ph[0] += 1
bert_ori = extract_bert_feature_onnx(
norm_text,
word2ph,
language_str,
onnx_providers,
onnx_provider_options,
assist_text,
assist_text_weight,
)
del word2ph
assert bert_ori.shape[-1] == len(phone), phone
if language_str == Languages.ZH:
bert = bert_ori
ja_bert = np.zeros((1024, len(phone)))
en_bert = np.zeros((1024, len(phone)))
elif language_str == Languages.JP:
bert = np.zeros((1024, len(phone)))
ja_bert = bert_ori
en_bert = np.zeros((1024, len(phone)))
elif language_str == Languages.EN:
bert = np.zeros((1024, len(phone)))
ja_bert = np.zeros((1024, len(phone)))
en_bert = bert_ori
else:
raise ValueError("language_str should be ZH, JP or EN")
assert bert.shape[-1] == len(
phone
), f"Bert seq len {bert.shape[-1]} != {len(phone)}"
phone = np.array(phone)
tone = np.array(tone)
language = np.array(language)
return bert, ja_bert, en_bert, phone, tone, language
def infer_onnx(
text: str,
style_vec: NDArray[Any],
sdp_ratio: float,
noise_scale: float,
noise_scale_w: float,
length_scale: float,
sid: int, # In the original Bert-VITS2, its speaker_name: str, but here it's id
language: Languages,
hps: HyperParameters,
onnx_session: onnxruntime.InferenceSession,
onnx_providers: list[str],
onnx_provider_options: Optional[Sequence[dict[str, Any]]],
skip_start: bool = False,
skip_end: bool = False,
assist_text: Optional[str] = None,
assist_text_weight: float = 0.7,
given_phone: Optional[list[str]] = None,
given_tone: Optional[list[int]] = None,
) -> NDArray[Any]:
is_jp_extra = hps.version.endswith("JP-Extra")
bert, ja_bert, en_bert, phones, tones, lang_ids = get_text_onnx(
text,
language,
hps,
onnx_providers=onnx_providers,
onnx_provider_options=onnx_provider_options,
assist_text=assist_text,
assist_text_weight=assist_text_weight,
given_phone=given_phone,
given_tone=given_tone,
)
if skip_start:
phones = phones[3:]
tones = tones[3:]
lang_ids = lang_ids[3:]
bert = bert[:, 3:]
ja_bert = ja_bert[:, 3:]
en_bert = en_bert[:, 3:]
if skip_end:
phones = phones[:-2]
tones = tones[:-2]
lang_ids = lang_ids[:-2]
bert = bert[:, :-2]
ja_bert = ja_bert[:, :-2]
en_bert = en_bert[:, :-2]
x_tst = np.expand_dims(phones, axis=0)
tones = np.expand_dims(tones, axis=0)
lang_ids = np.expand_dims(lang_ids, axis=0)
bert = np.expand_dims(bert, axis=0)
ja_bert = np.expand_dims(ja_bert, axis=0)
en_bert = np.expand_dims(en_bert, axis=0)
x_tst_lengths = np.array([phones.shape[0]], dtype=np.int64)
style_vec_tensor = np.expand_dims(style_vec, axis=0)
del phones
sid_tensor = np.array([sid], dtype=np.int64)
input_names = [input.name for input in onnx_session.get_inputs()]
output_name = onnx_session.get_outputs()[0].name
if is_jp_extra:
output = onnx_session.run(
[output_name],
{
input_names[0]: x_tst,
input_names[1]: x_tst_lengths,
input_names[2]: sid_tensor,
input_names[3]: tones,
input_names[4]: lang_ids,
input_names[5]: ja_bert,
input_names[6]: style_vec_tensor,
input_names[7]: length_scale,
input_names[8]: sdp_ratio,
},
)
else:
raise NotImplementedError("Not implemented yet")
audio = output[0][0, 0]
del (
x_tst,
tones,
lang_ids,
bert,
x_tst_lengths,
sid_tensor,
ja_bert,
en_bert,
style_vec,
) # , emo
return audio