tuna2134/sbv2-v2
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Add: Support for speech synthesis in English and Chinese for ONNX inference in Non-JP-Extra models

Browse Source
This commit is contained in:
tsukumi
2024-09-23 07:14:44 +09:00
parent fe253611d3
commit dcca37b9a8
20 changed files with 342182 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
style_bert_vits2/nlp/__init__.py
View File

@@ -81,6 +81,10 @@ def extract_bert_feature_onnx(
if language == Languages.JP:
from style_bert_vits2.nlp.japanese.bert_feature import extract_bert_feature_onnx
elif language == Languages.EN:
from style_bert_vits2.nlp.english.bert_feature import extract_bert_feature_onnx
elif language == Languages.ZH:
from style_bert_vits2.nlp.chinese.bert_feature import extract_bert_feature_onnx
else:
raise ValueError(f"Language {language} not supported")

87
style_bert_vits2/nlp/chinese/bert_feature.py
View File

@@ -1,9 +1,16 @@
from typing import Optional
from __future__ import annotations
import torch
from typing import TYPE_CHECKING, Any, Optional, Sequence, Union
import numpy as np
from numpy.typing import NDArray
from style_bert_vits2.constants import Languages
from style_bert_vits2.nlp import bert_models
from style_bert_vits2.nlp import bert_models, onnx_bert_models
if TYPE_CHECKING:
import torch
def extract_bert_feature(
@@ -14,7 +21,7 @@ def extract_bert_feature(
assist_text_weight: float = 0.7,
) -> torch.Tensor:
"""
中国語のテキストから BERT の特徴量を抽出する
中国語のテキストから BERT の特徴量を抽出する (PyTorch 推論)
Args:
text (str): 中国語のテキスト
@@ -27,6 +34,8 @@ def extract_bert_feature(
torch.Tensor: BERT の特徴量
"""
import torch
if device == "cuda" and not torch.cuda.is_available():
device = "cpu"
model = bert_models.load_model(Languages.ZH, device_map=device)
@@ -67,6 +76,76 @@ def extract_bert_feature(
return phone_level_feature.T
def extract_bert_feature_onnx(
text: str,
word2ph: list[int],
onnx_providers: Sequence[Union[str, tuple[str, dict[str, Any]]]],
assist_text: Optional[str] = None,
assist_text_weight: float = 0.7,
) -> NDArray[Any]:
"""
中国語のテキストから BERT の特徴量を抽出する (ONNX 推論)
Args:
text (str): 中国語のテキスト
word2ph (list[int]): 元のテキストの各文字に音素が何個割り当てられるかを表すリスト
onnx_providers (list[str]): ONNX 推論で利用する ExecutionProvider (CPUExecutionProvider, CUDAExecutionProvider など)
assist_text (Optional[str], optional): 補助テキスト (デフォルト: None)
assist_text_weight (float, optional): 補助テキストの重み (デフォルト: 0.7)
Returns:
NDArray[Any]: BERT の特徴量
"""
tokenizer = onnx_bert_models.load_tokenizer(Languages.ZH)
inputs = tokenizer(text, return_tensors="pt")
session = onnx_bert_models.load_model(
language=Languages.ZH,
onnx_providers=onnx_providers,
)
output_name = session.get_outputs()[0].name
res = session.run(
[output_name],
{
"input_ids": inputs["input_ids"].detach().numpy(), # type: ignore
"token_type_ids": inputs["token_type_ids"].detach().numpy(), # type: ignore
"attention_mask": inputs["attention_mask"].detach().numpy(), # type: ignore
},
)[0]
style_res_mean = None
if assist_text:
style_inputs = tokenizer(assist_text, return_tensors="pt")
style_res = session.run(
[output_name],
{
"input_ids": style_inputs["input_ids"].detach().numpy(), # type: ignore
"token_type_ids": style_inputs["token_type_ids"].detach().numpy(), # type: ignore
"attention_mask": style_inputs["attention_mask"].detach().numpy(), # type: ignore
},
)[0]
style_res_mean = np.mean(style_res, axis=0)
assert len(word2ph) == len(text) + 2
word2phone = word2ph
phone_level_feature = []
for i in range(len(word2phone)):
if assist_text:
assert style_res_mean is not None
repeat_feature = (
np.tile(res[i], (word2phone[i], 1)) * (1 - assist_text_weight)
+ np.tile(style_res_mean, (word2phone[i], 1)) * assist_text_weight
)
else:
repeat_feature = np.tile(res[i], (word2phone[i], 1))
phone_level_feature.append(repeat_feature)
phone_level_feature = np.concatenate(phone_level_feature, axis=0)
return phone_level_feature.T
if __name__ == "__main__":
word_level_feature = torch.rand(38, 1024) # 12个词,每个词1024维特征
word2phone = [

85
style_bert_vits2/nlp/english/bert_feature.py
View File

@@ -1,9 +1,16 @@
from typing import Optional
from __future__ import annotations
import torch
from typing import TYPE_CHECKING, Any, Optional, Sequence, Union
import numpy as np
from numpy.typing import NDArray
from style_bert_vits2.constants import Languages
from style_bert_vits2.nlp import bert_models
from style_bert_vits2.nlp import bert_models, onnx_bert_models
if TYPE_CHECKING:
import torch
def extract_bert_feature(
@@ -14,7 +21,7 @@ def extract_bert_feature(
assist_text_weight: float = 0.7,
) -> torch.Tensor:
"""
英語のテキストから BERT の特徴量を抽出する
英語のテキストから BERT の特徴量を抽出する (PyTorch 推論)
Args:
text (str): 英語のテキスト
@@ -27,6 +34,8 @@ def extract_bert_feature(
torch.Tensor: BERT の特徴量
"""
import torch
if device == "cuda" and not torch.cuda.is_available():
device = "cpu"
model = bert_models.load_model(Languages.EN, device_map=device)
@@ -65,3 +74,71 @@ def extract_bert_feature(
phone_level_feature = torch.cat(phone_level_feature, dim=0)
return phone_level_feature.T
def extract_bert_feature_onnx(
text: str,
word2ph: list[int],
onnx_providers: Sequence[Union[str, tuple[str, dict[str, Any]]]],
assist_text: Optional[str] = None,
assist_text_weight: float = 0.7,
) -> NDArray[Any]:
"""
英語のテキストから BERT の特徴量を抽出する (ONNX 推論)
Args:
text (str): 英語のテキスト
word2ph (list[int]): 元のテキストの各文字に音素が何個割り当てられるかを表すリスト
onnx_providers (list[str]): ONNX 推論で利用する ExecutionProvider (CPUExecutionProvider, CUDAExecutionProvider など)
assist_text (Optional[str], optional): 補助テキスト (デフォルト: None)
assist_text_weight (float, optional): 補助テキストの重み (デフォルト: 0.7)
Returns:
NDArray[Any]: BERT の特徴量
"""
tokenizer = onnx_bert_models.load_tokenizer(Languages.EN)
inputs = tokenizer(text, return_tensors="pt")
session = onnx_bert_models.load_model(
language=Languages.EN,
onnx_providers=onnx_providers,
)
output_name = session.get_outputs()[0].name
res = session.run(
[output_name],
{
"input_ids": inputs["input_ids"].detach().numpy(), # type: ignore
"attention_mask": inputs["attention_mask"].detach().numpy(), # type: ignore
},
)[0]
style_res_mean = None
if assist_text:
style_inputs = tokenizer(assist_text, return_tensors="pt")
style_res = session.run(
[output_name],
{
"input_ids": style_inputs["input_ids"].detach().numpy(), # type: ignore
"attention_mask": style_inputs["attention_mask"].detach().numpy(), # type: ignore
},
)[0]
style_res_mean = np.mean(style_res, axis=0)
assert len(word2ph) == res.shape[0], (text, res.shape[0], len(word2ph))
word2phone = word2ph
phone_level_feature = []
for i in range(len(word2phone)):
if assist_text:
assert style_res_mean is not None
repeat_feature = (
np.tile(res[i], (word2phone[i], 1)) * (1 - assist_text_weight)
+ np.tile(style_res_mean, (word2phone[i], 1)) * assist_text_weight
)
else:
repeat_feature = np.tile(res[i], (word2phone[i], 1))
phone_level_feature.append(repeat_feature)
phone_level_feature = np.concatenate(phone_level_feature, axis=0)
return phone_level_feature.T

8
style_bert_vits2/nlp/japanese/bert_feature.py
View File

@@ -121,8 +121,8 @@ def extract_bert_feature_onnx(
res = session.run(
[output_name],
{
"input_ids": inputs["input_ids"].detach().numpy(),
"attention_mask": inputs["attention_mask"].detach().numpy(),
"input_ids": inputs["input_ids"].detach().numpy(), # type: ignore
"attention_mask": inputs["attention_mask"].detach().numpy(), # type: ignore
},
)[0]
@@ -132,8 +132,8 @@ def extract_bert_feature_onnx(
style_res = session.run(
[output_name],
{
"input_ids": style_inputs["input_ids"].detach().numpy(),
"attention_mask": style_inputs["attention_mask"].detach().numpy(),
"input_ids": style_inputs["input_ids"].detach().numpy(), # type: ignore
"attention_mask": style_inputs["attention_mask"].detach().numpy(), # type: ignore
},
)[0]
style_res_mean = np.mean(style_res, axis=0)