tuna2134/sbv2-v2
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Style

Browse Source
This commit is contained in:
litagin02
2024-06-22 14:54:47 +09:00
parent a8f44649ed
commit 9eab977138
2 changed files with 168 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

206
gradio_tabs/inference.py
View File

@@ -1,6 +1,6 @@
import datetime import datetime
import json import json
from typing import Optional, Any, Union from typing import Any, Optional, Union
import gradio as gr import gradio as gr
@@ -190,6 +190,7 @@ voice_pitch_keys = ["flow"]
speech_style_keys = ["enc_p"] speech_style_keys = ["enc_p"]
tempo_keys = ["sdp", "dp"] tempo_keys = ["sdp", "dp"]
def make_interactive(): def make_interactive():
return gr.update(interactive=True, value="音声合成") return gr.update(interactive=True, value="音声合成")
@@ -204,27 +205,38 @@ def gr_util(item):
else: else:
return (gr.update(visible=False), gr.update(visible=True)) return (gr.update(visible=False), gr.update(visible=True))
null_models_frame = 0 null_models_frame = 0
def change_null_model_row(null_model_index:int, null_model_name:str, null_model_path:str,null_voice_weights:float,
null_voice_pitch_weights:float, null_speech_style_weights:float,null_tempo_weights:float,
null_models:dict[int,dict[str, Any]]): def change_null_model_row(
#logger.debug("change_null_model_row:sta"+str(null_models)) null_model_index: int,
mid_result={} null_model_name: str,
mid_result["name"]=null_model_name null_model_path: str,
mid_result["path"]=null_model_path null_voice_weights: float,
mid_result["weight"]=null_tempo_weights null_voice_pitch_weights: float,
mid_result["pitch"]=null_voice_pitch_weights null_speech_style_weights: float,
mid_result["style"]=null_speech_style_weights null_tempo_weights: float,
mid_result["tempo"]=null_tempo_weights null_models: dict[int, dict[str, Any]],
):
# logger.debug("change_null_model_row:sta"+str(null_models))
mid_result = {}
mid_result["name"] = null_model_name
mid_result["path"] = null_model_path
mid_result["weight"] = null_tempo_weights
mid_result["pitch"] = null_voice_pitch_weights
mid_result["style"] = null_speech_style_weights
mid_result["tempo"] = null_tempo_weights
null_models[null_model_index] = mid_result null_models[null_model_index] = mid_result
#logger.debug("decreasing:"+str(null_models_frame)+":"+str(len(null_models.keys()))) # logger.debug("decreasing:"+str(null_models_frame)+":"+str(len(null_models.keys())))
if null_models_frame < len(null_models.keys()): if null_models_frame < len(null_models.keys()):
for i in range(null_models_frame ,len(null_models.keys())): for i in range(null_models_frame, len(null_models.keys())):
_ = null_models.pop(i, None) _ = null_models.pop(i, None)
result = null_models result = null_models
#logger.debug("change_null_model_row:res"+str(null_models)) # logger.debug("change_null_model_row:res"+str(null_models))
return result, True return result, True
def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks: def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks:
def tts_fn( def tts_fn(
model_name, model_name,
@@ -248,8 +260,8 @@ def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks:
speaker, speaker,
pitch_scale, pitch_scale,
intonation_scale, intonation_scale,
null_models:dict[int, dict[str, Union[str, float]]], null_models: dict[int, dict[str, Union[str, float]]],
force_reload_model:bool force_reload_model: bool,
): ):
model_holder.get_model(model_name, model_path) model_holder.get_model(model_name, model_path)
assert model_holder.current_model is not None assert model_holder.current_model is not None
@@ -307,8 +319,8 @@ def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks:
speaker_id=speaker_id, speaker_id=speaker_id,
pitch_scale=pitch_scale, pitch_scale=pitch_scale,
intonation_scale=intonation_scale, intonation_scale=intonation_scale,
null_model_params = null_models, null_model_params=null_models,
force_reload_model = force_reload_model force_reload_model=force_reload_model,
) )
except InvalidToneError as e: except InvalidToneError as e:
logger.error(f"Tone error: {e}") logger.error(f"Tone error: {e}")
@@ -467,15 +479,18 @@ def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks:
) )
with gr.Accordion(label="ヌルモデル", open=False): with gr.Accordion(label="ヌルモデル", open=False):
with gr.Row() as null_row: with gr.Row() as null_row:
null_models_count = gr.Number(label="ヌルモデルの数", value=0, step=1) null_models_count = gr.Number(
label="ヌルモデルの数", value=0, step=1
)
with gr.Column(variant="panel") as null_column: with gr.Column(variant="panel") as null_column:
@gr.render( @gr.render(
inputs=[ inputs=[
null_models_count, null_models_count,
] ]
) )
def render_style( def render_style(
null_models_count:int, null_models_count: int,
): ):
global null_models_frame global null_models_frame
null_models_frame = null_models_count null_models_frame = null_models_count
@@ -484,28 +499,34 @@ def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks:
null_model_index = gr.Number( null_model_index = gr.Number(
value=i, value=i,
key=f"null_model_index_{i}", key=f"null_model_index_{i}",
visible=False visible=False,
) )
null_model_name = gr.Dropdown( null_model_name = gr.Dropdown(
label="モデル一覧", label="モデル一覧",
choices=model_names, choices=model_names,
key=f"null_model_name_{i}", key=f"null_model_name_{i}",
value=model_names[initial_id], value=model_names[initial_id],
interactive=True interactive=True,
) )
if i in null_models.value: if i in null_models.value:
logger.debug(f"null model parameter exists in index {i}") logger.debug(
null_model_name.value=null_models.value[i]["name"] f"null model parameter exists in index {i}"
)
null_model_name.value = null_models.value[i][
"name"
]
null_model_path = gr.Dropdown( null_model_path = gr.Dropdown(
label="モデルファイル", label="モデルファイル",
choices=initial_pth_files, choices=initial_pth_files,
key=f"null_model_path_{i}", key=f"null_model_path_{i}",
value=initial_pth_files[0], value=initial_pth_files[0],
interactive=True interactive=True,
) )
if i in null_models.value: if i in null_models.value:
#null_model_path.choices = #ToDo # null_model_path.choices = #ToDo
null_model_path.value=null_models.value[i]["path"] null_model_path.value = null_models.value[i][
"path"
]
null_voice_weights = gr.Slider( null_voice_weights = gr.Slider(
minimum=0, minimum=0,
maximum=1, maximum=1,
@@ -513,10 +534,12 @@ def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks:
step=0.1, step=0.1,
key=f"null_voice_weights_{i}", key=f"null_voice_weights_{i}",
label="声質", label="声質",
interactive=True interactive=True,
) )
if i in null_models.value: if i in null_models.value:
null_voice_weights.value=null_models.value[i]["weight"] null_voice_weights.value = null_models.value[i][
"weight"
]
null_voice_pitch_weights = gr.Slider( null_voice_pitch_weights = gr.Slider(
minimum=0, minimum=0,
maximum=1, maximum=1,
@@ -524,10 +547,12 @@ def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks:
step=0.1, step=0.1,
key=f"null_voice_pitch_weights_{i}", key=f"null_voice_pitch_weights_{i}",
label="声の高さ", label="声の高さ",
interactive=True interactive=True,
) )
if i in null_models.value: if i in null_models.value:
null_voice_pitch_weights.value=null_models.value[i]["pitch"] null_voice_pitch_weights.value = (
null_models.value[i]["pitch"]
)
null_speech_style_weights = gr.Slider( null_speech_style_weights = gr.Slider(
minimum=0, minimum=0,
maximum=1, maximum=1,
@@ -535,10 +560,12 @@ def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks:
step=0.1, step=0.1,
key=f"null_speech_style_weights_{i}", key=f"null_speech_style_weights_{i}",
label="話し方", label="話し方",
interactive=True interactive=True,
) )
if i in null_models.value: if i in null_models.value:
null_speech_style_weights.value=null_models.value[i]["style"] null_speech_style_weights.value = (
null_models.value[i]["style"]
)
null_tempo_weights = gr.Slider( null_tempo_weights = gr.Slider(
minimum=0, minimum=0,
maximum=1, maximum=1,
@@ -546,48 +573,93 @@ def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks:
step=0.1, step=0.1,
key=f"null_tempo_weights_{i}", key=f"null_tempo_weights_{i}",
label="テンポ", label="テンポ",
interactive=True interactive=True,
) )
if i in null_models.value: if i in null_models.value:
null_tempo_weights.value=null_models.value[i]["tempo"] null_tempo_weights.value = null_models.value[i][
"tempo"
]
null_model_name.change( null_model_name.change(
model_holder.update_model_files_for_gradio, model_holder.update_model_files_for_gradio,
inputs=[null_model_name], inputs=[null_model_name],
outputs=[null_model_path], outputs=[null_model_path],
) )
#null_model_name.change(model_holder.refresh, outputs=[]) # null_model_name.change(model_holder.refresh, outputs=[])
null_model_path.change(make_non_interactive, outputs=[tts_button]) null_model_path.change(
#愚直すぎるのでもう少しなんとかしたい make_non_interactive, outputs=[tts_button]
null_model_path.change(change_null_model_row,
inputs=[null_model_index, null_model_name, null_model_path,null_voice_weights,
null_voice_pitch_weights, null_speech_style_weights,null_tempo_weights,
null_models],
outputs=[null_models,force_reload_model]
) )
null_voice_weights.change(change_null_model_row, # 愚直すぎるのでもう少しなんとかしたい
inputs=[null_model_index, null_model_name, null_model_path,null_voice_weights, null_model_path.change(
null_voice_pitch_weights, null_speech_style_weights,null_tempo_weights, change_null_model_row,
null_models], inputs=[
outputs=[null_models,force_reload_model] null_model_index,
null_model_name,
null_model_path,
null_voice_weights,
null_voice_pitch_weights,
null_speech_style_weights,
null_tempo_weights,
null_models,
],
outputs=[null_models, force_reload_model],
) )
null_voice_pitch_weights.change(change_null_model_row, null_voice_weights.change(
inputs=[null_model_index, null_model_name, null_model_path,null_voice_weights, change_null_model_row,
null_voice_pitch_weights, null_speech_style_weights,null_tempo_weights, inputs=[
null_models], null_model_index,
outputs=[null_models,force_reload_model] null_model_name,
null_model_path,
null_voice_weights,
null_voice_pitch_weights,
null_speech_style_weights,
null_tempo_weights,
null_models,
],
outputs=[null_models, force_reload_model],
) )
null_speech_style_weights.change(change_null_model_row, null_voice_pitch_weights.change(
inputs=[null_model_index, null_model_name, null_model_path,null_voice_weights, change_null_model_row,
null_voice_pitch_weights, null_speech_style_weights,null_tempo_weights, inputs=[
null_models], null_model_index,
outputs=[null_models,force_reload_model] null_model_name,
null_model_path,
null_voice_weights,
null_voice_pitch_weights,
null_speech_style_weights,
null_tempo_weights,
null_models,
],
outputs=[null_models, force_reload_model],
) )
null_tempo_weights.change(change_null_model_row, null_speech_style_weights.change(
inputs=[null_model_index, null_model_name, null_model_path,null_voice_weights, change_null_model_row,
null_voice_pitch_weights, null_speech_style_weights,null_tempo_weights, inputs=[
null_models], null_model_index,
outputs=[null_models,force_reload_model] null_model_name,
null_model_path,
null_voice_weights,
null_voice_pitch_weights,
null_speech_style_weights,
null_tempo_weights,
null_models,
],
outputs=[null_models, force_reload_model],
) )
null_tempo_weights.change(
change_null_model_row,
inputs=[
null_model_index,
null_model_name,
null_model_path,
null_voice_weights,
null_voice_pitch_weights,
null_speech_style_weights,
null_tempo_weights,
null_models,
],
outputs=[null_models, force_reload_model],
)
add_btn = gr.Button("ヌルモデルを増やす") add_btn = gr.Button("ヌルモデルを増やす")
del_btn = gr.Button("ヌルモデルを減らす") del_btn = gr.Button("ヌルモデルを減らす")
add_btn.click( add_btn.click(
@@ -659,7 +731,7 @@ def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks:
pitch_scale, pitch_scale,
intonation_scale, intonation_scale,
null_models, null_models,
force_reload_model force_reload_model,
], ],
outputs=[text_output, audio_output, tone, force_reload_model], outputs=[text_output, audio_output, tone, force_reload_model],
) )
@@ -691,10 +763,12 @@ def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks:
return app return app
if __name__ == "__main__": if __name__ == "__main__":
from config import get_path_config
import torch import torch
from config import get_path_config
path_config = get_path_config() path_config = get_path_config()
assets_root = path_config.assets_root assets_root = path_config.assets_root
device = "cuda" if torch.cuda.is_available() else "cpu" device = "cuda" if torch.cuda.is_available() else "cpu"

41
style_bert_vits2/tts_model.py
View File

@@ -61,7 +61,7 @@ class TTSModel:
self.model_path: Path = model_path self.model_path: Path = model_path
self.device: str = device self.device: str = device
self.null_model_params: dict[int, dict[str,Union[float, str]]] = {} self.null_model_params: dict[int, dict[str, Union[float, str]]] = {}
# ハイパーパラメータの Pydantic モデルが直接指定された # ハイパーパラメータの Pydantic モデルが直接指定された
if isinstance(config_path, HyperParameters): if isinstance(config_path, HyperParameters):
@@ -114,7 +114,7 @@ class TTSModel:
device=self.device, device=self.device,
hps=self.hyper_parameters, hps=self.hyper_parameters,
) )
if(len(self.null_model_params.keys())==0): if len(self.null_model_params.keys()) == 0:
return return
for index, null_model in enumerate(self.null_model_params.keys()): for index, null_model in enumerate(self.null_model_params.keys()):
@@ -124,26 +124,39 @@ class TTSModel:
device=self.device, device=self.device,
hps=self.hyper_parameters, hps=self.hyper_parameters,
) )
#愚直。もっと上手い方法ありそう # 愚直。もっと上手い方法ありそう
print(str(self.null_model_params[index]["weight"])) print(str(self.null_model_params[index]["weight"]))
params = zip(self.__net_g.dec.parameters(), null_model_add.dec.parameters()) params = zip(self.__net_g.dec.parameters(), null_model_add.dec.parameters())
for v in params: for v in params:
v[0].data.add_(v[1].data,alpha=float(self.null_model_params[index]["weight"])) v[0].data.add_(
params = zip(self.__net_g.flow.parameters(), null_model_add.flow.parameters()) v[1].data, alpha=float(self.null_model_params[index]["weight"])
)
params = zip(
self.__net_g.flow.parameters(), null_model_add.flow.parameters()
)
for v in params: for v in params:
v[0].data.add_(v[1].data,alpha=float(self.null_model_params[index]["pitch"])) v[0].data.add_(
v[1].data, alpha=float(self.null_model_params[index]["pitch"])
)
params = zip(self.__net_g.enc_p.parameters(), null_model_add.enc_p.parameters()) params = zip(
self.__net_g.enc_p.parameters(), null_model_add.enc_p.parameters()
)
for v in params: for v in params:
v[0].data.add_(v[1].data,alpha=float(self.null_model_params[index]["style"])) v[0].data.add_(
#テンポはsdpとdp二つあるからとりあえずどっちも足す v[1].data, alpha=float(self.null_model_params[index]["style"])
)
# テンポはsdpとdp二つあるからとりあえずどっちも足す
params = zip(self.__net_g.sdp.parameters(), null_model_add.sdp.parameters()) params = zip(self.__net_g.sdp.parameters(), null_model_add.sdp.parameters())
for v in params: for v in params:
v[0].data.add_(v[1].data,alpha=float(self.null_model_params[index]["tempo"])) v[0].data.add_(
v[1].data, alpha=float(self.null_model_params[index]["tempo"])
)
params = zip(self.__net_g.dp.parameters(), null_model_add.dp.parameters()) params = zip(self.__net_g.dp.parameters(), null_model_add.dp.parameters())
for v in params: for v in params:
v[0].data.add_(v[1].data,alpha=float(self.null_model_params[index]["tempo"])) v[0].data.add_(
v[1].data, alpha=float(self.null_model_params[index]["tempo"])
)
def __get_style_vector(self, style_id: int, weight: float = 1.0) -> NDArray[Any]: def __get_style_vector(self, style_id: int, weight: float = 1.0) -> NDArray[Any]:
""" """
@@ -258,8 +271,8 @@ class TTSModel:
given_tone: Optional[list[int]] = None, given_tone: Optional[list[int]] = None,
pitch_scale: float = 1.0, pitch_scale: float = 1.0,
intonation_scale: float = 1.0, intonation_scale: float = 1.0,
null_model_params: dict[int,dict[str,Union[str, float]]] = {}, null_model_params: dict[int, dict[str, Union[str, float]]] = {},
force_reload_model:bool = False force_reload_model: bool = False,
) -> tuple[int, NDArray[Any]]: ) -> tuple[int, NDArray[Any]]:
""" """
テキストから音声を合成する。 テキストから音声を合成する。