tuna2134/sbv2-v2
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

推論時ヌルモデルマージ(できるけどバグバグ)

Browse Source
This commit is contained in:
liruk
2024-06-19 18:31:27 +09:00
parent 86c2a1b087
commit fb99a776ce
2 changed files with 81 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
gradio_tabs/inference.py
View File

@@ -1,6 +1,6 @@
import datetime import datetime
import json import json
from typing import Optional from typing import Optional, Any, Union
import gradio as gr import gradio as gr
@@ -190,8 +190,6 @@ voice_pitch_keys = ["flow"]
speech_style_keys = ["enc_p"] speech_style_keys = ["enc_p"]
tempo_keys = ["sdp", "dp"] tempo_keys = ["sdp", "dp"]
null_models = {}#グローバルに置いてるけどもっと良い置き場所ありそう
def make_interactive(): def make_interactive():
return gr.update(interactive=True, value="音声合成") return gr.update(interactive=True, value="音声合成")
@@ -205,7 +203,19 @@ def gr_util(item):
return (gr.update(visible=True), gr.Audio(visible=False, value=None)) return (gr.update(visible=True), gr.Audio(visible=False, value=None))
else: else:
return (gr.update(visible=False), gr.update(visible=True)) return (gr.update(visible=False), gr.update(visible=True))
def change_null_model_row(null_model_index:int, null_model_name:str, null_model_path:str,null_voice_weights:float,
null_voice_pitch_weights:float, null_speech_style_weights:float,null_tempo_weights:float,
null_models:dict[int,dict[str, Any]]):
mid_result={}
mid_result["name"]=null_model_name
mid_result["path"]=null_model_path
mid_result["weight"]=null_tempo_weights
mid_result["pitch"]=null_voice_pitch_weights
mid_result["style"]=null_speech_style_weights
mid_result["tempo"]=null_tempo_weights
null_models[null_model_index] = mid_result
result = null_models
return result
def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks: def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks:
def tts_fn( def tts_fn(
@@ -230,17 +240,14 @@ def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks:
speaker, speaker,
pitch_scale, pitch_scale,
intonation_scale, intonation_scale,
null_models:dict[int, dict[str, Union[str, float]]]
): ):
model_holder.get_model(model_name, model_path) model_holder.get_model(model_name, model_path)
assert model_holder.current_model is not None assert model_holder.current_model is not None
#null_model_paths,
#null_voice_weights, if len(null_models.keys()) > 0:
#null_voice_pitch_weights,
#null_speech_style_weights,
#null_tempo_weights
if len(null_models) > 0 and len(null_models["names"].keys()) > 0:
model_holder.get_null_models(null_models) model_holder.get_null_models(null_models)
assert len(model_holder.current_null_models) > 0 assert len(model_holder.null_models_params.keys()) > 0
wrong_tone_message = "" wrong_tone_message = ""
kata_tone: Optional[list[tuple[str, int]]] = None kata_tone: Optional[list[tuple[str, int]]] = None
@@ -337,6 +344,7 @@ def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks:
with gr.Blocks(theme=GRADIO_THEME) as app: with gr.Blocks(theme=GRADIO_THEME) as app:
gr.Markdown(initial_md) gr.Markdown(initial_md)
gr.Markdown(terms_of_use_md) gr.Markdown(terms_of_use_md)
null_models = gr.State({})
with gr.Accordion(label="使い方", open=False): with gr.Accordion(label="使い方", open=False):
gr.Markdown(how_to_md) gr.Markdown(how_to_md)
with gr.Row(): with gr.Row():
@@ -451,29 +459,24 @@ def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks:
outputs=[assist_text, assist_text_weight], outputs=[assist_text, assist_text_weight],
) )
with gr.Accordion(label="ヌルモデル", open=False): with gr.Accordion(label="ヌルモデル", open=False):
#null_voice_weights,
#null_voice_pitch_weights,
#null_speech_style_weights,
#null_tempo_weights
with gr.Row(): with gr.Row():
style_count = gr.Number(label="作るスタイルの数", value=0, step=1) null_models_count = gr.Number(label="ヌルモデルの数", value=0, step=1)
with gr.Column(variant="panel"): with gr.Column(variant="panel"):
@gr.render( @gr.render(
inputs=[ inputs=[
style_count, null_models_count,
] ]
) )
def render_style( def render_style(
style_count null_models_count:int,
): ):
name_components = {} for i in range(0, null_models_count):
path_components = {}
weight_components = {}
pitch_components = {}
style_components = {}
tempo_components = {}
for i in range(style_count):
with gr.Row(): with gr.Row():
null_model_index = gr.Number(
value=i,
key=f"null_model_index_{i}",
visible=False
)
null_model_name = gr.Dropdown( null_model_name = gr.Dropdown(
label="モデル一覧", label="モデル一覧",
choices=model_names, choices=model_names,
@@ -530,31 +533,51 @@ def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks:
outputs=[null_model_path], outputs=[null_model_path],
) )
null_model_path.change(make_non_interactive, outputs=[tts_button]) null_model_path.change(make_non_interactive, outputs=[tts_button])
#もっといい方法ありそう #愚直すぎるのでもう少しなんとかしたい
name_components[str(i)]=(null_model_name) null_model_path.change(change_null_model_row,
path_components[str(i)]=(null_model_path) inputs=[null_model_index, null_model_name, null_model_path,null_voice_weights,
weight_components[str(i)]=(null_voice_weights) null_voice_pitch_weights, null_speech_style_weights,null_tempo_weights,
pitch_components[str(i)]=(null_voice_pitch_weights) null_models],
style_components[str(i)]=(null_speech_style_weights) outputs=[null_models]
tempo_components[str(i)]=(null_tempo_weights) )
null_models["names"]=(name_components) null_voice_weights.change(change_null_model_row,
null_models["paths"]=(path_components) inputs=[null_model_index, null_model_name, null_model_path,null_voice_weights,
null_models["weights"]=(weight_components) null_voice_pitch_weights, null_speech_style_weights,null_tempo_weights,
null_models["pitchs"]=(pitch_components) null_models],
null_models["styles"]=(style_components) outputs=[null_models]
null_models["tempos"]=(tempo_components) )
null_voice_pitch_weights.change(change_null_model_row,
inputs=[null_model_index, null_model_name, null_model_path,null_voice_weights,
null_voice_pitch_weights, null_speech_style_weights,null_tempo_weights,
null_models],
outputs=[null_models]
)
null_speech_style_weights.change(change_null_model_row,
inputs=[null_model_index, null_model_name, null_model_path,null_voice_weights,
null_voice_pitch_weights, null_speech_style_weights,null_tempo_weights,
null_models],
outputs=[null_models]
)
null_tempo_weights.change(change_null_model_row,
inputs=[null_model_index, null_model_name, null_model_path,null_voice_weights,
null_voice_pitch_weights, null_speech_style_weights,null_tempo_weights,
null_models],
outputs=[null_models]
)
if null_models_count < len(null_models.value.keys()):
for i in range(null_models_count ,len(null_models.value.keys())):
_ = null_models.value.pop(i, None)
add_btn = gr.Button("ヌルモデルを増やす") add_btn = gr.Button("ヌルモデルを増やす")
del_btn = gr.Button("ヌルモデルを減らす") del_btn = gr.Button("ヌルモデルを減らす")
add_btn.click( add_btn.click(
lambda x: x + 1, lambda x: x + 1,
inputs=[style_count], inputs=[null_models_count],
outputs=[style_count], outputs=[null_models_count],
) )
del_btn.click( del_btn.click(
lambda x: x - 1 if x > 0 else 0, lambda x: x - 1 if x > 0 else 0,
inputs=[style_count], inputs=[null_models_count],
outputs=[style_count], outputs=[null_models_count],
) )
with gr.Column(): with gr.Column():
@@ -614,6 +637,7 @@ def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks:
speaker, speaker,
pitch_scale, pitch_scale,
intonation_scale, intonation_scale,
null_models
], ],
outputs=[text_output, audio_output, tone], outputs=[text_output, audio_output, tone],
) )

48
style_bert_vits2/tts_model.py
View File

@@ -61,7 +61,7 @@ class TTSModel:
self.model_path: Path = model_path self.model_path: Path = model_path
self.device: str = device self.device: str = device
self.null_model_params: dict[str, dict[str,Any]] = {} self.null_model_params: dict[int, dict[str,Union[float, str]]] = {}
# ハイパーパラメータの Pydantic モデルが直接指定された # ハイパーパラメータの Pydantic モデルが直接指定された
if isinstance(config_path, HyperParameters): if isinstance(config_path, HyperParameters):
@@ -117,32 +117,32 @@ class TTSModel:
if(len(self.null_model_params.keys())==0): if(len(self.null_model_params.keys())==0):
return return
for index, null_model in enumerate(self.null_model_params["names"].keys()): for index, null_model in enumerate(self.null_model_params.keys()):
null_model_add = get_net_g( null_model_add = get_net_g(
model_path=str(self.null_model_params["paths"][str(index)].value), model_path=str(self.null_model_params[index]["path"]),
version=self.hyper_parameters.version, version=self.hyper_parameters.version,
device=self.device, device=self.device,
hps=self.hyper_parameters, hps=self.hyper_parameters,
) )
#愚直。もっと上手い方法ありそう #愚直。もっと上手い方法ありそう
print(str(self.null_model_params[index]["weight"]))
params = zip(self.__net_g.dec.parameters(), null_model_add.dec.parameters()) params = zip(self.__net_g.dec.parameters(), null_model_add.dec.parameters())
for v in params: for v in params:
v[0].data.add(v[1].data,alpha=self.null_model_params["weights"][str(index)].value) v[0].data.add_(v[1].data,alpha=float(self.null_model_params[index]["weight"]))
params = zip(self.__net_g.flow.parameters(), null_model_add.flow.parameters()) params = zip(self.__net_g.flow.parameters(), null_model_add.flow.parameters())
for v in params: for v in params:
v[0].data.add(v[1].data,alpha=self.null_model_params["pitchs"][str(index)].value) v[0].data.add_(v[1].data,alpha=float(self.null_model_params[index]["pitch"]))
params = zip(self.__net_g.enc_p.parameters(), null_model_add.enc_p.parameters()) params = zip(self.__net_g.enc_p.parameters(), null_model_add.enc_p.parameters())
for v in params: for v in params:
v[0].data.add(v[1].data,alpha=self.null_model_params["styles"][str(index)].value) v[0].data.add_(v[1].data,alpha=float(self.null_model_params[index]["style"]))
#テンポはsdpとdp二つあるからとりあえずどっちも足す #テンポはsdpとdp二つあるからとりあえずどっちも足す
params = zip(self.__net_g.sdp.parameters(), null_model_add.sdp.parameters()) params = zip(self.__net_g.sdp.parameters(), null_model_add.sdp.parameters())
for v in params: for v in params:
v[0].data.add(v[1].data,alpha=self.null_model_params["tempos"][str(index)].value) v[0].data.add_(v[1].data,alpha=float(self.null_model_params[index]["tempo"]))
params = zip(self.__net_g.dp.parameters(), null_model_add.dp.parameters()) params = zip(self.__net_g.dp.parameters(), null_model_add.dp.parameters())
for v in params: for v in params:
v[0].data.add(v[1].data,alpha=self.null_model_params["tempos"][str(index)].value) v[0].data.add_(v[1].data,alpha=float(self.null_model_params[index]["tempo"]))
def __get_style_vector(self, style_id: int, weight: float = 1.0) -> NDArray[Any]: def __get_style_vector(self, style_id: int, weight: float = 1.0) -> NDArray[Any]:
@@ -258,7 +258,7 @@ class TTSModel:
given_tone: Optional[list[int]] = None, given_tone: Optional[list[int]] = None,
pitch_scale: float = 1.0, pitch_scale: float = 1.0,
intonation_scale: float = 1.0, intonation_scale: float = 1.0,
null_model_params: dict[str,dict[str,Any]] = {} null_model_params: dict[int,dict[str,Union[str, float]]] = {}
) -> tuple[int, NDArray[Any]]: ) -> tuple[int, NDArray[Any]]:
""" """
テキストから音声を合成する。 テキストから音声を合成する。
@@ -283,7 +283,7 @@ class TTSModel:
given_tone (Optional[list[int]], optional): アクセントのトーンのリスト. Defaults to None. given_tone (Optional[list[int]], optional): アクセントのトーンのリスト. Defaults to None.
pitch_scale (float, optional): ピッチの高さ (1.0 から変更すると若干音質が低下する). Defaults to 1.0. pitch_scale (float, optional): ピッチの高さ (1.0 から変更すると若干音質が低下する). Defaults to 1.0.
intonation_scale (float, optional): 抑揚の平均からの変化幅 (1.0 から変更すると若干音質が低下する). Defaults to 1.0. intonation_scale (float, optional): 抑揚の平均からの変化幅 (1.0 から変更すると若干音質が低下する). Defaults to 1.0.
null_model_params(dict[str,dict[str,gr.Component],optional):推論時に使用するヌルモデルの名前、重みのdictが入ったdict。 null_model_params(dict[int,dict[str,Union[str,float]],optional):推論時に使用するヌルモデルの名前、適用割合のdictが入ったdict。
Returns: Returns:
tuple[int, NDArray[Any]]: サンプリングレートと音声データ (16bit PCM) tuple[int, NDArray[Any]]: サンプリングレートと音声データ (16bit PCM)
""" """
@@ -407,8 +407,7 @@ class TTSModelHolder:
self.device: str = device self.device: str = device
self.model_files_dict: dict[str, list[Path]] = {} self.model_files_dict: dict[str, list[Path]] = {}
self.current_model: Optional[TTSModel] = None self.current_model: Optional[TTSModel] = None
self.current_null_models: dict[str,TTSModel] = {} self.null_models_params: dict[int,dict[str,Union[str, float]]] = {}
self.null_models_params: dict[str,dict[str,Any]] = {}
self.model_names: list[str] = [] self.model_names: list[str] = []
self.models_info: list[TTSModelInfo] = [] self.models_info: list[TTSModelInfo] = []
self.refresh() self.refresh()
@@ -421,7 +420,6 @@ class TTSModelHolder:
self.model_files_dict = {} self.model_files_dict = {}
self.model_names = [] self.model_names = []
self.current_model = None self.current_model = None
self.current_null_models = {}
self.null_models_params = {} self.null_models_params = {}
self.models_info = [] self.models_info = []
@@ -484,29 +482,13 @@ class TTSModelHolder:
) )
return self.current_model return self.current_model
def get_null_models(self, null_model_index:dict[str,dict[str,Any]]) -> dict[str, TTSModel]: def get_null_models(self, null_model_index:dict[int,dict[str,Union[str, float]]]) -> dict[int, dict[str, Union[str, float]]]:
""" """
get_modelをヌルモデル用に改変。複数まとめて使うかも知れないのでdictで戻す get_modelをヌルモデル用に改変。複数まとめて使うかも知れないのでdictで戻す
""" """
self.current_null_models = {}
self.null_models_params = null_model_index self.null_models_params = null_model_index
for index, value in enumerate(null_model_index["names"]): return self.null_models_params
model_path = Path(null_model_index["paths"][str(index)].value)
model_name = null_model_index["names"][str(index)].value
if model_name not in self.model_files_dict:
raise ValueError(f"Model `{model_name}` is not found")
if model_path not in self.model_files_dict[model_name]:
raise ValueError(f"Model file `{model_path}` is not found")
if len(self.current_null_models) == 0 or model_path not in self.current_null_models.keys():
self.current_null_models[model_name] = TTSModel(
model_path=model_path,
config_path=self.root_dir / model_name / "config.json",
style_vec_path=self.root_dir / model_name / "style_vectors.npy",
device=self.device,
)
return self.current_null_models
def get_model_for_gradio(self, model_name: str, model_path_str: str): def get_model_for_gradio(self, model_name: str, model_path_str: str):
import gradio as gr import gradio as gr
@@ -536,8 +518,6 @@ class TTSModelHolder:
) )
speakers = list(self.current_model.spk2id.keys()) speakers = list(self.current_model.spk2id.keys())
styles = list(self.current_model.style2id.keys()) styles = list(self.current_model.style2id.keys())
#if len(null_models.keys())!=0:
# self.get_null_models(null_models)
return ( return (
gr.Dropdown(choices=styles, value=styles[0]), # type: ignore gr.Dropdown(choices=styles, value=styles[0]), # type: ignore
gr.Button(interactive=True, value="音声合成"), gr.Button(interactive=True, value="音声合成"),