From fb99a776ce17ef97a736575647961afbf5d0549e Mon Sep 17 00:00:00 2001 From: liruk <34234523+liruk@users.noreply.github.com> Date: Wed, 19 Jun 2024 18:31:27 +0900 Subject: [PATCH] =?UTF-8?q?=E6=8E=A8=E8=AB=96=E6=99=82=E3=83=8C=E3=83=AB?= =?UTF-8?q?=E3=83=A2=E3=83=87=E3=83=AB=E3=83=9E=E3=83=BC=E3=82=B8(?= =?UTF-8?q?=E3=81=A7=E3=81=8D=E3=82=8B=E3=81=91=E3=81=A9=E3=83=90=E3=82=B0?= =?UTF-8?q?=E3=83=90=E3=82=B0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- gradio_tabs/inference.py | 110 +++++++++++++++++++++------------- style_bert_vits2/tts_model.py | 48 +++++---------- 2 files changed, 81 insertions(+), 77 deletions(-) diff --git a/gradio_tabs/inference.py b/gradio_tabs/inference.py index c7a8d5e..38c2af6 100644 --- a/gradio_tabs/inference.py +++ b/gradio_tabs/inference.py @@ -1,6 +1,6 @@ import datetime import json -from typing import Optional +from typing import Optional, Any, Union import gradio as gr @@ -190,8 +190,6 @@ voice_pitch_keys = ["flow"] speech_style_keys = ["enc_p"] tempo_keys = ["sdp", "dp"] -null_models = {}#グローバルに置いてるけどもっと良い置き場所ありそう - def make_interactive(): return gr.update(interactive=True, value="音声合成") @@ -205,7 +203,19 @@ def gr_util(item): return (gr.update(visible=True), gr.Audio(visible=False, value=None)) else: return (gr.update(visible=False), gr.update(visible=True)) - +def change_null_model_row(null_model_index:int, null_model_name:str, null_model_path:str,null_voice_weights:float, + null_voice_pitch_weights:float, null_speech_style_weights:float,null_tempo_weights:float, + null_models:dict[int,dict[str, Any]]): + mid_result={} + mid_result["name"]=null_model_name + mid_result["path"]=null_model_path + mid_result["weight"]=null_tempo_weights + mid_result["pitch"]=null_voice_pitch_weights + mid_result["style"]=null_speech_style_weights + mid_result["tempo"]=null_tempo_weights + null_models[null_model_index] = mid_result + result = null_models + return result def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks: def tts_fn( @@ -230,17 +240,14 @@ def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks: speaker, pitch_scale, intonation_scale, + null_models:dict[int, dict[str, Union[str, float]]] ): model_holder.get_model(model_name, model_path) assert model_holder.current_model is not None - #null_model_paths, - #null_voice_weights, - #null_voice_pitch_weights, - #null_speech_style_weights, - #null_tempo_weights - if len(null_models) > 0 and len(null_models["names"].keys()) > 0: + + if len(null_models.keys()) > 0: model_holder.get_null_models(null_models) - assert len(model_holder.current_null_models) > 0 + assert len(model_holder.null_models_params.keys()) > 0 wrong_tone_message = "" kata_tone: Optional[list[tuple[str, int]]] = None @@ -337,6 +344,7 @@ def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks: with gr.Blocks(theme=GRADIO_THEME) as app: gr.Markdown(initial_md) gr.Markdown(terms_of_use_md) + null_models = gr.State({}) with gr.Accordion(label="使い方", open=False): gr.Markdown(how_to_md) with gr.Row(): @@ -451,29 +459,24 @@ def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks: outputs=[assist_text, assist_text_weight], ) with gr.Accordion(label="ヌルモデル", open=False): - #null_voice_weights, - #null_voice_pitch_weights, - #null_speech_style_weights, - #null_tempo_weights with gr.Row(): - style_count = gr.Number(label="作るスタイルの数", value=0, step=1) + null_models_count = gr.Number(label="ヌルモデルの数", value=0, step=1) with gr.Column(variant="panel"): @gr.render( inputs=[ - style_count, + null_models_count, ] ) def render_style( - style_count + null_models_count:int, ): - name_components = {} - path_components = {} - weight_components = {} - pitch_components = {} - style_components = {} - tempo_components = {} - for i in range(style_count): + for i in range(0, null_models_count): with gr.Row(): + null_model_index = gr.Number( + value=i, + key=f"null_model_index_{i}", + visible=False + ) null_model_name = gr.Dropdown( label="モデル一覧", choices=model_names, @@ -530,31 +533,51 @@ def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks: outputs=[null_model_path], ) null_model_path.change(make_non_interactive, outputs=[tts_button]) - #もっといい方法ありそう - name_components[str(i)]=(null_model_name) - path_components[str(i)]=(null_model_path) - weight_components[str(i)]=(null_voice_weights) - pitch_components[str(i)]=(null_voice_pitch_weights) - style_components[str(i)]=(null_speech_style_weights) - tempo_components[str(i)]=(null_tempo_weights) - null_models["names"]=(name_components) - null_models["paths"]=(path_components) - null_models["weights"]=(weight_components) - null_models["pitchs"]=(pitch_components) - null_models["styles"]=(style_components) - null_models["tempos"]=(tempo_components) - + #愚直すぎるのでもう少しなんとかしたい + null_model_path.change(change_null_model_row, + inputs=[null_model_index, null_model_name, null_model_path,null_voice_weights, + null_voice_pitch_weights, null_speech_style_weights,null_tempo_weights, + null_models], + outputs=[null_models] + ) + null_voice_weights.change(change_null_model_row, + inputs=[null_model_index, null_model_name, null_model_path,null_voice_weights, + null_voice_pitch_weights, null_speech_style_weights,null_tempo_weights, + null_models], + outputs=[null_models] + ) + null_voice_pitch_weights.change(change_null_model_row, + inputs=[null_model_index, null_model_name, null_model_path,null_voice_weights, + null_voice_pitch_weights, null_speech_style_weights,null_tempo_weights, + null_models], + outputs=[null_models] + ) + null_speech_style_weights.change(change_null_model_row, + inputs=[null_model_index, null_model_name, null_model_path,null_voice_weights, + null_voice_pitch_weights, null_speech_style_weights,null_tempo_weights, + null_models], + outputs=[null_models] + ) + null_tempo_weights.change(change_null_model_row, + inputs=[null_model_index, null_model_name, null_model_path,null_voice_weights, + null_voice_pitch_weights, null_speech_style_weights,null_tempo_weights, + null_models], + outputs=[null_models] + ) + if null_models_count < len(null_models.value.keys()): + for i in range(null_models_count ,len(null_models.value.keys())): + _ = null_models.value.pop(i, None) add_btn = gr.Button("ヌルモデルを増やす") del_btn = gr.Button("ヌルモデルを減らす") add_btn.click( lambda x: x + 1, - inputs=[style_count], - outputs=[style_count], + inputs=[null_models_count], + outputs=[null_models_count], ) del_btn.click( lambda x: x - 1 if x > 0 else 0, - inputs=[style_count], - outputs=[style_count], + inputs=[null_models_count], + outputs=[null_models_count], ) with gr.Column(): @@ -614,6 +637,7 @@ def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks: speaker, pitch_scale, intonation_scale, + null_models ], outputs=[text_output, audio_output, tone], ) diff --git a/style_bert_vits2/tts_model.py b/style_bert_vits2/tts_model.py index bb38b3a..f8a387b 100644 --- a/style_bert_vits2/tts_model.py +++ b/style_bert_vits2/tts_model.py @@ -61,7 +61,7 @@ class TTSModel: self.model_path: Path = model_path self.device: str = device - self.null_model_params: dict[str, dict[str,Any]] = {} + self.null_model_params: dict[int, dict[str,Union[float, str]]] = {} # ハイパーパラメータの Pydantic モデルが直接指定された if isinstance(config_path, HyperParameters): @@ -117,32 +117,32 @@ class TTSModel: if(len(self.null_model_params.keys())==0): return - for index, null_model in enumerate(self.null_model_params["names"].keys()): + for index, null_model in enumerate(self.null_model_params.keys()): null_model_add = get_net_g( - model_path=str(self.null_model_params["paths"][str(index)].value), + model_path=str(self.null_model_params[index]["path"]), version=self.hyper_parameters.version, device=self.device, hps=self.hyper_parameters, ) #愚直。もっと上手い方法ありそう + print(str(self.null_model_params[index]["weight"])) params = zip(self.__net_g.dec.parameters(), null_model_add.dec.parameters()) for v in params: - v[0].data.add(v[1].data,alpha=self.null_model_params["weights"][str(index)].value) - + v[0].data.add_(v[1].data,alpha=float(self.null_model_params[index]["weight"])) params = zip(self.__net_g.flow.parameters(), null_model_add.flow.parameters()) for v in params: - v[0].data.add(v[1].data,alpha=self.null_model_params["pitchs"][str(index)].value) + v[0].data.add_(v[1].data,alpha=float(self.null_model_params[index]["pitch"])) params = zip(self.__net_g.enc_p.parameters(), null_model_add.enc_p.parameters()) for v in params: - v[0].data.add(v[1].data,alpha=self.null_model_params["styles"][str(index)].value) + v[0].data.add_(v[1].data,alpha=float(self.null_model_params[index]["style"])) #テンポはsdpとdp二つあるからとりあえずどっちも足す params = zip(self.__net_g.sdp.parameters(), null_model_add.sdp.parameters()) for v in params: - v[0].data.add(v[1].data,alpha=self.null_model_params["tempos"][str(index)].value) + v[0].data.add_(v[1].data,alpha=float(self.null_model_params[index]["tempo"])) params = zip(self.__net_g.dp.parameters(), null_model_add.dp.parameters()) for v in params: - v[0].data.add(v[1].data,alpha=self.null_model_params["tempos"][str(index)].value) + v[0].data.add_(v[1].data,alpha=float(self.null_model_params[index]["tempo"])) def __get_style_vector(self, style_id: int, weight: float = 1.0) -> NDArray[Any]: @@ -258,7 +258,7 @@ class TTSModel: given_tone: Optional[list[int]] = None, pitch_scale: float = 1.0, intonation_scale: float = 1.0, - null_model_params: dict[str,dict[str,Any]] = {} + null_model_params: dict[int,dict[str,Union[str, float]]] = {} ) -> tuple[int, NDArray[Any]]: """ テキストから音声を合成する。 @@ -283,7 +283,7 @@ class TTSModel: given_tone (Optional[list[int]], optional): アクセントのトーンのリスト. Defaults to None. pitch_scale (float, optional): ピッチの高さ (1.0 から変更すると若干音質が低下する). Defaults to 1.0. intonation_scale (float, optional): 抑揚の平均からの変化幅 (1.0 から変更すると若干音質が低下する). Defaults to 1.0. - null_model_params(dict[str,dict[str,gr.Component],optional):推論時に使用するヌルモデルの名前、重みのdictが入ったdict。 + null_model_params(dict[int,dict[str,Union[str,float]],optional):推論時に使用するヌルモデルの名前、適用割合のdictが入ったdict。 Returns: tuple[int, NDArray[Any]]: サンプリングレートと音声データ (16bit PCM) """ @@ -407,8 +407,7 @@ class TTSModelHolder: self.device: str = device self.model_files_dict: dict[str, list[Path]] = {} self.current_model: Optional[TTSModel] = None - self.current_null_models: dict[str,TTSModel] = {} - self.null_models_params: dict[str,dict[str,Any]] = {} + self.null_models_params: dict[int,dict[str,Union[str, float]]] = {} self.model_names: list[str] = [] self.models_info: list[TTSModelInfo] = [] self.refresh() @@ -421,7 +420,6 @@ class TTSModelHolder: self.model_files_dict = {} self.model_names = [] self.current_model = None - self.current_null_models = {} self.null_models_params = {} self.models_info = [] @@ -484,29 +482,13 @@ class TTSModelHolder: ) return self.current_model - def get_null_models(self, null_model_index:dict[str,dict[str,Any]]) -> dict[str, TTSModel]: + def get_null_models(self, null_model_index:dict[int,dict[str,Union[str, float]]]) -> dict[int, dict[str, Union[str, float]]]: """ get_modelをヌルモデル用に改変。複数まとめて使うかも知れないのでdictで戻す """ - self.current_null_models = {} self.null_models_params = null_model_index - for index, value in enumerate(null_model_index["names"]): - model_path = Path(null_model_index["paths"][str(index)].value) - model_name = null_model_index["names"][str(index)].value - if model_name not in self.model_files_dict: - raise ValueError(f"Model `{model_name}` is not found") - if model_path not in self.model_files_dict[model_name]: - raise ValueError(f"Model file `{model_path}` is not found") - - if len(self.current_null_models) == 0 or model_path not in self.current_null_models.keys(): - self.current_null_models[model_name] = TTSModel( - model_path=model_path, - config_path=self.root_dir / model_name / "config.json", - style_vec_path=self.root_dir / model_name / "style_vectors.npy", - device=self.device, - ) - return self.current_null_models + return self.null_models_params def get_model_for_gradio(self, model_name: str, model_path_str: str): import gradio as gr @@ -536,8 +518,6 @@ class TTSModelHolder: ) speakers = list(self.current_model.spk2id.keys()) styles = list(self.current_model.style2id.keys()) - #if len(null_models.keys())!=0: - # self.get_null_models(null_models) return ( gr.Dropdown(choices=styles, value=styles[0]), # type: ignore gr.Button(interactive=True, value="音声合成"),