diff --git a/gradio_tabs/inference.py b/gradio_tabs/inference.py index 53393be..c7a8d5e 100644 --- a/gradio_tabs/inference.py +++ b/gradio_tabs/inference.py @@ -185,7 +185,12 @@ style_md = f""" - どのくらいに強さがいいかはモデルやスタイルによって異なるようです。 - 音声ファイルを入力する場合は、学習データと似た声音の話者（特に同じ性別）でないとよい効果が出ないかもしれません。 """ +voice_keys = ["dec"] +voice_pitch_keys = ["flow"] +speech_style_keys = ["enc_p"] +tempo_keys = ["sdp", "dp"] +null_models = {}#グローバルに置いてるけどもっと良い置き場所ありそう def make_interactive(): return gr.update(interactive=True, value="音声合成") @@ -228,6 +233,14 @@ def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks: ): model_holder.get_model(model_name, model_path) assert model_holder.current_model is not None + #null_model_paths, + #null_voice_weights, + #null_voice_pitch_weights, + #null_speech_style_weights, + #null_tempo_weights + if len(null_models) > 0 and len(null_models["names"].keys()) > 0: + model_holder.get_null_models(null_models) + assert len(model_holder.current_null_models) > 0 wrong_tone_message = "" kata_tone: Optional[list[tuple[str, int]]] = None @@ -282,6 +295,7 @@ def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks: speaker_id=speaker_id, pitch_scale=pitch_scale, intonation_scale=intonation_scale, + null_model_params = null_models ) except InvalidToneError as e: logger.error(f"Tone error: {e}") @@ -436,6 +450,113 @@ def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks: inputs=[use_assist_text], outputs=[assist_text, assist_text_weight], ) + with gr.Accordion(label="ヌルモデル", open=False): + #null_voice_weights, + #null_voice_pitch_weights, + #null_speech_style_weights, + #null_tempo_weights + with gr.Row(): + style_count = gr.Number(label="作るスタイルの数", value=0, step=1) + with gr.Column(variant="panel"): + @gr.render( + inputs=[ + style_count, + ] + ) + def render_style( + style_count + ): + name_components = {} + path_components = {} + weight_components = {} + pitch_components = {} + style_components = {} + tempo_components = {} + for i in range(style_count): + with gr.Row(): + null_model_name = gr.Dropdown( + label="モデル一覧", + choices=model_names, + key=f"null_model_name_{i}", + value=model_names[initial_id], + interactive=True + ) + null_model_path = gr.Dropdown( + label="モデルファイル", + choices=initial_pth_files, + key=f"null_model_path_{i}", + value=initial_pth_files[0], + interactive=True + ) + null_voice_weights = gr.Slider( + minimum=0, + maximum=1, + value=1, + step=0.1, + key=f"null_voice_weights_{i}", + label="声質", + interactive=True + ) + null_voice_pitch_weights = gr.Slider( + minimum=0, + maximum=1, + value=1, + step=0.1, + key=f"null_voice_pitch_weights_{i}", + label="声の高さ", + interactive=True + ) + null_speech_style_weights = gr.Slider( + minimum=0, + maximum=1, + value=1, + step=0.1, + key=f"null_speech_style_weights_{i}", + label="話し方", + interactive=True + ) + null_tempo_weights = gr.Slider( + minimum=0, + maximum=1, + value=1, + step=0.1, + key=f"null_tempo_weights_{i}", + label="テンポ", + interactive=True + ) + null_model_name.change( + model_holder.update_model_files_for_gradio, + inputs=[null_model_name], + outputs=[null_model_path], + ) + null_model_path.change(make_non_interactive, outputs=[tts_button]) + #もっといい方法ありそう + name_components[str(i)]=(null_model_name) + path_components[str(i)]=(null_model_path) + weight_components[str(i)]=(null_voice_weights) + pitch_components[str(i)]=(null_voice_pitch_weights) + style_components[str(i)]=(null_speech_style_weights) + tempo_components[str(i)]=(null_tempo_weights) + null_models["names"]=(name_components) + null_models["paths"]=(path_components) + null_models["weights"]=(weight_components) + null_models["pitchs"]=(pitch_components) + null_models["styles"]=(style_components) + null_models["tempos"]=(tempo_components) + + add_btn = gr.Button("ヌルモデルを増やす") + del_btn = gr.Button("ヌルモデルを減らす") + add_btn.click( + lambda x: x + 1, + inputs=[style_count], + outputs=[style_count], + ) + del_btn.click( + lambda x: x - 1 if x > 0 else 0, + inputs=[style_count], + outputs=[style_count], + ) + with gr.Column(): with gr.Accordion("スタイルについて詳細", open=False): gr.Markdown(style_md) @@ -524,7 +645,6 @@ def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks: return app - if __name__ == "__main__": from config import get_path_config import torch diff --git a/style_bert_vits2/tts_model.py b/style_bert_vits2/tts_model.py index 6df8394..bb38b3a 100644 --- a/style_bert_vits2/tts_model.py +++ b/style_bert_vits2/tts_model.py @@ -61,6 +61,7 @@ class TTSModel: self.model_path: Path = model_path self.device: str = device + self.null_model_params: dict[str, dict[str,Any]] = {} # ハイパーパラメータの Pydantic モデルが直接指定された if isinstance(config_path, HyperParameters): @@ -113,6 +114,36 @@ class TTSModel: device=self.device, hps=self.hyper_parameters, ) + if(len(self.null_model_params.keys())==0): + return + + for index, null_model in enumerate(self.null_model_params["names"].keys()): + null_model_add = get_net_g( + model_path=str(self.null_model_params["paths"][str(index)].value), + version=self.hyper_parameters.version, + device=self.device, + hps=self.hyper_parameters, + ) + #愚直。もっと上手い方法ありそう + params = zip(self.__net_g.dec.parameters(), null_model_add.dec.parameters()) + for v in params: + v[0].data.add(v[1].data,alpha=self.null_model_params["weights"][str(index)].value) + + params = zip(self.__net_g.flow.parameters(), null_model_add.flow.parameters()) + for v in params: + v[0].data.add(v[1].data,alpha=self.null_model_params["pitchs"][str(index)].value) + + params = zip(self.__net_g.enc_p.parameters(), null_model_add.enc_p.parameters()) + for v in params: + v[0].data.add(v[1].data,alpha=self.null_model_params["styles"][str(index)].value) + #テンポはsdpとdp二つあるからとりあえずどっちも足す + params = zip(self.__net_g.sdp.parameters(), null_model_add.sdp.parameters()) + for v in params: + v[0].data.add(v[1].data,alpha=self.null_model_params["tempos"][str(index)].value) + params = zip(self.__net_g.dp.parameters(), null_model_add.dp.parameters()) + for v in params: + v[0].data.add(v[1].data,alpha=self.null_model_params["tempos"][str(index)].value) + def __get_style_vector(self, style_id: int, weight: float = 1.0) -> NDArray[Any]: """ @@ -227,6 +258,7 @@ class TTSModel: given_tone: Optional[list[int]] = None, pitch_scale: float = 1.0, intonation_scale: float = 1.0, + null_model_params: dict[str,dict[str,Any]] = {} ) -> tuple[int, NDArray[Any]]: """ テキストから音声を合成する。 @@ -251,7 +283,7 @@ class TTSModel: given_tone (Optional[list[int]], optional): アクセントのトーンのリスト. Defaults to None. pitch_scale (float, optional): ピッチの高さ (1.0 から変更すると若干音質が低下する). Defaults to 1.0. intonation_scale (float, optional): 抑揚の平均からの変化幅 (1.0 から変更すると若干音質が低下する). Defaults to 1.0. - + null_model_params(dict[str,dict[str,gr.Component],optional):推論時に使用するヌルモデルの名前、重みのdictが入ったdict。 Returns: tuple[int, NDArray[Any]]: サンプリングレートと音声データ (16bit PCM) """ @@ -265,7 +297,10 @@ class TTSModel: reference_audio_path = None if assist_text == "" or not use_assist_text: assist_text = None - + if null_model_params is not {}: + self.null_model_params = null_model_params + else: + self.null_model_params = {} if self.__net_g is None: self.load() assert self.__net_g is not None @@ -372,6 +407,8 @@ class TTSModelHolder: self.device: str = device self.model_files_dict: dict[str, list[Path]] = {} self.current_model: Optional[TTSModel] = None + self.current_null_models: dict[str,TTSModel] = {} + self.null_models_params: dict[str,dict[str,Any]] = {} self.model_names: list[str] = [] self.models_info: list[TTSModelInfo] = [] self.refresh() @@ -384,6 +421,8 @@ class TTSModelHolder: self.model_files_dict = {} self.model_names = [] self.current_model = None + self.current_null_models = {} + self.null_models_params = {} self.models_info = [] model_dirs = [d for d in self.root_dir.iterdir() if d.is_dir()] @@ -445,6 +484,29 @@ class TTSModelHolder: ) return self.current_model + def get_null_models(self, null_model_index:dict[str,dict[str,Any]]) -> dict[str, TTSModel]: + """ + get_modelをヌルモデル用に改変。複数まとめて使うかも知れないのでdictで戻す + + """ + self.current_null_models = {} + self.null_models_params = null_model_index + for index, value in enumerate(null_model_index["names"]): + model_path = Path(null_model_index["paths"][str(index)].value) + model_name = null_model_index["names"][str(index)].value + if model_name not in self.model_files_dict: + raise ValueError(f"Model `{model_name}` is not found") + if model_path not in self.model_files_dict[model_name]: + raise ValueError(f"Model file `{model_path}` is not found") + + if len(self.current_null_models) == 0 or model_path not in self.current_null_models.keys(): + self.current_null_models[model_name] = TTSModel( + model_path=model_path, + config_path=self.root_dir / model_name / "config.json", + style_vec_path=self.root_dir / model_name / "style_vectors.npy", + device=self.device, + ) + return self.current_null_models def get_model_for_gradio(self, model_name: str, model_path_str: str): import gradio as gr @@ -474,6 +536,8 @@ class TTSModelHolder: ) speakers = list(self.current_model.spk2id.keys()) styles = list(self.current_model.style2id.keys()) + #if len(null_models.keys())!=0: + # self.get_null_models(null_models) return ( gr.Dropdown(choices=styles, value=styles[0]), # type: ignore gr.Button(interactive=True, value="音声合成"),