From d2fd378b565239fd272520a011f2df84d1916a3b Mon Sep 17 00:00:00 2001 From: tsukumi Date: Sun, 10 Mar 2024 03:47:17 +0000 Subject: [PATCH] Refactor: rename Model / ModelHolder to TTSModel / TTSModelHolder for clarification and add comments to each method --- app.py | 4 +- server_editor.py | 14 ++-- server_fastapi.py | 14 ++-- speech_mos.py | 4 +- style_bert_vits2/tts_model.py | 133 +++++++++++++++++++++++++++------- webui/inference.py | 12 +-- webui/merge.py | 8 +- 7 files changed, 133 insertions(+), 56 deletions(-) diff --git a/app.py b/app.py index eaf07b7..b4fbc3e 100644 --- a/app.py +++ b/app.py @@ -6,7 +6,7 @@ import torch import yaml from style_bert_vits2.constants import GRADIO_THEME, VERSION -from style_bert_vits2.tts_model import ModelHolder +from style_bert_vits2.tts_model import TTSModelHolder from webui import ( create_dataset_app, create_inference_app, @@ -34,7 +34,7 @@ device = args.device if device == "cuda" and not torch.cuda.is_available(): device = "cpu" -model_holder = ModelHolder(Path(assets_root), device) +model_holder = TTSModelHolder(Path(assets_root), device) with gr.Blocks(theme=GRADIO_THEME) as app: gr.Markdown(f"# Style-Bert-VITS2 WebUI (version {VERSION})") diff --git a/server_editor.py b/server_editor.py index 3c0d2c8..aa85ef0 100644 --- a/server_editor.py +++ b/server_editor.py @@ -52,7 +52,7 @@ from style_bert_vits2.nlp.japanese.user_dict import ( rewrite_word, update_dict, ) -from style_bert_vits2.tts_model import ModelHolder +from style_bert_vits2.tts_model import TTSModelHolder # ---フロントエンド部分に関する処理--- @@ -198,7 +198,7 @@ if device == "cuda" and not torch.cuda.is_available(): model_dir = Path(args.model_dir) port = int(args.port) -model_holder = ModelHolder(model_dir, device) +model_holder = TTSModelHolder(model_dir, device) if len(model_holder.model_names) == 0: logger.error(f"Models not found in {model_dir}.") sys.exit(1) @@ -283,7 +283,7 @@ def synthesis(request: SynthesisRequest): detail=f"1行の文字数は{args.line_length}文字以下にしてください。", ) try: - model = model_holder.load_model( + model = model_holder.get_model( model_name=request.model, model_path_str=request.modelFile ) except Exception as e: @@ -310,7 +310,7 @@ def synthesis(request: SynthesisRequest): language=request.language, sdp_ratio=request.sdpRatio, noise=request.noise, - noisew=request.noisew, + noise_w=request.noisew, length=1 / request.speed, given_tone=tone, style=request.style, @@ -321,7 +321,7 @@ def synthesis(request: SynthesisRequest): line_split=False, pitch_scale=request.pitchScale, intonation_scale=request.intonationScale, - sid=sid, + speaker_id=sid, ) with BytesIO() as wavContent: @@ -350,7 +350,7 @@ def multi_synthesis(request: MultiSynthesisRequest): detail=f"1行の文字数は{args.line_length}文字以下にしてください。", ) try: - model = model_holder.load_model( + model = model_holder.get_model( model_name=req.model, model_path_str=req.modelFile ) except Exception as e: @@ -370,7 +370,7 @@ def multi_synthesis(request: MultiSynthesisRequest): language=req.language, sdp_ratio=req.sdpRatio, noise=req.noise, - noisew=req.noisew, + noise_w=req.noisew, length=1 / req.speed, given_tone=tone, style=req.style, diff --git a/server_fastapi.py b/server_fastapi.py index 30f253f..1f60b3c 100644 --- a/server_fastapi.py +++ b/server_fastapi.py @@ -36,7 +36,7 @@ from style_bert_vits2.constants import ( from style_bert_vits2.logging import logger from style_bert_vits2.nlp import bert_models from style_bert_vits2.nlp.japanese import pyopenjtalk_worker as pyopenjtalk -from style_bert_vits2.tts_model import Model, ModelHolder +from style_bert_vits2.tts_model import TTSModel, TTSModelHolder ln = config.server_config.language @@ -67,16 +67,16 @@ class AudioResponse(Response): media_type = "audio/wav" -def load_models(model_holder: ModelHolder): +def load_models(model_holder: TTSModelHolder): model_holder.models = [] for model_name, model_paths in model_holder.model_files_dict.items(): - model = Model( + model = TTSModel( model_path=model_paths[0], config_path=model_holder.root_dir / model_name / "config.json", style_vec_path=model_holder.root_dir / model_name / "style_vectors.npy", device=model_holder.device, ) - model.load_net_g() + model.load() model_holder.models.append(model) @@ -94,7 +94,7 @@ if __name__ == "__main__": device = "cuda" if torch.cuda.is_available() else "cpu" model_dir = Path(args.dir) - model_holder = ModelHolder(model_dir, device) + model_holder = TTSModelHolder(model_dir, device) if len(model_holder.model_names) == 0: logger.error(f"Models not found in {model_dir}.") sys.exit(1) @@ -194,11 +194,11 @@ if __name__ == "__main__": sr, audio = model.infer( text=text, language=language, - sid=speaker_id, + speaker_id=speaker_id, reference_audio_path=reference_audio_path, sdp_ratio=sdp_ratio, noise=noise, - noisew=noisew, + noise_w=noisew, length=length, line_split=auto_split, split_interval=split_interval, diff --git a/speech_mos.py b/speech_mos.py index 79221ad..6dd2caa 100644 --- a/speech_mos.py +++ b/speech_mos.py @@ -12,7 +12,7 @@ from tqdm import tqdm from config import config from style_bert_vits2.logging import logger -from style_bert_vits2.tts_model import Model +from style_bert_vits2.tts_model import TTSModel warnings.filterwarnings("ignore") @@ -54,7 +54,7 @@ safetensors_files = model_path.glob("*.safetensors") def get_model(model_file: Path): - return Model( + return TTSModel( model_path=str(model_file), config_path=str(model_file.parent / "config.json"), style_vec_path=str(model_file.parent / "style_vectors.npy"), diff --git a/style_bert_vits2/tts_model.py b/style_bert_vits2/tts_model.py index 42f6b84..109ef0e 100644 --- a/style_bert_vits2/tts_model.py +++ b/style_bert_vits2/tts_model.py @@ -29,9 +29,9 @@ from style_bert_vits2.logging import logger from style_bert_vits2.voice import adjust_voice -class Model: +class TTSModel: """ - Style-Bert-Vits2 の音声合成モデルを操作するためのクラス。 + Style-Bert-Vits2 の音声合成モデルを操作するクラス。 モデル/ハイパーパラメータ/スタイルベクトルのパスとデバイスを指定して初期化し、model.infer() メソッドを呼び出すと音声合成を行える。 """ @@ -43,6 +43,17 @@ class Model: style_vec_path: Path, device: str, ) -> None: + """ + Style-Bert-Vits2 の音声合成モデルを初期化する。 + この時点ではモデルはロードされていない (明示的にロードしたい場合は model.load() を呼び出す)。 + + Args: + model_path (Path): モデル (.safetensors) のパス + config_path (Path): ハイパーパラメータ (config.json) のパス + style_vec_path (Path): スタイルベクトル (style_vectors.npy) のパス + device (str): 音声合成時に利用するデバイス (cpu, cuda, mps など) + """ + self.model_path: Path = model_path self.config_path: Path = config_path self.style_vec_path: Path = style_vec_path @@ -71,24 +82,24 @@ class Model: self.__net_g: Union[SynthesizerTrn, SynthesizerTrnJPExtra, None] = None - def load_net_g(self) -> None: + def load(self) -> None: """ - net_g をロードする。 + 音声合成モデルをデバイスにロードする。 """ self.__net_g = get_net_g( - model_path=str(self.model_path), - version=self.hyper_parameters.version, - device=self.device, - hps=self.hyper_parameters, + model_path = str(self.model_path), + version = self.hyper_parameters.version, + device = self.device, + hps = self.hyper_parameters, ) - def get_style_vector(self, style_id: int, weight: float = 1.0) -> NDArray[Any]: + def __get_style_vector(self, style_id: int, weight: float = 1.0) -> NDArray[Any]: """ スタイルベクトルを取得する。 Args: - style_id (int): スタイル ID + style_id (int): スタイル ID (0 から始まるインデックス) weight (float, optional): スタイルベクトルの重み. Defaults to 1.0. Returns: @@ -100,7 +111,7 @@ class Model: return style_vec - def get_style_vector_from_audio(self, audio_path: str, weight: float = 1.0) -> NDArray[Any]: + def __get_style_vector_from_audio(self, audio_path: str, weight: float = 1.0) -> NDArray[Any]: """ 音声からスタイルベクトルを推論する。 @@ -130,11 +141,11 @@ class Model: self, text: str, language: Languages = Languages.JP, - sid: int = 0, + speaker_id: int = 0, reference_audio_path: Optional[str] = None, sdp_ratio: float = DEFAULT_SDP_RATIO, noise: float = DEFAULT_NOISE, - noisew: float = DEFAULT_NOISEW, + noise_w: float = DEFAULT_NOISEW, length: float = DEFAULT_LENGTH, line_split: bool = DEFAULT_LINE_SPLIT, split_interval: float = DEFAULT_SPLIT_INTERVAL, @@ -147,6 +158,33 @@ class Model: pitch_scale: float = 1.0, intonation_scale: float = 1.0, ) -> tuple[int, NDArray[Any]]: + """ + テキストから音声を合成する。 + + Args: + text (str): 読み上げるテキスト + language (Languages, optional): 言語. Defaults to Languages.JP. + speaker_id (int, optional): 話者 ID. Defaults to 0. + reference_audio_path (Optional[str], optional): 音声スタイルの参照元の音声ファイルのパス. Defaults to None. + sdp_ratio (float, optional): SDP レシオ (値を大きくするとより感情豊かになる傾向がある). Defaults to DEFAULT_SDP_RATIO. + noise (float, optional): ノイズの大きさ. Defaults to DEFAULT_NOISE. + noise_w (float, optional): ノイズの大きさの重み. Defaults to DEFAULT_NOISEW. + length (float, optional): 長さ. Defaults to DEFAULT_LENGTH. + line_split (bool, optional): テキストを改行ごとに分割して生成するかどうか. Defaults to DEFAULT_LINE_SPLIT. + split_interval (float, optional): 改行ごとに分割する場合の無音 (秒). Defaults to DEFAULT_SPLIT_INTERVAL. + assist_text (Optional[str], optional): 感情表現の参照元の補助テキスト. Defaults to None. + assist_text_weight (float, optional): 感情表現の補助テキストを適用する強さ. Defaults to DEFAULT_ASSIST_TEXT_WEIGHT. + use_assist_text (bool, optional): 音声合成時に感情表現の補助テキストを使用するかどうか. Defaults to False. + style (str, optional): 音声スタイル (Neutral, Happy など). Defaults to DEFAULT_STYLE. + style_weight (float, optional): 音声スタイルを適用する強さ. Defaults to DEFAULT_STYLE_WEIGHT. + given_tone (Optional[list[int]], optional): アクセントのトーンのリスト. Defaults to None. + pitch_scale (float, optional): ピッチの高さ (1.0 から変更すると若干音質が低下する). Defaults to 1.0. + intonation_scale (float, optional): イントネーションの高さ (1.0 から変更すると若干音質が低下する). Defaults to 1.0. + + Returns: + tuple[int, NDArray[Any]]: サンプリングレートと音声データ (16bit PCM) + """ + logger.info(f"Start generating audio data from text:\n{text}") if language != "JP" and self.hyper_parameters.version.endswith("JP-Extra"): raise ValueError( @@ -158,13 +196,13 @@ class Model: assist_text = None if self.__net_g is None: - self.load_net_g() + self.load() assert self.__net_g is not None if reference_audio_path is None: style_id = self.style2id[style] - style_vector = self.get_style_vector(style_id, style_weight) + style_vector = self.__get_style_vector(style_id, style_weight) else: - style_vector = self.get_style_vector_from_audio( + style_vector = self.__get_style_vector_from_audio( reference_audio_path, style_weight ) if not line_split: @@ -173,9 +211,9 @@ class Model: text = text, sdp_ratio = sdp_ratio, noise_scale = noise, - noise_scale_w = noisew, + noise_scale_w = noise_w, length_scale = length, - sid = sid, + sid = speaker_id, language = language, hps = self.hyper_parameters, net_g = self.__net_g, @@ -196,9 +234,9 @@ class Model: text = t, sdp_ratio = sdp_ratio, noise_scale = noise, - noise_scale_w = noisew, + noise_scale_w = noise_w, length_scale = length, - sid = sid, + sid = speaker_id, language = language, hps = self.hyper_parameters, net_g = self.__net_g, @@ -225,24 +263,50 @@ class Model: return (self.hyper_parameters.data.sampling_rate, audio) -class ModelHolder: +class TTSModelHolder: """ - Style-Bert-Vits2 の音声合成モデルを管理するためのクラス。 + Style-Bert-Vits2 の音声合成モデルを管理するクラス。 + model_holder.models_info から指定されたディレクトリ内にある音声合成モデルの一覧を取得できる。 """ def __init__(self, model_root_dir: Path, device: str) -> None: + """ + Style-Bert-Vits2 の音声合成モデルを管理するクラスを初期化する。 + 音声合成モデルは下記のように配置されていることを前提とする (.safetensors のファイル名は自由) 。 + ``` + model_root_dir + ├── model-name-1 + │ ├── config.json + │ ├── model-name-1_e160_s14000.safetensors + │ └── style_vectors.npy + ├── model-name-2 + │ ├── config.json + │ ├── model-name-2_e160_s14000.safetensors + │ └── style_vectors.npy + └── ... + ``` + + Args: + model_root_dir (Path): 音声合成モデルが配置されているディレクトリのパス + device (str): 音声合成時に利用するデバイス (cpu, cuda, mps など) + """ + self.root_dir: Path = model_root_dir self.device: str = device self.model_files_dict: dict[str, list[Path]] = {} - self.current_model: Optional[Model] = None + self.current_model: Optional[TTSModel] = None self.model_names: list[str] = [] - self.models: list[Model] = [] + self.models: list[TTSModel] = [] self.models_info: list[dict[str, Union[str, list[str]]]] = [] self.refresh() def refresh(self) -> None: + """ + 音声合成モデルの一覧を更新する。 + """ + self.model_files_dict = {} self.model_names = [] self.current_model = None @@ -279,23 +343,36 @@ class ModelHolder: }) - def load_model(self, model_name: str, model_path_str: str) -> Model: + def get_model(self, model_name: str, model_path_str: str) -> TTSModel: + """ + 指定された音声合成モデルのインスタンスを取得する。 + この時点ではモデルはロードされていない (明示的にロードしたい場合は model.load() を呼び出す)。 + + Args: + model_name (str): 音声合成モデルの名前 + model_path_str (str): 音声合成モデルのファイルパス (.safetensors) + + Returns: + TTSModel: 音声合成モデルのインスタンス + """ + model_path = Path(model_path_str) if model_name not in self.model_files_dict: raise ValueError(f"Model `{model_name}` is not found") if model_path not in self.model_files_dict[model_name]: raise ValueError(f"Model file `{model_path}` is not found") if self.current_model is None or self.current_model.model_path != model_path: - self.current_model = Model( + self.current_model = TTSModel( model_path = model_path, config_path = self.root_dir / model_name / "config.json", style_vec_path = self.root_dir / model_name / "style_vectors.npy", device = self.device, ) + return self.current_model - def load_model_for_gradio(self, model_name: str, model_path_str: str) -> tuple[gr.Dropdown, gr.Button, gr.Dropdown]: + def get_model_for_gradio(self, model_name: str, model_path_str: str) -> tuple[gr.Dropdown, gr.Button, gr.Dropdown]: model_path = Path(model_path_str) if model_name not in self.model_files_dict: raise ValueError(f"Model `{model_name}` is not found") @@ -313,7 +390,7 @@ class ModelHolder: gr.Button(interactive=True, value="音声合成"), gr.Dropdown(choices=speakers, value=speakers[0]), # type: ignore ) - self.current_model = Model( + self.current_model = TTSModel( model_path = model_path, config_path = self.root_dir / model_name / "config.json", style_vec_path = self.root_dir / model_name / "style_vectors.npy", diff --git a/webui/inference.py b/webui/inference.py index 6d6eae6..e32907c 100644 --- a/webui/inference.py +++ b/webui/inference.py @@ -23,7 +23,7 @@ from style_bert_vits2.nlp import bert_models from style_bert_vits2.nlp.japanese import pyopenjtalk_worker as pyopenjtalk from style_bert_vits2.nlp.japanese.g2p_utils import g2kata_tone, kata_tone2phone_tone from style_bert_vits2.nlp.japanese.normalizer import normalize_text -from style_bert_vits2.tts_model import ModelHolder +from style_bert_vits2.tts_model import TTSModelHolder # pyopenjtalk_worker を起動 @@ -151,7 +151,7 @@ def gr_util(item): return (gr.update(visible=False), gr.update(visible=True)) -def create_inference_app(model_holder: ModelHolder) -> gr.Blocks: +def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks: def tts_fn( model_name, model_path, @@ -175,7 +175,7 @@ def create_inference_app(model_holder: ModelHolder) -> gr.Blocks: pitch_scale, intonation_scale, ): - model_holder.load_model(model_name, model_path) + model_holder.get_model(model_name, model_path) assert model_holder.current_model is not None wrong_tone_message = "" @@ -218,7 +218,7 @@ def create_inference_app(model_holder: ModelHolder) -> gr.Blocks: reference_audio_path=reference_audio_path, sdp_ratio=sdp_ratio, noise=noise_scale, - noisew=noise_scale_w, + noise_w=noise_scale_w, length=length_scale, line_split=line_split, split_interval=split_interval, @@ -228,7 +228,7 @@ def create_inference_app(model_holder: ModelHolder) -> gr.Blocks: style=style, style_weight=style_weight, given_tone=tone, - sid=speaker_id, + speaker_id=speaker_id, pitch_scale=pitch_scale, intonation_scale=intonation_scale, ) @@ -459,7 +459,7 @@ def create_inference_app(model_holder: ModelHolder) -> gr.Blocks: ) load_button.click( - model_holder.load_model_for_gradio, + model_holder.get_model_for_gradio, inputs=[model_name, model_path], outputs=[style, tts_button, speaker], ) diff --git a/webui/merge.py b/webui/merge.py index 85806f4..f692b67 100644 --- a/webui/merge.py +++ b/webui/merge.py @@ -11,7 +11,7 @@ from safetensors.torch import save_file from style_bert_vits2.constants import DEFAULT_STYLE, GRADIO_THEME from style_bert_vits2.logging import logger -from style_bert_vits2.tts_model import Model, ModelHolder +from style_bert_vits2.tts_model import TTSModel, TTSModelHolder voice_keys = ["dec"] @@ -250,11 +250,11 @@ def simple_tts(model_name, text, style=DEFAULT_STYLE, style_weight=1.0): config_path = os.path.join(assets_root, model_name, "config.json") style_vec_path = os.path.join(assets_root, model_name, "style_vectors.npy") - model = Model(Path(model_path), Path(config_path), Path(style_vec_path), device) + model = TTSModel(Path(model_path), Path(config_path), Path(style_vec_path), device) return model.infer(text, style=style, style_weight=style_weight) -def update_two_model_names_dropdown(model_holder: ModelHolder): +def update_two_model_names_dropdown(model_holder: TTSModelHolder): new_names, new_files, _ = model_holder.update_model_names_for_gradio() return new_names, new_files, new_names, new_files @@ -328,7 +328,7 @@ Happy, Surprise, HappySurprise """ -def create_merge_app(model_holder: ModelHolder) -> gr.Blocks: +def create_merge_app(model_holder: TTSModelHolder) -> gr.Blocks: model_names = model_holder.model_names if len(model_names) == 0: logger.error(