From ef4e82defce6af9695ea4c6bf4a5fa9e22a4c0eb Mon Sep 17 00:00:00 2001 From: litagin02 Date: Sun, 31 Dec 2023 14:55:04 +0900 Subject: [PATCH] Refactor and add merge --- Merge.bat | 13 + README.md | 2 - app.py | 275 +++--------------- attentions.py | 2 +- bert_gen.py | 2 +- common/constants.py | 20 ++ {tools => common}/log.py | 0 {tools => common}/stdout_wrapper.py | 0 {tools => common}/subprocess_utils.py | 68 ++--- common/tts_model.py | 219 +++++++++++++++ config.py | 2 +- data_utils.py | 2 +- default_style.py | 7 +- docs/CHANGELOG.md | 10 +- infer.py | 14 +- initialize.py | 2 +- preprocess_text.py | 2 +- resample.py | 4 +- server_fastapi.py | 72 ++--- server_test.ipynb | 74 +++++ slice.py | 2 +- style_gen.py | 2 +- text/__init__.py | 6 +- text/chinese_bert.py | 14 +- text/english_bert_mock.py | 14 +- text/japanese_bert.py | 18 +- train_ms.py | 5 +- transcribe.py | 2 +- utils.py | 2 +- webui.py | 4 +- webui_dataset.py | 4 +- webui_merge.py | 389 ++++++++++++++++++++++++++ webui_style_vectors.py | 12 +- webui_train.py | 4 +- 34 files changed, 898 insertions(+), 370 deletions(-) create mode 100644 Merge.bat create mode 100644 common/constants.py rename {tools => common}/log.py (100%) rename {tools => common}/stdout_wrapper.py (100%) rename {tools => common}/subprocess_utils.py (96%) create mode 100644 common/tts_model.py create mode 100644 server_test.ipynb create mode 100644 webui_merge.py diff --git a/Merge.bat b/Merge.bat new file mode 100644 index 0000000..8e1dedb --- /dev/null +++ b/Merge.bat @@ -0,0 +1,13 @@ +chcp 65001 > NUL + +@echo off + +pushd %~dp0 + +echo Running webui_merge.py... +venv\Scripts\python webui_merge.py + +if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) + +popd +pause \ No newline at end of file diff --git a/README.md b/README.md index 4761a17..aa46aba 100644 --- a/README.md +++ b/README.md @@ -44,8 +44,6 @@ Windowsを前提としています。 #### GitやPython使える人 -Windowsの通常環境のPython 3.10で動作確認していますが、他の環境でも動くと思います。 - ```bash git clone https://github.com/litagin02/Style-Bert-VITS2.git cd Style-Bert-VITS2 diff --git a/app.py b/app.py index c7844a1..f8d4e19 100644 --- a/app.py +++ b/app.py @@ -2,225 +2,30 @@ import argparse import datetime import os import sys -import warnings -import enum import gradio as gr import numpy as np import torch from gradio.processing_utils import convert_to_16_bit_wav -from typing import Dict, List -import utils +from common.constants import ( + DEFAULT_ASSIST_TEXT_WEIGHT, + DEFAULT_LENGTH, + DEFAULT_LINE_SPLIT, + DEFAULT_NOISE, + DEFAULT_NOISEW, + DEFAULT_SDP_RATIO, + DEFAULT_SPLIT_INTERVAL, + DEFAULT_STYLE, + DEFAULT_STYLE_WEIGHT, + Languages, +) +from common.log import logger +from common.tts_model import ModelHolder from config import config -from infer import get_net_g, infer -from tools.log import logger - - -class Languages(str, enum.Enum): - JP = "JP" - EN = "EN" - ZH = "ZH" - languages = [l.value for l in Languages] -DEFAULT_SDP_RATIO: float = 0.2 -DEFAULT_NOISE: float = 0.6 -DEFAULT_NOISEW: float = 0.8 -DEFAULT_LENGTH: float = 1 -DEFAULT_LINE_SPLIT: bool = True -DEFAULT_SPLIT_INTERVAL: float = 0.5 -DEFAULT_STYLE_WEIGHT: float = 0.7 -DEFAULT_EMOTION_WEIGHT: float = 1.0 - - -class Model: - def __init__(self, model_path, config_path, style_vec_path, device): - self.model_path = model_path - self.config_path = config_path - self.device = device - self.style_vec_path = style_vec_path - self.hps = utils.get_hparams_from_file(self.config_path) - self.spk2id: Dict[str, int] = self.hps.data.spk2id - self.id2spk: Dict[int, str] = {v: k for k, v in self.spk2id.items()} - self.num_styles = self.hps.data.num_styles - if hasattr(self.hps.data, "style2id"): - self.style2id = self.hps.data.style2id - else: - self.style2id = {str(i): i for i in range(self.num_styles)} - - self.style_vectors = np.load(self.style_vec_path) - self.net_g = None - - def load_net_g(self): - self.net_g = get_net_g( - model_path=self.model_path, - version=self.hps.version, - device=self.device, - hps=self.hps, - ) - - def get_style_vector(self, style_id, weight=1.0): - mean = self.style_vectors[0] - style_vec = self.style_vectors[style_id] - style_vec = mean + (style_vec - mean) * weight - return style_vec - - def get_style_vector_from_audio(self, audio_path, weight=1.0): - from style_gen import extract_style_vector - - xvec = extract_style_vector(audio_path) - mean = self.style_vectors[0] - xvec = mean + (xvec - mean) * weight - return xvec - - def infer( - self, - text, - language="JP", - sid=0, - reference_audio_path=None, - sdp_ratio=DEFAULT_SDP_RATIO, - noise=DEFAULT_NOISE, - noisew=DEFAULT_NOISEW, - length=DEFAULT_LENGTH, - line_split=DEFAULT_LINE_SPLIT, - split_interval=DEFAULT_SPLIT_INTERVAL, - style_text="", - style_weight=DEFAULT_STYLE_WEIGHT, - use_style_text=False, - style="0", - emotion_weight=DEFAULT_EMOTION_WEIGHT, - ): - if reference_audio_path == "": - reference_audio_path = None - if style_text == "" or not use_style_text: - style_text = None - - if self.net_g is None: - self.load_net_g() - if reference_audio_path is None: - style_id = self.style2id[style] - style_vector = self.get_style_vector(style_id, emotion_weight) - else: - style_vector = self.get_style_vector_from_audio( - reference_audio_path, emotion_weight - ) - if not line_split: - with torch.no_grad(): - audio = infer( - text=text, - sdp_ratio=sdp_ratio, - noise_scale=noise, - noise_scale_w=noisew, - length_scale=length, - sid=sid, - language=language, - hps=self.hps, - net_g=self.net_g, - device=self.device, - style_text=style_text, - style_weight=style_weight, - style_vec=style_vector, - ) - else: - texts = text.split("\n") - texts = [t for t in texts if t != ""] - audios = [] - with torch.no_grad(): - for i, t in enumerate(texts): - audios.append( - infer( - text=t, - sdp_ratio=sdp_ratio, - noise_scale=noise, - noise_scale_w=noisew, - length_scale=length, - sid=sid, - language=language, - hps=self.hps, - net_g=self.net_g, - device=self.device, - style_text=style_text, - style_weight=style_weight, - style_vec=style_vector, - ) - ) - if i != len(texts) - 1: - audios.append(np.zeros(int(44100 * split_interval))) - audio = np.concatenate(audios) - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - audio = convert_to_16_bit_wav(audio) - return (self.hps.data.sampling_rate, audio) - - -class ModelHolder: - def __init__(self, root_dir, device): - self.root_dir = root_dir - self.device = device - self.model_files_dict = {} - self.current_model = None - self.model_names = [] - self.models = [] - self.refresh() - - def refresh(self): - self.model_files_dict: Dict[str, List[str]] = {} - self.model_names: List[str] = [] - self.current_model = None - model_dirs = [ - d - for d in os.listdir(self.root_dir) - if os.path.isdir(os.path.join(self.root_dir, d)) - ] - for model_name in model_dirs: - model_dir = os.path.join(self.root_dir, model_name) - model_files = [ - os.path.join(model_dir, f) - for f in os.listdir(model_dir) - if f.endswith(".pth") or f.endswith(".pt") or f.endswith(".safetensors") - ] - if len(model_files) == 0: - logger.info( - f"No model files found in {self.root_dir}/{model_name}, so skip it" - ) - continue - self.model_files_dict[model_name] = model_files - self.model_names.append(model_name) - - def load_model(self, model_name, model_path): - if model_name not in self.model_files_dict: - raise Exception(f"モデル名{model_name}は存在しません") - if model_path not in self.model_files_dict[model_name]: - raise Exception(f"pthファイル{model_path}は存在しません") - self.current_model = Model( - model_path=model_path, - config_path=os.path.join(self.root_dir, model_name, "config.json"), - style_vec_path=os.path.join(self.root_dir, model_name, "style_vectors.npy"), - device=self.device, - ) - styles = list(self.current_model.style2id.keys()) - return ( - gr.Dropdown(choices=styles, value=styles[0]), - gr.update(interactive=True, value="音声合成"), - ) - - def update_model_files_dropdown(self, model_name): - model_files = self.model_files_dict[model_name] - return gr.Dropdown(choices=model_files, value=model_files[0]) - - def update_model_names_dropdown(self): - self.refresh() - initial_model_name = self.model_names[0] - initial_model_files = self.model_files_dict[initial_model_name] - return ( - gr.Dropdown(choices=self.model_names, value=initial_model_name), - gr.Dropdown(choices=initial_model_files, value=initial_model_files[0]), - gr.update(interactive=False), # For tts_button - ) - def tts_fn( text, @@ -232,11 +37,11 @@ def tts_fn( length_scale, line_split, split_interval, - style_text, + assist_text, + assist_text_weight, + use_assist_text, + style, style_weight, - use_style_text, - emotion, - emotion_weight, ): assert model_holder.current_model is not None @@ -252,11 +57,11 @@ def tts_fn( length=length_scale, line_split=line_split, split_interval=split_interval, - style_text=style_text, + assist_text=assist_text, + assist_text_weight=assist_text_weight, + use_assist_text=use_assist_text, + style=style, style_weight=style_weight, - use_style_text=use_style_text, - style=emotion, - emotion_weight=emotion_weight, ) end_time = datetime.datetime.now() @@ -349,9 +154,9 @@ model_assets TODO: 現在のところはspeaker_id = 0に固定しており複数話者の合成には対応していません。 """ -style_md = """ +style_md = f""" - プリセットまたは音声ファイルから読み上げの声音・感情・スタイルのようなものを制御できます。 -- デフォルトのNeutralでも、十分に読み上げる文に応じた感情で感情豊かに読み上げられます。このスタイル制御は、それを重み付きで上書きするような感じです。 +- デフォルトの{DEFAULT_STYLE}でも、十分に読み上げる文に応じた感情で感情豊かに読み上げられます。このスタイル制御は、それを重み付きで上書きするような感じです。 - 強さを大きくしすぎると発音が変になったり声にならなかったりと崩壊することがあります。 - どのくらいに強さがいいかはモデルやスタイルによって異なるようです。 - 音声ファイルを入力する場合は、学習データと似た声音の話者（特に同じ性別）でないとよい効果が出ないかもしれません。 @@ -459,25 +264,25 @@ if __name__ == "__main__": step=0.1, label="Length", ) - use_style_text = gr.Checkbox(label="Style textを使う", value=False) - style_text = gr.Textbox( - label="Style text", + use_assist_text = gr.Checkbox(label="Assist textを使う", value=False) + assist_text = gr.Textbox( + label="Assist text", placeholder="どうして私の意見を無視するの？許せない、ムカつく！死ねばいいのに。", info="このテキストの読み上げと似た声音・感情になりやすくなります。ただ抑揚やテンポ等が犠牲になる傾向があります。", visible=False, ) - style_text_weight = gr.Slider( + assist_text_weight = gr.Slider( minimum=0, maximum=1, - value=DEFAULT_STYLE_WEIGHT, + value=DEFAULT_ASSIST_TEXT_WEIGHT, step=0.1, label="Style textの強さ", visible=False, ) - use_style_text.change( + use_assist_text.change( lambda x: (gr.Textbox(visible=x), gr.Slider(visible=x)), - inputs=[use_style_text], - outputs=[style_text, style_text_weight], + inputs=[use_assist_text], + outputs=[assist_text, assist_text_weight], ) with gr.Column(): with gr.Accordion("スタイルについて詳細", open=False): @@ -488,14 +293,14 @@ if __name__ == "__main__": value="プリセットから選ぶ", ) style = gr.Dropdown( - label="スタイル（Neutralが平均スタイル）", + label="スタイル（{DEFAULT_STYLE}が平均スタイル）", choices=["モデルをロードしてください"], value="モデルをロードしてください", ) style_weight = gr.Slider( minimum=0, maximum=50, - value=DEFAULT_EMOTION_WEIGHT, + value=DEFAULT_STYLE_WEIGHT, step=0.1, label="スタイルの強さ", ) @@ -520,9 +325,9 @@ if __name__ == "__main__": length_scale, line_split, split_interval, - style_text, - style_text_weight, - use_style_text, + assist_text, + assist_text_weight, + use_assist_text, style, style_weight, ], @@ -530,7 +335,7 @@ if __name__ == "__main__": ) model_name.change( - model_holder.update_model_files_dropdown, + model_holder.update_model_files_gr, inputs=[model_name], outputs=[model_path], ) @@ -538,12 +343,12 @@ if __name__ == "__main__": model_path.change(make_non_interactive, outputs=[tts_button]) refresh_button.click( - model_holder.update_model_names_dropdown, + model_holder.update_model_names_gr, outputs=[model_name, model_path, tts_button], ) load_button.click( - model_holder.load_model, + model_holder.load_model_gr, inputs=[model_name, model_path], outputs=[style, tts_button], ) diff --git a/attentions.py b/attentions.py index dcf46b3..9a4bba9 100644 --- a/attentions.py +++ b/attentions.py @@ -4,7 +4,7 @@ from torch import nn from torch.nn import functional as F import commons -from tools.log import logger as logging +from common.log import logger as logging class LayerNorm(nn.Module): diff --git a/bert_gen.py b/bert_gen.py index e41484d..934b81b 100644 --- a/bert_gen.py +++ b/bert_gen.py @@ -9,7 +9,7 @@ import commons import utils from config import config from text import cleaned_text_to_sequence, get_bert -from tools.stdout_wrapper import SAFE_STDOUT +from common.stdout_wrapper import SAFE_STDOUT def process_line(x): diff --git a/common/constants.py b/common/constants.py new file mode 100644 index 0000000..f000095 --- /dev/null +++ b/common/constants.py @@ -0,0 +1,20 @@ +import enum + +DEFAULT_STYLE: str = "Neutral" +DEFAULT_STYLE_WEIGHT: float = 5.0 + + +class Languages(str, enum.Enum): + JP = "JP" + EN = "EN" + ZH = "ZH" + + +DEFAULT_SDP_RATIO: float = 0.2 +DEFAULT_NOISE: float = 0.6 +DEFAULT_NOISEW: float = 0.8 +DEFAULT_LENGTH: float = 1.0 +DEFAULT_LINE_SPLIT: bool = True +DEFAULT_SPLIT_INTERVAL: float = 0.5 +DEFAULT_ASSIST_TEXT_WEIGHT: float = 0.7 +DEFAULT_ASSIST_TEXT_WEIGHT: float = 1.0 diff --git a/tools/log.py b/common/log.py similarity index 100% rename from tools/log.py rename to common/log.py diff --git a/tools/stdout_wrapper.py b/common/stdout_wrapper.py similarity index 100% rename from tools/stdout_wrapper.py rename to common/stdout_wrapper.py diff --git a/tools/subprocess_utils.py b/common/subprocess_utils.py similarity index 96% rename from tools/subprocess_utils.py rename to common/subprocess_utils.py index ad15575..def8822 100644 --- a/tools/subprocess_utils.py +++ b/common/subprocess_utils.py @@ -1,34 +1,34 @@ -import subprocess -import sys - -from .log import logger -from .stdout_wrapper import SAFE_STDOUT - -python = sys.executable - - -def run_script_with_log(cmd: list[str]) -> tuple[bool, str]: - logger.info(f"Running: {' '.join(cmd)}") - result = subprocess.run( - [python] + cmd, - stdout=SAFE_STDOUT, # type: ignore - stderr=subprocess.PIPE, - text=True, - ) - if result.returncode != 0: - logger.error(f"Error: {' '.join(cmd)}") - print(result.stderr) - return False, result.stderr - elif result.stderr: - logger.warning(f"Warning: {' '.join(cmd)}") - print(result.stderr) - return True, result.stderr - logger.success(f"Success: {' '.join(cmd)}") - return True, "" - - -def second_elem_of(original_function): - def inner_function(*args, **kwargs): - return original_function(*args, **kwargs)[1] - - return inner_function +import subprocess +import sys + +from .log import logger +from .stdout_wrapper import SAFE_STDOUT + +python = sys.executable + + +def run_script_with_log(cmd: list[str]) -> tuple[bool, str]: + logger.info(f"Running: {' '.join(cmd)}") + result = subprocess.run( + [python] + cmd, + stdout=SAFE_STDOUT, # type: ignore + stderr=subprocess.PIPE, + text=True, + ) + if result.returncode != 0: + logger.error(f"Error: {' '.join(cmd)}") + print(result.stderr) + return False, result.stderr + elif result.stderr: + logger.warning(f"Warning: {' '.join(cmd)}") + print(result.stderr) + return True, result.stderr + logger.success(f"Success: {' '.join(cmd)}") + return True, "" + + +def second_elem_of(original_function): + def inner_function(*args, **kwargs): + return original_function(*args, **kwargs)[1] + + return inner_function diff --git a/common/tts_model.py b/common/tts_model.py new file mode 100644 index 0000000..f83a806 --- /dev/null +++ b/common/tts_model.py @@ -0,0 +1,219 @@ +import numpy as np +import gradio as gr +import torch +import os +import warnings +from gradio.processing_utils import convert_to_16_bit_wav +from typing import Dict, List, Optional + +import utils +from infer import get_net_g, infer +from models import SynthesizerTrn + +from .log import logger +from .constants import ( + DEFAULT_ASSIST_TEXT_WEIGHT, + DEFAULT_LENGTH, + DEFAULT_LINE_SPLIT, + DEFAULT_NOISE, + DEFAULT_NOISEW, + DEFAULT_SDP_RATIO, + DEFAULT_SPLIT_INTERVAL, + DEFAULT_STYLE, + DEFAULT_STYLE_WEIGHT, +) + + +class Model: + def __init__( + self, model_path: str, config_path: str, style_vec_path: str, device: str + ): + self.model_path: str = model_path + self.config_path: str = config_path + self.device: str = device + self.style_vec_path: str = style_vec_path + self.hps: utils.HParams = utils.get_hparams_from_file(self.config_path) + self.spk2id: Dict[str, int] = self.hps.data.spk2id + self.id2spk: Dict[int, str] = {v: k for k, v in self.spk2id.items()} + self.num_styles: int = self.hps.data.num_styles + if hasattr(self.hps.data, "style2id"): + self.style2id: Dict[str, int] = self.hps.data.style2id + else: + self.style2id: Dict[str, int] = {str(i): i for i in range(self.num_styles)} + + self.style_vectors: np.ndarray = np.load(self.style_vec_path) + self.net_g: Optional[SynthesizerTrn] = None + + def load_net_g(self): + self.net_g = get_net_g( + model_path=self.model_path, + version=self.hps.version, + device=self.device, + hps=self.hps, + ) + + def get_style_vector(self, style_id: int, weight: float = 1.0) -> np.ndarray: + mean = self.style_vectors[0] + style_vec = self.style_vectors[style_id] + style_vec = mean + (style_vec - mean) * weight + return style_vec + + def get_style_vector_from_audio( + self, audio_path: str, weight: float = 1.0 + ) -> np.ndarray: + from style_gen import extract_style_vector + + xvec = extract_style_vector(audio_path) + mean = self.style_vectors[0] + xvec = mean + (xvec - mean) * weight + return xvec + + def infer( + self, + text: str, + language: str = "JP", + sid: int = 0, + reference_audio_path: Optional[str] = None, + sdp_ratio: float = DEFAULT_SDP_RATIO, + noise: float = DEFAULT_NOISE, + noisew: float = DEFAULT_NOISEW, + length: float = DEFAULT_LENGTH, + line_split: bool = DEFAULT_LINE_SPLIT, + split_interval: float = DEFAULT_SPLIT_INTERVAL, + assist_text: Optional[str] = None, + assist_text_weight: float = DEFAULT_ASSIST_TEXT_WEIGHT, + use_assist_text: bool = False, + style: str = DEFAULT_STYLE, + style_weight: float = DEFAULT_STYLE_WEIGHT, + ) -> tuple[int, np.ndarray]: + logger.info(f"Start generating audio data from text:\n{text}") + if reference_audio_path == "": + reference_audio_path = None + if assist_text == "" or not use_assist_text: + assist_text = None + + if self.net_g is None: + self.load_net_g() + if reference_audio_path is None: + style_id = self.style2id[style] + style_vector = self.get_style_vector(style_id, style_weight) + else: + style_vector = self.get_style_vector_from_audio( + reference_audio_path, style_weight + ) + if not line_split: + with torch.no_grad(): + audio = infer( + text=text, + sdp_ratio=sdp_ratio, + noise_scale=noise, + noise_scale_w=noisew, + length_scale=length, + sid=sid, + language=language, + hps=self.hps, + net_g=self.net_g, + device=self.device, + assist_text=assist_text, + assist_text_weight=assist_text_weight, + style_vec=style_vector, + ) + else: + texts = text.split("\n") + texts = [t for t in texts if t != ""] + audios = [] + with torch.no_grad(): + for i, t in enumerate(texts): + audios.append( + infer( + text=t, + sdp_ratio=sdp_ratio, + noise_scale=noise, + noise_scale_w=noisew, + length_scale=length, + sid=sid, + language=language, + hps=self.hps, + net_g=self.net_g, + device=self.device, + assist_text=assist_text, + assist_text_weight=assist_text_weight, + style_vec=style_vector, + ) + ) + if i != len(texts) - 1: + audios.append(np.zeros(int(44100 * split_interval))) + audio = np.concatenate(audios) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + audio = convert_to_16_bit_wav(audio) + logger.info("Audio data generated successfully") + return (self.hps.data.sampling_rate, audio) + + +class ModelHolder: + def __init__(self, root_dir: str, device: str): + self.root_dir: str = root_dir + self.device: str = device + self.model_files_dict: Dict[str, List[str]] = {} + self.current_model: Optional[Model] = None + self.model_names: List[str] = [] + self.models: List[Model] = [] + self.refresh() + + def refresh(self): + self.model_files_dict = {} + self.model_names = [] + self.current_model = None + model_dirs = [ + d + for d in os.listdir(self.root_dir) + if os.path.isdir(os.path.join(self.root_dir, d)) + ] + for model_name in model_dirs: + model_dir = os.path.join(self.root_dir, model_name) + model_files = [ + os.path.join(model_dir, f) + for f in os.listdir(model_dir) + if f.endswith(".pth") or f.endswith(".pt") or f.endswith(".safetensors") + ] + if len(model_files) == 0: + logger.info( + f"No model files found in {self.root_dir}/{model_name}, so skip it" + ) + continue + self.model_files_dict[model_name] = model_files + self.model_names.append(model_name) + + def load_model_gr( + self, model_name: str, model_path: str + ) -> tuple[gr.Dropdown, gr.Button]: + if model_name not in self.model_files_dict: + raise Exception(f"モデル名{model_name}は存在しません") + if model_path not in self.model_files_dict[model_name]: + raise Exception(f"pthファイル{model_path}は存在しません") + self.current_model = Model( + model_path=model_path, + config_path=os.path.join(self.root_dir, model_name, "config.json"), + style_vec_path=os.path.join(self.root_dir, model_name, "style_vectors.npy"), + device=self.device, + ) + styles = list(self.current_model.style2id.keys()) + return ( + gr.Dropdown(choices=styles, value=styles[0]), + gr.Button(interactive=True, value="音声合成"), + ) + + def update_model_files_gr(self, model_name: str) -> gr.Dropdown: + model_files = self.model_files_dict[model_name] + return gr.Dropdown(choices=model_files, value=model_files[0]) + + def update_model_names_gr(self) -> tuple[gr.Dropdown, gr.Dropdown, gr.Button]: + self.refresh() + initial_model_name = self.model_names[0] + initial_model_files = self.model_files_dict[initial_model_name] + return ( + gr.Dropdown(choices=self.model_names, value=initial_model_name), + gr.Dropdown(choices=initial_model_files, value=initial_model_files[0]), + gr.Button(interactive=False), # For tts_button + ) diff --git a/config.py b/config.py index e36bdf0..0acfe76 100644 --- a/config.py +++ b/config.py @@ -8,7 +8,7 @@ from typing import Dict, List import yaml -from tools.log import logger +from common.log import logger class Resample_config: diff --git a/data_utils.py b/data_utils.py index dc5aa44..cc658c5 100644 --- a/data_utils.py +++ b/data_utils.py @@ -11,7 +11,7 @@ import commons from config import config from mel_processing import mel_spectrogram_torch, spectrogram_torch from text import cleaned_text_to_sequence -from tools.log import logger +from common.log import logger from utils import load_filepaths_and_text, load_wav_to_torch """Multi speaker version""" diff --git a/default_style.py b/default_style.py index 424f8eb..6caddac 100644 --- a/default_style.py +++ b/default_style.py @@ -1,5 +1,6 @@ import os -from tools.log import logger +from common.log import logger +from common.constants import DEFAULT_STYLE import numpy as np import json @@ -9,10 +10,10 @@ def set_style_config(json_path, output_path): with open(json_path, "r") as f: json_dict = json.load(f) json_dict["data"]["num_styles"] = 1 - json_dict["data"]["style2id"] = {"Neutral": 0} + json_dict["data"]["style2id"] = {DEFAULT_STYLE: 0} with open(output_path, "w") as f: json.dump(json_dict, f, indent=2) - logger.info(f"Update style config (only Neutral style) to {output_path}") + logger.info(f"Save style config (only {DEFAULT_STYLE}) to {output_path}") def save_mean_vector(wav_dir, output_path): diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 487a53f..5582338 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,11 +1,13 @@ # Changelog -## v1.2 (2023-12-30) +## v1.2 (2023-12-31) -- グラボがないCPUユーザーでも音声合成機能は使えるように -- Google colabでの学習をサポート、ノートブックを追加 +- グラボがないユーザーでの音声合成をサポート、`Install-Style-Bert-VITS2-CPU.bat`でインストール。 +- Google Colabでの学習をサポート、[ノートブック](../colab.ipynb)を追加 - 前処理のリサンプリング時に音声ファイルの開始・終了部分の無音を削除するオプションを追加（デフォルトでオン） -- 学習時に自動的にデフォルトスタイル Neutral を生成するように。これで学習したらそのまま音声合成を試せます。 +- 学習時に自動的にデフォルトスタイル Neutral を生成するように。特にスタイル指定が必要のない方は、学習したらそのまま音声合成を試せます。これまで通りスタイルを自分で作ることもできます。 +- マージ機能の新規追加: `Merge.bat`, `webui_merge.py` +- 音声合成のAPIサーバーを追加、`python server_fastapi.py`で起動します。API仕様は起動後に`/docs`にて確認ください。（ @darai0512 様によるPRです、ありがとうございます！） ## v1.1 (2023-12-29) - TrainとDatasetのWebUIの改良・調整（一括事前処理ボタン等） diff --git a/infer.py b/infer.py index b576b29..31ff6c3 100644 --- a/infer.py +++ b/infer.py @@ -23,13 +23,13 @@ def get_net_g(model_path: str, version: str, device: str, hps): if model_path.endswith(".pth") or model_path.endswith(".pt"): _ = utils.load_checkpoint(model_path, net_g, None, skip_optimizer=True) elif model_path.endswith(".safetensors"): - _ = utils.load_safetensors(model_path, net_g, device) + _ = utils.load_safetensors(model_path, net_g, True) else: raise ValueError(f"Unknown model format: {model_path}") return net_g -def get_text(text, language_str, hps, device, style_text=None, style_weight=0.7): +def get_text(text, language_str, hps, device, assist_text=None, assist_text_weight=0.7): # 在此处实现当前版本的get_text norm_text, phone, tone, word2ph = clean_text(text, language_str) phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str) @@ -42,7 +42,7 @@ def get_text(text, language_str, hps, device, style_text=None, style_weight=0.7) word2ph[i] = word2ph[i] * 2 word2ph[0] += 1 bert_ori = get_bert( - norm_text, word2ph, language_str, device, style_text, style_weight + norm_text, word2ph, language_str, device, assist_text, assist_text_weight ) del word2ph assert bert_ori.shape[-1] == len(phone), phone @@ -86,16 +86,16 @@ def infer( device, skip_start=False, skip_end=False, - style_text=None, - style_weight=0.7, + assist_text=None, + assist_text_weight=0.7, ): bert, ja_bert, en_bert, phones, tones, lang_ids = get_text( text, language, hps, device, - style_text=style_text, - style_weight=style_weight, + assist_text=assist_text, + assist_text_weight=assist_text_weight, ) if skip_start: phones = phones[3:] diff --git a/initialize.py b/initialize.py index 8cac785..1656c26 100644 --- a/initialize.py +++ b/initialize.py @@ -3,7 +3,7 @@ from pathlib import Path from huggingface_hub import hf_hub_download -from tools.log import logger +from common.log import logger def download_bert_models(): diff --git a/preprocess_text.py b/preprocess_text.py index d905f19..a0ae9f1 100644 --- a/preprocess_text.py +++ b/preprocess_text.py @@ -9,7 +9,7 @@ from tqdm import tqdm from config import config from text.cleaner import clean_text -from tools.stdout_wrapper import SAFE_STDOUT +from common.stdout_wrapper import SAFE_STDOUT preprocess_text_config = config.preprocess_text_config diff --git a/resample.py b/resample.py index e3d9b94..a123bb3 100644 --- a/resample.py +++ b/resample.py @@ -8,8 +8,8 @@ import soundfile from tqdm import tqdm from config import config -from tools.log import logger -from tools.stdout_wrapper import SAFE_STDOUT +from common.log import logger +from common.stdout_wrapper import SAFE_STDOUT def normalize_audio(data, sr): diff --git a/server_fastapi.py b/server_fastapi.py index 3148f75..b0cecd5 100644 --- a/server_fastapi.py +++ b/server_fastapi.py @@ -1,40 +1,43 @@ """ -api服务 多版本多模型 fastapi实现 +API server for TTS """ import argparse -from fastapi import FastAPI, Query, Request, status, HTTPException -from fastapi.responses import Response, FileResponse -from fastapi.middleware.cors import CORSMiddleware +import os +import sys from io import BytesIO -from scipy.io import wavfile -import uvicorn -import torch -import psutil -import GPUtil -from typing import Dict, Optional, List, Union -import os, sys -from tools.log import logger +from typing import Dict, Optional, Union from urllib.parse import unquote -from config import config -from app import ( - Model, - ModelHolder, - Languages, - DEFAULT_SDP_RATIO, - DEFAULT_NOISE, - DEFAULT_NOISEW, + +import GPUtil +import psutil +import torch +import uvicorn +from fastapi import FastAPI, HTTPException, Query, Request, status +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import FileResponse, Response +from scipy.io import wavfile + +from common.constants import ( + DEFAULT_ASSIST_TEXT_WEIGHT, DEFAULT_LENGTH, DEFAULT_LINE_SPLIT, + DEFAULT_NOISE, + DEFAULT_NOISEW, + DEFAULT_SDP_RATIO, DEFAULT_SPLIT_INTERVAL, + DEFAULT_STYLE, DEFAULT_STYLE_WEIGHT, - DEFAULT_EMOTION_WEIGHT, + Languages, ) -from webui_style_vectors import DEFAULT_EMOTION +from common.log import logger +from common.tts_model import Model, ModelHolder +from config import config ln = config.server_config.language def raise_validation_error(msg: str, param: str): + logger.warning(f"Validation error: {msg}") raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=[dict(type="invalid_params", msg=msg, loc=["query", param])], @@ -125,15 +128,15 @@ if __name__ == "__main__": split_interval: float = Query( DEFAULT_SPLIT_INTERVAL, description="分けた場合に挟む無音の長さ（秒）" ), - style_text: Optional[str] = Query( + assist_text: Optional[str] = Query( None, description="このテキストの読み上げと似た声音・感情になりやすくなる。ただし抑揚やテンポ等が犠牲になる傾向がある" ), - style_weight: float = Query(DEFAULT_STYLE_WEIGHT, description="style_textの強さ"), - emotion: Optional[Union[int, str]] = Query(DEFAULT_EMOTION, description="スタイル"), - emotion_weight: float = Query(DEFAULT_EMOTION_WEIGHT, description="emotionの強さ"), - reference_audio_path: Optional[str] = Query( - None, description="emotionを音声ファイルで行う" + assist_text_weight: float = Query( + DEFAULT_ASSIST_TEXT_WEIGHT, description="assist_textの強さ" ), + style: Optional[Union[int, str]] = Query(DEFAULT_STYLE, description="スタイル"), + style_weight: float = Query(DEFAULT_STYLE_WEIGHT, description="スタイルの強さ"), + reference_audio_path: Optional[str] = Query(None, description="スタイルを音声ファイルで行う"), ): """Infer text to speech(テキストから感情付き音声を生成する)""" logger.info( @@ -154,8 +157,8 @@ if __name__ == "__main__": f"speaker_name={speaker_name} not found", "speaker_name" ) speaker_id = model.spk2id[speaker_name] - if emotion not in model.style2id.keys(): - raise_validation_error(f"emotion={emotion} not found", "emotion") + if style not in model.style2id.keys(): + raise_validation_error(f"style={style} not found", "style") if encoding is not None: text = unquote(text, encoding=encoding) sr, audio = model.infer( @@ -169,12 +172,13 @@ if __name__ == "__main__": length=length, line_split=auto_split, split_interval=split_interval, - style_text=style_text, + assist_text=assist_text, + assist_text_weight=assist_text_weight, + use_assist_text=bool(assist_text), + style=style, style_weight=style_weight, - use_style_text=bool(style_text), - style=emotion, - emotion_weight=emotion_weight, ) + logger.success("Audio data generated and sent successfully") with BytesIO() as wavContent: wavfile.write(wavContent, sr, audio) return Response(content=wavContent.getvalue(), media_type="audio/wav") diff --git a/server_test.ipynb b/server_test.ipynb new file mode 100644 index 0000000..36742eb --- /dev/null +++ b/server_test.ipynb @@ -0,0 +1,74 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import requests\n", + "from IPython.display import Audio\n", + "\n", + "\n", + "def fecth_and_play_voice(params):\n", + " url = \"http://127.0.0.1:5000/voice\"\n", + " response = requests.get(url, params=params)\n", + "\n", + " if response.status_code == 200:\n", + " display(Audio(response.content))\n", + " else:\n", + " print(f\"Error: {response.status_code}\")\n", + " print(response.json())\n", + "\n", + "\n", + "params = {\n", + " \"text\": \"こんにちは、吾輩は猫である。名前はまだない。\",\n", + " \"model_id\": 10,\n", + " \"style\": \"Neutral\",\n", + " \"style_weight\": 100\n", + "}\n", + "\n", + "fecth_and_play_voice(params)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.11" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/slice.py b/slice.py index 3cd2399..b831dcb 100644 --- a/slice.py +++ b/slice.py @@ -6,7 +6,7 @@ import soundfile as sf import torch from tqdm import tqdm -from tools.stdout_wrapper import SAFE_STDOUT +from common.stdout_wrapper import SAFE_STDOUT vad_model, utils = torch.hub.load( repo_or_dir="snakers4/silero-vad", diff --git a/style_gen.py b/style_gen.py index 6fdf9e3..778a6a4 100644 --- a/style_gen.py +++ b/style_gen.py @@ -8,7 +8,7 @@ from tqdm import tqdm import utils from config import config -from tools.stdout_wrapper import SAFE_STDOUT +from common.stdout_wrapper import SAFE_STDOUT warnings.filterwarnings("ignore", category=UserWarning) from pyannote.audio import Inference, Model diff --git a/text/__init__.py b/text/__init__.py index b47b4d4..495e57b 100644 --- a/text/__init__.py +++ b/text/__init__.py @@ -18,13 +18,15 @@ def cleaned_text_to_sequence(cleaned_text, tones, language): return phones, tones, lang_ids -def get_bert(norm_text, word2ph, language, device, style_text=None, style_weight=0.7): +def get_bert( + norm_text, word2ph, language, device, assist_text=None, assist_text_weight=0.7 +): from .chinese_bert import get_bert_feature as zh_bert from .english_bert_mock import get_bert_feature as en_bert from .japanese_bert import get_bert_feature as jp_bert lang_bert_func_map = {"ZH": zh_bert, "EN": en_bert, "JP": jp_bert} bert = lang_bert_func_map[language]( - norm_text, word2ph, device, style_text, style_weight + norm_text, word2ph, device, assist_text, assist_text_weight ) return bert diff --git a/text/chinese_bert.py b/text/chinese_bert.py index baddf11..94e1408 100644 --- a/text/chinese_bert.py +++ b/text/chinese_bert.py @@ -16,8 +16,8 @@ def get_bert_feature( text, word2ph, device=config.bert_gen_config.device, - style_text=None, - style_weight=0.7, + assist_text=None, + assist_text_weight=0.7, ): if ( sys.platform == "darwin" @@ -37,8 +37,8 @@ def get_bert_feature( inputs[i] = inputs[i].to(device) res = models[device](**inputs, output_hidden_states=True) res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu() - if style_text: - style_inputs = tokenizer(style_text, return_tensors="pt") + if assist_text: + style_inputs = tokenizer(assist_text, return_tensors="pt") for i in style_inputs: style_inputs[i] = style_inputs[i].to(device) style_res = models[device](**style_inputs, output_hidden_states=True) @@ -48,10 +48,10 @@ def get_bert_feature( word2phone = word2ph phone_level_feature = [] for i in range(len(word2phone)): - if style_text: + if assist_text: repeat_feature = ( - res[i].repeat(word2phone[i], 1) * (1 - style_weight) - + style_res_mean.repeat(word2phone[i], 1) * style_weight + res[i].repeat(word2phone[i], 1) * (1 - assist_text_weight) + + style_res_mean.repeat(word2phone[i], 1) * assist_text_weight ) else: repeat_feature = res[i].repeat(word2phone[i], 1) diff --git a/text/english_bert_mock.py b/text/english_bert_mock.py index feee598..782b65d 100644 --- a/text/english_bert_mock.py +++ b/text/english_bert_mock.py @@ -17,8 +17,8 @@ def get_bert_feature( text, word2ph, device=config.bert_gen_config.device, - style_text=None, - style_weight=0.7, + assist_text=None, + assist_text_weight=0.7, ): if ( sys.platform == "darwin" @@ -38,8 +38,8 @@ def get_bert_feature( inputs[i] = inputs[i].to(device) res = models[device](**inputs, output_hidden_states=True) res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu() - if style_text: - style_inputs = tokenizer(style_text, return_tensors="pt") + if assist_text: + style_inputs = tokenizer(assist_text, return_tensors="pt") for i in style_inputs: style_inputs[i] = style_inputs[i].to(device) style_res = models[device](**style_inputs, output_hidden_states=True) @@ -49,10 +49,10 @@ def get_bert_feature( word2phone = word2ph phone_level_feature = [] for i in range(len(word2phone)): - if style_text: + if assist_text: repeat_feature = ( - res[i].repeat(word2phone[i], 1) * (1 - style_weight) - + style_res_mean.repeat(word2phone[i], 1) * style_weight + res[i].repeat(word2phone[i], 1) * (1 - assist_text_weight) + + style_res_mean.repeat(word2phone[i], 1) * assist_text_weight ) else: repeat_feature = res[i].repeat(word2phone[i], 1) diff --git a/text/japanese_bert.py b/text/japanese_bert.py index f8127cd..7e528e9 100644 --- a/text/japanese_bert.py +++ b/text/japanese_bert.py @@ -17,12 +17,12 @@ def get_bert_feature( text, word2ph, device=config.bert_gen_config.device, - style_text=None, - style_weight=0.7, + assist_text=None, + assist_text_weight=0.7, ): text = "".join(text2sep_kata(text)[0]) - if style_text: - style_text = "".join(text2sep_kata(style_text)[0]) + if assist_text: + assist_text = "".join(text2sep_kata(assist_text)[0]) if ( sys.platform == "darwin" and torch.backends.mps.is_available() @@ -41,8 +41,8 @@ def get_bert_feature( inputs[i] = inputs[i].to(device) res = models[device](**inputs, output_hidden_states=True) res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu() - if style_text: - style_inputs = tokenizer(style_text, return_tensors="pt") + if assist_text: + style_inputs = tokenizer(assist_text, return_tensors="pt") for i in style_inputs: style_inputs[i] = style_inputs[i].to(device) style_res = models[device](**style_inputs, output_hidden_states=True) @@ -53,10 +53,10 @@ def get_bert_feature( word2phone = word2ph phone_level_feature = [] for i in range(len(word2phone)): - if style_text: + if assist_text: repeat_feature = ( - res[i].repeat(word2phone[i], 1) * (1 - style_weight) - + style_res_mean.repeat(word2phone[i], 1) * style_weight + res[i].repeat(word2phone[i], 1) * (1 - assist_text_weight) + + style_res_mean.repeat(word2phone[i], 1) * assist_text_weight ) else: repeat_feature = res[i].repeat(word2phone[i], 1) diff --git a/train_ms.py b/train_ms.py index 5c49dfc..544a765 100644 --- a/train_ms.py +++ b/train_ms.py @@ -29,8 +29,8 @@ from losses import discriminator_loss, feature_loss, generator_loss, kl_loss from mel_processing import mel_spectrogram_torch, spec_to_mel_torch from models import DurationDiscriminator, MultiPeriodDiscriminator, SynthesizerTrn from text.symbols import symbols -from tools.log import logger -from tools.stdout_wrapper import SAFE_STDOUT +from common.log import logger +from common.stdout_wrapper import SAFE_STDOUT torch.backends.cuda.matmul.allow_tf32 = True torch.backends.cudnn.allow_tf32 = ( @@ -157,6 +157,7 @@ def run(): if rank == 0: # logger = utils.get_logger(hps.model_dir) # logger.info(hps) + logger.add(os.path.join(hps.model_dir, "train.log")) utils.check_git_hash(hps.model_dir) writer = SummaryWriter(log_dir=hps.model_dir) writer_eval = SummaryWriter(log_dir=os.path.join(hps.model_dir, "eval")) diff --git a/transcribe.py b/transcribe.py index 7d2dd1a..275d1be 100644 --- a/transcribe.py +++ b/transcribe.py @@ -5,7 +5,7 @@ import sys from faster_whisper import WhisperModel from tqdm import tqdm -from tools.stdout_wrapper import SAFE_STDOUT +from common.stdout_wrapper import SAFE_STDOUT def transcribe(wav_path, initial_prompt=None): diff --git a/utils.py b/utils.py index ea83465..33f6e10 100644 --- a/utils.py +++ b/utils.py @@ -13,7 +13,7 @@ from safetensors import safe_open from safetensors.torch import save_file from scipy.io.wavfile import read -from tools.log import logger +from common.log import logger MATPLOTLIB_FLAG = False diff --git a/webui.py b/webui.py index fc1a0c7..e7c1abd 100644 --- a/webui.py +++ b/webui.py @@ -71,8 +71,8 @@ def generate_audio( device=device, skip_start=skip_start, skip_end=skip_end, - style_text=style_text, - style_weight=style_weight, + assist_text=style_text, + assist_text_weight=style_weight, ) audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio) audio_list.append(audio16bit) diff --git a/webui_dataset.py b/webui_dataset.py index 436944d..a711818 100644 --- a/webui_dataset.py +++ b/webui_dataset.py @@ -2,8 +2,8 @@ import os import gradio as gr -from tools.log import logger -from tools.subprocess_utils import run_script_with_log, second_elem_of +from common.log import logger +from common.subprocess_utils import run_script_with_log, second_elem_of def do_slice(model_name, normalize): diff --git a/webui_merge.py b/webui_merge.py new file mode 100644 index 0000000..2020c36 --- /dev/null +++ b/webui_merge.py @@ -0,0 +1,389 @@ +import json +import os +import sys + +import gradio as gr +import numpy as np +import torch +from safetensors import safe_open +from safetensors.torch import save_file + + +from config import config +from common.tts_model import Model, ModelHolder +from common.log import logger +from common.constants import DEFAULT_STYLE + +voice_keys = ["dec", "flow"] +speech_style_keys = ["enc_p"] +tempo_keys = ["sdp", "dp"] + +device = "cuda" if torch.cuda.is_available() else "cpu" + +model_dir = config.out_dir +model_holder = ModelHolder(model_dir, device) + + +def merge_style(model_name_a, model_name_b, weight, output_name, style_triple_list): + """ + style_triple_list: list[(model_aでのスタイル名, model_bでのスタイル名, 出力するスタイル名)] + """ + # 新スタイル名リストにNeutralが含まれているか確認し、Neutralを先頭に持ってくる + if any(triple[2] == DEFAULT_STYLE for triple in style_triple_list): + # 存在する場合、リストをソート + sorted_list = sorted(style_triple_list, key=lambda x: x[2] != DEFAULT_STYLE) + else: + # 存在しない場合、エラーを発生 + raise ValueError("No element with {DEFAULT_STYLE} output style name found.") + + style_vectors_a = np.load( + os.path.join(model_dir, model_name_a, "style_vectors.npy") + ) # (style_num_a, 256) + style_vectors_b = np.load( + os.path.join(model_dir, model_name_b, "style_vectors.npy") + ) # (style_num_b, 256) + with open(os.path.join(model_dir, model_name_a, "config.json")) as f: + config_a = json.load(f) + with open(os.path.join(model_dir, model_name_b, "config.json")) as f: + config_b = json.load(f) + style2id_a = config_a["data"]["style2id"] + style2id_b = config_b["data"]["style2id"] + new_style_vecs = [] + new_style2id = {} + for style_a, style_b, style_out in sorted_list: + if style_a not in style2id_a: + logger.error(f"{style_a} is not in {model_name_a}.") + raise ValueError(f"{style_a} は {model_name_a} にありません。") + if style_b not in style2id_b: + logger.error(f"{style_b} is not in {model_name_b}.") + raise ValueError(f"{style_b} は {model_name_b} にありません。") + new_style = ( + style_vectors_a[style2id_a[style_a]] * (1 - weight) + + style_vectors_b[style2id_b[style_b]] * weight + ) + new_style_vecs.append(new_style) + new_style2id[style_out] = len(new_style_vecs) - 1 + new_style_vecs = np.array(new_style_vecs) + + output_style_path = os.path.join(model_dir, output_name, "style_vectors.npy") + np.save(output_style_path, new_style_vecs) + + new_config = config_a.copy() + new_config["data"]["num_styles"] = len(new_style2id) + new_config["data"]["style2id"] = new_style2id + new_config["model_name"] = output_name + with open(os.path.join(model_dir, output_name, "config.json"), "w") as f: + json.dump(new_config, f, indent=2) + + return output_style_path, list(new_style2id.keys()) + + +def merge_models( + model_path_a, + model_path_b, + voice_weight, + speech_style_weight, + tempo_weight, + output_name, +): + """model Aを起点に、model Bの各要素を重み付けしてマージする。 + safetensors形式を前提とする。""" + model_a_weight = {} + with safe_open(model_path_a, framework="pt", device="cpu") as f: + for k in f.keys(): + model_a_weight[k] = f.get_tensor(k) + + model_b_weight = {} + with safe_open(model_path_b, framework="pt", device="cpu") as f: + for k in f.keys(): + model_b_weight[k] = f.get_tensor(k) + + merged_model_weight = model_a_weight.copy() + + for key in model_a_weight.keys(): + if any([key.startswith(prefix) for prefix in voice_keys]): + weight = voice_weight + elif any([key.startswith(prefix) for prefix in speech_style_keys]): + weight = speech_style_weight + elif any([key.startswith(prefix) for prefix in tempo_keys]): + weight = tempo_weight + else: + continue + merged_model_weight[key] = ( + model_a_weight[key] * (1 - weight) + model_b_weight[key] * weight + ) + + merged_model_path = os.path.join( + model_dir, output_name, f"{output_name}.safetensors" + ) + os.makedirs(os.path.dirname(merged_model_path), exist_ok=True) + save_file(merged_model_weight, merged_model_path) + return merged_model_path + + +def merge_models_gr( + model_name_a, + model_path_a, + model_name_b, + model_path_b, + output_name, + voice_weight, + speech_style_weight, + tempo_weight, +): + merged_model_path = merge_models( + model_path_a, + model_path_b, + voice_weight, + speech_style_weight, + tempo_weight, + output_name, + ) + return f"Success: モデルを{merged_model_path}に保存しました。" + + +def merge_style_gr( + model_name_a, + model_name_b, + weight, + output_name, + style_triple_list_str: str, +): + style_triple_list = [] + for line in style_triple_list_str.split("\n"): + if not line: + continue + style_triple = line.split(",") + if len(style_triple) != 3: + logger.error(f"Invalid style triple: {line}") + return f"Error: スタイルを3つのカンマ区切りで入力してください:\n{line}", None + style_a, style_b, style_out = style_triple + style_a = style_a.strip() + style_b = style_b.strip() + style_out = style_out.strip() + style_triple_list.append((style_a, style_b, style_out)) + try: + new_style_path, new_styles = merge_style( + model_name_a, model_name_b, weight, output_name, style_triple_list + ) + except ValueError as e: + return f"Error: {e}" + return f"Success: スタイルを{new_style_path}に保存しました。", gr.Dropdown( + choices=new_styles, value=new_styles[0] + ) + + +def simple_tts(model_name, text, style=DEFAULT_STYLE, emotion_weight=1.0): + model_path = os.path.join(model_dir, model_name, f"{model_name}.safetensors") + config_path = os.path.join(model_dir, model_name, "config.json") + style_vec_path = os.path.join(model_dir, model_name, "style_vectors.npy") + + model = Model(model_path, config_path, style_vec_path, device) + return model.infer(text, style=style, style_weight=emotion_weight) + + +def update_two_model_names_dropdown(): + new_names, new_files, _ = model_holder.update_model_names_gr() + return new_names, new_files, new_names, new_files + + +def get_styles_gr(model_name_a, model_name_b): + config_path_a = os.path.join(model_dir, model_name_a, "config.json") + with open(config_path_a) as f: + config_a = json.load(f) + styles_a = list(config_a["data"]["style2id"].keys()) + + config_path_b = os.path.join(model_dir, model_name_b, "config.json") + with open(config_path_b) as f: + config_b = json.load(f) + styles_b = list(config_b["data"]["style2id"].keys()) + return gr.Textbox(value=", ".join(styles_a)), gr.Textbox(value=", ".join(styles_b)) + + +initial_md = """ +# Style-Bert-VITS2 モデルマージツール + +2つのStyle-Bert-VITS2モデルから、声質・話し方・話す速さを取り替えたり混ぜたりできます。 + +## 使い方 + +1. マージしたい2つのモデルを選択してください（`model_assets`フォルダの中から選ばれます）。 +2. マージ後のモデルの名前を入力してください。 +3. マージ後のモデルの声質・話し方・話す速さを調整してください。 +4. 「モデルファイルのマージ」ボタンを押してください（safetensorsファイルがマージされる）。 +5. スタイルベクトルファイルも生成する必要があるので、指示に従ってマージ方法を入力後、「スタイルのマージ」ボタンを押してください。 + +以上でマージは完了で、`model_assets/マージ後のモデル名`にマージ後のモデルが保存され、音声合成のときに使えます。 + +一番下にマージしたモデルによる簡易的な音声合成機能もつけています。 +""" + +style_merge_md = f""" +## スタイルベクトルのマージ + +1行に「モデルAのスタイル名, モデルBのスタイル名, 左の2つを混ぜて出力するスタイル名」 +という形式で入力してください。例えば、 +``` +{DEFAULT_STYLE}, {DEFAULT_STYLE}, {DEFAULT_STYLE} +Happy, Surprise, HappySurprise +``` +と入力すると、マージ後のスタイルベクトルは、 +- `{DEFAULT_STYLE}`: モデルAの`{DEFAULT_STYLE}`とモデルBの`{DEFAULT_STYLE}`を混ぜたもの +- `HappySurprise`: モデルAの`Happy`とモデルBの`Surprise`を混ぜたもの +の2つになります。 + +### 注意 +- 必ず「{DEFAULT_STYLE}」という名前のスタイルを作ってください。これは、マージ後のモデルの平均スタイルになります。 +- 構造上の相性の関係で、スタイルベクトルを混ぜる重みは、上の「話し方」と同じ比率で混ぜられます。例えば「話し方」が0のときはモデルAのみしか使われません。 +""" + +model_names = model_holder.model_names +if len(model_names) == 0: + logger.error(f"モデルが見つかりませんでした。{model_dir}にモデルを置いてください。") + sys.exit(1) +initial_id = 0 +initial_model_files = model_holder.model_files_dict[model_names[initial_id]] + +with gr.Blocks(theme="NoCrypt/miku") as app: + gr.Markdown(initial_md) + with gr.Accordion(label="使い方", open=False): + gr.Markdown(initial_md) + with gr.Row(): + with gr.Column(scale=3): + model_name_a = gr.Dropdown( + label="モデルA", + choices=model_names, + value=model_names[initial_id], + ) + model_path_a = gr.Dropdown( + label="モデルファイル", + choices=initial_model_files, + value=initial_model_files[0], + ) + with gr.Column(scale=3): + model_name_b = gr.Dropdown( + label="モデルB", + choices=model_names, + value=model_names[initial_id], + ) + model_path_b = gr.Dropdown( + label="モデルファイル", + choices=initial_model_files, + value=initial_model_files[0], + ) + refresh_button = gr.Button("更新", scale=1, visible=True) + with gr.Column(variant="panel"): + new_name = gr.Textbox(label="新しいモデル名", placeholder="new_model") + with gr.Row(): + voice_slider = gr.Slider( + label="声質", + value=0, + minimum=0, + maximum=1, + step=0.1, + ) + speech_style_slider = gr.Slider( + label="話し方（抑揚・感情表現等）", + value=0, + minimum=0, + maximum=1, + step=0.1, + ) + tempo_slider = gr.Slider( + label="話す速さ・リズム・テンポ", + value=0, + minimum=0, + maximum=1, + step=0.1, + ) + with gr.Column(variant="panel"): + gr.Markdown("## モデルファイル（safetensors）のマージ") + model_merge_button = gr.Button("モデルファイルのマージ", variant="primary") + info_model_merge = gr.Textbox(label="情報") + with gr.Column(variant="panel"): + gr.Markdown(style_merge_md) + with gr.Row(): + load_button = gr.Button("スタイル一覧をロード", scale=1) + styles_a = gr.Textbox(label="モデルAのスタイル一覧") + styles_b = gr.Textbox(label="モデルBのスタイル一覧") + style_triple_list = gr.TextArea( + label="スタイルのマージリスト", + placeholder=f"{DEFAULT_STYLE}, {DEFAULT_STYLE},{DEFAULT_STYLE}\nAngry, Angry, Angry", + value=f"{DEFAULT_STYLE}, {DEFAULT_STYLE}, {DEFAULT_STYLE}", + ) + style_merge_button = gr.Button("スタイルのマージ", variant="primary") + info_style_merge = gr.Textbox(label="情報") + + text_input = gr.TextArea(label="テキスト", value="これはテストです。聞こえていますか？") + style = gr.Dropdown( + label="スタイル", + choices=["スタイルをマージしてください"], + value="スタイルをマージしてください", + ) + emotion_weight = gr.Slider( + minimum=0, + maximum=50, + value=1, + step=0.1, + label="スタイルの強さ", + ) + tts_button = gr.Button("音声合成", variant="primary") + audio_output = gr.Audio(label="結果") + + model_name_a.change( + model_holder.update_model_files_gr, + inputs=[model_name_a], + outputs=[model_path_a], + ) + model_name_b.change( + model_holder.update_model_files_gr, + inputs=[model_name_b], + outputs=[model_path_b], + ) + + refresh_button.click( + update_two_model_names_dropdown, + outputs=[model_name_a, model_path_a, model_name_b, model_path_b], + ) + + load_button.click( + get_styles_gr, + inputs=[model_name_a, model_name_b], + outputs=[styles_a, styles_b], + ) + + model_merge_button.click( + merge_models_gr, + inputs=[ + model_name_a, + model_path_a, + model_name_b, + model_path_b, + new_name, + voice_slider, + speech_style_slider, + tempo_slider, + ], + outputs=[info_model_merge], + ) + + style_merge_button.click( + merge_style_gr, + inputs=[ + model_name_a, + model_name_b, + speech_style_slider, + new_name, + style_triple_list, + ], + outputs=[info_style_merge, style], + ) + + tts_button.click( + simple_tts, + inputs=[new_name, text_input, style, emotion_weight], + outputs=[audio_output], + ) + + +app.launch(inbrowser=True) diff --git a/webui_style_vectors.py b/webui_style_vectors.py index c04be12..3ca3f15 100644 --- a/webui_style_vectors.py +++ b/webui_style_vectors.py @@ -7,9 +7,9 @@ import numpy as np from sklearn.cluster import AgglomerativeClustering, KMeans from sklearn.manifold import TSNE from config import config +from common.constants import DEFAULT_STYLE MAX_CLUSTER_NUM = 10 -DEFAULT_EMOTION: str = "Neutral" tsne = TSNE(n_components=2, random_state=42, metric="cosine") @@ -125,7 +125,7 @@ def save_style_vectors(model_name, style_names: str): config_path = os.path.join(result_dir, "config.json") if not os.path.exists(config_path): return f"{config_path}が存在しません。" - style_name_list = [DEFAULT_EMOTION] + style_name_list = [DEFAULT_STYLE] style_name_list = style_name_list + style_names.split(",") if len(style_name_list) != len(centroids) + 1: return f"スタイルの数が合いません。`,`で正しく{len(centroids)}個に区切られているか確認してください: {style_names}" @@ -174,7 +174,7 @@ def save_style_vectors_from_files(model_name, audio_files_text, style_names_text config_path = os.path.join(result_dir, "config.json") if not os.path.exists(config_path): return f"{config_path}が存在しません。" - style_name_list = [DEFAULT_EMOTION] + style_name_list = [DEFAULT_STYLE] style_name_list = style_name_list + style_names assert len(style_name_list) == len(style_vectors) @@ -194,7 +194,7 @@ initial_md = f""" Style-Bert-VITS2でこまかくスタイルを指定して音声合成するには、モデルごとにスタイルベクトルのファイル`style_vectors.npy`を手動で作成する必要があります。 -ただし、学習の過程で自動的に平均スタイル「Neutral」のみは作成されるので、それをそのまま使うこともできます（その場合はこのWebUIは使いません）。 +ただし、学習の過程で自動的に平均スタイル「{DEFAULT_STYLE}」のみは作成されるので、それをそのまま使うこともできます（その場合はこのWebUIは使いません）。 このプロセスは学習とは全く関係がないので、何回でも独立して繰り返して試せます。また学習中にもたぶん軽いので動くはずです。 @@ -275,7 +275,7 @@ with gr.Blocks(theme="NoCrypt/miku") as app: style_names = gr.Textbox( "Angry, Sad, Happy", label="スタイルの名前", - info=f"スタイルの名前を`,`で区切って入力してください（日本語可）。例: `Angry, Sad, Happy`や`怒り, 悲しみ, 喜び`など。平均音声は{DEFAULT_EMOTION}として自動的に保存されます。", + info=f"スタイルの名前を`,`で区切って入力してください（日本語可）。例: `Angry, Sad, Happy`や`怒り, 悲しみ, 喜び`など。平均音声は{DEFAULT_STYLE}として自動的に保存されます。", ) with gr.Row(): save_button = gr.Button("スタイルベクトルを保存", variant="primary") @@ -288,7 +288,7 @@ with gr.Blocks(theme="NoCrypt/miku") as app: gr.Markdown("下のテキスト欄に、各スタイルの代表音声のファイル名を`,`区切りで、その横に対応するスタイル名を`,`区切りで入力してください。") gr.Markdown("例: `angry.wav, sad.wav, happy.wav`と`Angry, Sad, Happy`") gr.Markdown( - f"注意: {DEFAULT_EMOTION}スタイルは自動的に保存されます、手動では{DEFAULT_EMOTION}という名前のスタイルは指定しないでください。" + f"注意: {DEFAULT_STYLE}スタイルは自動的に保存されます、手動では{DEFAULT_STYLE}という名前のスタイルは指定しないでください。" ) with gr.Row(): audio_files_text = gr.Textbox( diff --git a/webui_train.py b/webui_train.py index acc49fd..a28ec7c 100644 --- a/webui_train.py +++ b/webui_train.py @@ -7,8 +7,8 @@ from multiprocessing import cpu_count import gradio as gr import yaml -from tools.log import logger -from tools.subprocess_utils import run_script_with_log, second_elem_of +from common.log import logger +from common.subprocess_utils import run_script_with_log, second_elem_of try: import google.colab