From c832706c7482798ebc455c226e0228ea334d1f27 Mon Sep 17 00:00:00 2001 From: RedRayz <71994877+RedRayz@users.noreply.github.com> Date: Mon, 12 Feb 2024 21:15:09 +0900 Subject: [PATCH 01/51] Create gen_yaml.py --- gen_yaml.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 gen_yaml.py diff --git a/gen_yaml.py b/gen_yaml.py new file mode 100644 index 0000000..8650dc0 --- /dev/null +++ b/gen_yaml.py @@ -0,0 +1,34 @@ +import os +import shutil +import yaml +import argparse + +parser = argparse.ArgumentParser(description="config.ymlの生成。あらかじめ前準備をしたデータをバッチファイルなどで連続で学習する時にtrain_ms.pyより前に使用する。") +# そうしないと最後の前準備したデータで学習してしまう +parser.add_argument( + "--model_name", + type=str, + help="Model name", + required=True +) +parser.add_argument( + "--dataset_path", + type=str, + help="Dataset path(example: Data\\your_model_name)", + required=True +) +args = parser.parse_args() + +def gen_yaml(model_name, dataset_path): + if not os.path.exists("config.yml"): + shutil.copy(src="default_config.yml", dst="config.yml") + with open("config.yml", "r", encoding="utf-8") as f: + yml_data = yaml.safe_load(f) + yml_data["model_name"] = model_name + yml_data["dataset_path"] = dataset_path + with open("config.yml", "w", encoding="utf-8") as f: + yaml.dump(yml_data, f, allow_unicode=True) + +if __name__ == "__main__": + gen_yaml(args.model_name, args.dataset_path) + From 6555114666aab813f3f679814c6ab6dc69447fba Mon Sep 17 00:00:00 2001 From: litagin02 Date: Wed, 14 Feb 2024 12:22:35 +0900 Subject: [PATCH 02/51] Feat: option for skipping invalid transcription --- preprocess_text.py | 27 ++++++++++++++++++++++----- webui_train.py | 25 ++++++++++++++++++++++--- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/preprocess_text.py b/preprocess_text.py index 9fba5eb..4f49100 100644 --- a/preprocess_text.py +++ b/preprocess_text.py @@ -15,6 +15,12 @@ from common.log import logger preprocess_text_config = config.preprocess_text_config +# Count lines for tqdm +def count_lines(file_path): + with file_path.open("r", encoding="utf-8") as file: + return sum(1 for _ in file) + + @click.command() @click.option( "--transcription-path", @@ -34,6 +40,7 @@ preprocess_text_config = config.preprocess_text_config @click.option("--clean/--no-clean", default=preprocess_text_config.clean) @click.option("-y", "--yml_config") @click.option("--use_jp_extra", is_flag=True) +@click.option("--skip_invalid", is_flag=True) def preprocess( transcription_path: str, cleaned_path: Optional[str], @@ -45,14 +52,16 @@ def preprocess( clean: bool, yml_config: str, # 这个不要删 use_jp_extra: bool, + skip_invalid: bool = False, ): if cleaned_path == "" or cleaned_path is None: cleaned_path = transcription_path + ".cleaned" if clean: + total_lines = count_lines(transcription_path) with open(cleaned_path, "w", encoding="utf-8") as out_file: with open(transcription_path, "r", encoding="utf-8") as trans_file: - for line in tqdm(trans_file, file=SAFE_STDOUT): + for line in tqdm(trans_file, file=SAFE_STDOUT, total=total_lines): try: utt, spk, language, text = line.strip().split("|") norm_text, phones, tones, word2ph = clean_text( @@ -70,10 +79,18 @@ def preprocess( ) ) except Exception as e: - logger.error( - f"An error occurred while generating the training set and validation set, at line:\n{line}\nDetails:\n{e}" - ) - raise + if skip_invalid: + logger.warning( + f"An error occurred while generating the training set and validation set, at line:\n{line}\nDetails:\n{e}" + ) + logger.warning( + "This line will be skipped from the training set and validation set." + ) + else: + logger.error( + f"An error occurred while generating the training set and validation set, at line:\n{line}\nDetails:\n{e}" + ) + raise transcription_path = cleaned_path spk_utt_map = defaultdict(list) diff --git a/webui_train.py b/webui_train.py index ebd292e..6ca1216 100644 --- a/webui_train.py +++ b/webui_train.py @@ -150,7 +150,7 @@ def resample(model_name, normalize, trim, num_processes): return True, "Step 2, Success: 音声ファイルの前処理が完了しました" -def preprocess_text(model_name, use_jp_extra, val_per_lang): +def preprocess_text(model_name, use_jp_extra, val_per_lang, skip_invalid=False): logger.info("Step 3: start preprocessing text...") dataset_path, lbl_path, train_path, val_path, config_path = get_path(model_name) try: @@ -180,6 +180,8 @@ def preprocess_text(model_name, use_jp_extra, val_per_lang): ] if use_jp_extra: cmd.append("--use_jp_extra") + if skip_invalid: + cmd.append("--skip_invalid") success, message = run_script_with_log(cmd) if not success: logger.error(f"Step 3: preprocessing text failed.") @@ -265,6 +267,7 @@ def preprocess_all( use_jp_extra, val_per_lang, log_interval, + skip_invalid=False, ): if model_name == "": return False, "Error: モデル名を入力してください" @@ -285,7 +288,9 @@ def preprocess_all( success, message = resample(model_name, normalize, trim, num_processes) if not success: return False, message - success, message = preprocess_text(model_name, use_jp_extra, val_per_lang) + success, message = preprocess_text( + model_name, use_jp_extra, val_per_lang, skip_invalid + ) if not success: return False, message success, message = bert_gen(model_name) # bert_genは重いのでプロセス数いじらない @@ -490,6 +495,10 @@ if __name__ == "__main__": maximum=1000, step=10, ) + skip_invalid = gr.Checkbox( + label="日本語処理でエラーが出たファイルは学習に使わない", + value=False, + ) gr.Markdown("学習時に特定の部分を凍結させるかどうか") freeze_EN_bert = gr.Checkbox( label="英語bert部分を凍結", @@ -599,6 +608,10 @@ if __name__ == "__main__": maximum=100, step=1, ) + skip_invalid_manual = gr.Checkbox( + label="日本語処理でエラーが出たファイルは学習に使わない", + value=False, + ) with gr.Column(): preprocess_text_btn = gr.Button(value="実行", variant="primary") info_preprocess_text = gr.Textbox(label="状況") @@ -661,6 +674,7 @@ if __name__ == "__main__": use_jp_extra, val_per_lang, log_interval, + skip_invalid, ], outputs=[info_all], ) @@ -694,7 +708,12 @@ if __name__ == "__main__": ) preprocess_text_btn.click( second_elem_of(preprocess_text), - inputs=[model_name, use_jp_extra_manual, val_per_lang_manual], + inputs=[ + model_name, + use_jp_extra_manual, + val_per_lang_manual, + skip_invalid_manual, + ], outputs=[info_preprocess_text], ) bert_gen_btn.click( From c474bbe5af18e301ea297249fbcc9125356cc001 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Thu, 15 Feb 2024 22:55:08 +0900 Subject: [PATCH 03/51] Feat: pitch intonation changer --- app.py | 20 ++++++++++++++++++ common/tts_model.py | 51 +++++++++++++++++++++++++++++++++++++++------ requirements.txt | 1 + 3 files changed, 66 insertions(+), 6 deletions(-) diff --git a/app.py b/app.py index f9ceb5e..d2e9c19 100644 --- a/app.py +++ b/app.py @@ -57,6 +57,8 @@ def tts_fn( kata_tone_json_str, use_tone, speaker, + pitch_scale, + intonation_scale, ): model_holder.load_model_gr(model_name, model_path) @@ -111,6 +113,8 @@ def tts_fn( style_weight=style_weight, given_tone=tone, sid=speaker_id, + pitch_scale=pitch_scale, + intonation_scale=intonation_scale, ) except InvalidToneError as e: logger.error(f"Tone error: {e}") @@ -306,6 +310,20 @@ if __name__ == "__main__": refresh_button = gr.Button("更新", scale=1, visible=True) load_button = gr.Button("ロード", scale=1, variant="primary") text_input = gr.TextArea(label="テキスト", value=initial_text) + pitch_scale = gr.Slider( + minimum=0, + maximum=2, + value=1, + step=0.1, + label="音程", + ) + intonation_scale = gr.Slider( + minimum=0, + maximum=2, + value=1, + step=0.1, + label="抑揚", + ) line_split = gr.Checkbox( label="改行で分けて生成（分けたほうが感情が乗ります）", @@ -441,6 +459,8 @@ if __name__ == "__main__": tone, use_tone, speaker, + pitch_scale, + intonation_scale, ], outputs=[text_output, audio_output, tone], ) diff --git a/common/tts_model.py b/common/tts_model.py index c148598..1bf5898 100644 --- a/common/tts_model.py +++ b/common/tts_model.py @@ -1,17 +1,18 @@ -import numpy as np -import gradio as gr -import torch import os import warnings -from gradio.processing_utils import convert_to_16_bit_wav from typing import Dict, List, Optional, Union +import gradio as gr +import numpy as np +import pyworld +import torch +from gradio.processing_utils import convert_to_16_bit_wav + import utils from infer import get_net_g, infer from models import SynthesizerTrn from models_jp_extra import SynthesizerTrn as SynthesizerTrnJPExtra -from .log import logger from .constants import ( DEFAULT_ASSIST_TEXT_WEIGHT, DEFAULT_LENGTH, @@ -23,6 +24,36 @@ from .constants import ( DEFAULT_STYLE, DEFAULT_STYLE_WEIGHT, ) +from .log import logger +from torchfcpe import spawn_bundled_infer_model +import librosa + + +def adjust_voice(fs, wave, pitch_scale, intonation_scale): + if pitch_scale == 1.0 and intonation_scale == 1.0: + return fs, wave + # pyworldでf0を加工して合成 + # pyworldよりもよいのがあるかもしれないが…… + + wave = wave.astype(np.double) + logger.debug(f"wave: shape={wave.shape}, max={max(wave)}, min={min(wave)}") + f0, t = pyworld.harvest(wave, fs) + # 質が高そうだしとりあえずharvestにしておく + + sp = pyworld.cheaptrick(wave, f0, t, fs) + ap = pyworld.d4c(wave, f0, t, fs) + + non_zero_f0 = [f for f in f0 if f != 0] + f0_mean = sum(non_zero_f0) / len(non_zero_f0) + logger.debug(f"f0: shape={f0.shape}, mean={f0_mean}, max={max(f0)}") + + for i, f in enumerate(f0): + if f == 0: + continue + f0[i] = pitch_scale * f0_mean + intonation_scale * (f - f0_mean) + + wave = pyworld.synthesize(f0, sp, ap, fs) + return fs, wave class Model: @@ -97,6 +128,8 @@ class Model: style: str = DEFAULT_STYLE, style_weight: float = DEFAULT_STYLE_WEIGHT, given_tone: Optional[list[int]] = None, + pitch_scale: float = 1.0, + intonation_scale: float = 1.0, ) -> tuple[int, np.ndarray]: logger.info(f"Start generating audio data from text:\n{text}") if language != "JP" and self.hps.version.endswith("JP-Extra"): @@ -165,7 +198,13 @@ class Model: warnings.simplefilter("ignore") audio = convert_to_16_bit_wav(audio) logger.info("Audio data generated successfully") - return (self.hps.data.sampling_rate, audio) + fs, audio = adjust_voice( + fs=self.hps.data.sampling_rate, + wave=audio, + pitch_scale=pitch_scale, + intonation_scale=intonation_scale, + ) + return (fs, audio) class ModelHolder: diff --git a/requirements.txt b/requirements.txt index aa84daf..62b4d60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,6 +19,7 @@ pyannote.audio>=3.1.0 pyloudnorm pyopenjtalk-prebuilt pypinyin +pyworld PyYAML requests safetensors From ff730476e7aeaf48e604934dc7519a35b0118aca Mon Sep 17 00:00:00 2001 From: litagin02 Date: Sat, 17 Feb 2024 11:01:16 +0900 Subject: [PATCH 04/51] Feat: huggingface hub push --- train_ms.py | 5 +++++ train_ms_jp_extra.py | 32 +++++++++++++++++++++++++++++--- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/train_ms.py b/train_ms.py index 57eb5f6..01c8cf4 100644 --- a/train_ms.py +++ b/train_ms.py @@ -84,6 +84,11 @@ def run(): action="store_true", help="Speed up training by disabling logging and evaluation.", ) + parser.add_argument( + "--repo_id", + help="Huggingface model repo id to backup the model.", + default=None, + ) args = parser.parse_args() # Set log file diff --git a/train_ms_jp_extra.py b/train_ms_jp_extra.py index 2c46eb2..4a95371 100644 --- a/train_ms_jp_extra.py +++ b/train_ms_jp_extra.py @@ -12,6 +12,7 @@ from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.data import DataLoader from torch.utils.tensorboard import SummaryWriter from tqdm import tqdm +from huggingface_hub import HfApi # logging.getLogger("numba").setLevel(logging.WARNING) import commons @@ -48,6 +49,8 @@ torch.backends.cuda.enable_mem_efficient_sdp( ) # Not available if torch version is lower than 2.0 global_step = 0 +api = HfApi() + def run(): # Command line configuration is not recommended unless necessary, use config.yml @@ -87,6 +90,11 @@ def run(): action="store_true", help="Speed up training by disabling logging and evaluation.", ) + parser.add_argument( + "--repo_id", + help="Huggingface model repo id to backup the model.", + default=None, + ) args = parser.parse_args() # Set log file @@ -126,6 +134,7 @@ def run(): # This is needed because we have to pass values to `train_and_evaluate() hps.model_dir = model_dir hps.speedup = args.speedup + hps.repo_id = args.repo_id # 比较路径是否相同 if os.path.realpath(args.config) != os.path.realpath( @@ -565,6 +574,15 @@ def run(): ), for_infer=True, ) + if hps.repo_id is not None: + api.upload_folder( + repo_id=hps.repo_id, + folder_path=model_dir, + ) + api.upload_folder( + repo_id=hps.repo_id, + folder_path=config.out_dir, + ) if pbar is not None: pbar.close() @@ -916,6 +934,15 @@ def train_and_evaluate( ), for_infer=True, ) + if hps.repo_id is not None: + api.upload_folder( + repo_id=hps.repo_id, + folder_path=model_dir, + ) + api.upload_folder( + repo_id=hps.repo_id, + folder_path=config.out_dir, + ) global_step += 1 if pbar is not None: @@ -926,9 +953,8 @@ def train_and_evaluate( ) pbar.update() # 本家ではこれをスピードアップのために消すと書かれていたので、一応消してみる - # と思ったけどメモリ使用量が減るかもしれないのでつけてみる - gc.collect() - torch.cuda.empty_cache() + # gc.collect() + # torch.cuda.empty_cache() if pbar is None and rank == 0: logger.info(f"====> Epoch: {epoch}, step: {global_step}") From 8b76109b7512a91749de69d6f004cda35c776147 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Sat, 17 Feb 2024 11:31:35 +0900 Subject: [PATCH 05/51] Fix hf push --- train_ms.py | 27 +++++++++++++++++++++++++++ train_ms_jp_extra.py | 26 ++++++++++++++++---------- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/train_ms.py b/train_ms.py index 01c8cf4..08d6af9 100644 --- a/train_ms.py +++ b/train_ms.py @@ -6,6 +6,7 @@ import platform import torch import torch.distributed as dist +from huggingface_hub import HfApi from torch.cuda.amp import GradScaler, autocast from torch.nn import functional as F from torch.nn.parallel import DistributedDataParallel as DDP @@ -45,6 +46,8 @@ torch.backends.cuda.enable_math_sdp(True) global_step = 0 +api = HfApi() + def run(): # Command line configuration is not recommended unless necessary, use config.yml @@ -483,6 +486,18 @@ def run(): ), for_infer=True, ) + if hps.repo_id is not None: + api.upload_folder( + repo_id=hps.repo_id, + folder_path=model_dir, + path_in_repo=f"Data/{config.model_name}/models", + delete_patterns="*.pth", + ) + api.upload_folder( + repo_id=hps.repo_id, + folder_path=config.out_dir, + path_in_repo=f"model_assets/{config.model_name}", + ) if pbar is not None: pbar.close() @@ -765,6 +780,18 @@ def train_and_evaluate( ), for_infer=True, ) + if hps.repo_id is not None: + api.upload_folder( + repo_id=hps.repo_id, + folder_path=hps.model_dir, + path_in_repo=f"Data/{config.model_name}/models", + delete_patterns="*.pth", + ) + api.upload_folder( + repo_id=hps.repo_id, + folder_path=config.out_dir, + path_in_repo=f"model_assets/{config.model_name}", + ) global_step += 1 if pbar is not None: diff --git a/train_ms_jp_extra.py b/train_ms_jp_extra.py index 4a95371..9c116ab 100644 --- a/train_ms_jp_extra.py +++ b/train_ms_jp_extra.py @@ -574,15 +574,18 @@ def run(): ), for_infer=True, ) - if hps.repo_id is not None: - api.upload_folder( - repo_id=hps.repo_id, - folder_path=model_dir, - ) - api.upload_folder( - repo_id=hps.repo_id, - folder_path=config.out_dir, - ) + if hps.repo_id is not None: + api.upload_folder( + repo_id=hps.repo_id, + folder_path=model_dir, + path_in_repo=f"Data/{config.model_name}/models", + delete_patterns="*.pth", + ) + api.upload_folder( + repo_id=hps.repo_id, + folder_path=config.out_dir, + path_in_repo=f"model_assets/{config.model_name}", + ) if pbar is not None: pbar.close() @@ -937,11 +940,14 @@ def train_and_evaluate( if hps.repo_id is not None: api.upload_folder( repo_id=hps.repo_id, - folder_path=model_dir, + folder_path=hps.model_dir, + path_in_repo=f"Data/{config.model_name}/models", + delete_patterns="*.pth", ) api.upload_folder( repo_id=hps.repo_id, folder_path=config.out_dir, + path_in_repo=f"model_assets/{config.model_name}", ) global_step += 1 From e0cc36cace826571d2eab58f780044ff3867f3a7 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Sat, 17 Feb 2024 17:22:04 +0900 Subject: [PATCH 06/51] Fix import --- common/tts_model.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/common/tts_model.py b/common/tts_model.py index 1bf5898..0482096 100644 --- a/common/tts_model.py +++ b/common/tts_model.py @@ -25,8 +25,6 @@ from .constants import ( DEFAULT_STYLE_WEIGHT, ) from .log import logger -from torchfcpe import spawn_bundled_infer_model -import librosa def adjust_voice(fs, wave, pitch_scale, intonation_scale): From e0a601737a1a457810ea28618993f503a0eb57bf Mon Sep 17 00:00:00 2001 From: litagin02 Date: Sat, 17 Feb 2024 20:26:58 +0900 Subject: [PATCH 07/51] Error handling for hug upload --- train_ms.py | 50 +++++++++++++++++++++++++------------------- train_ms_jp_extra.py | 50 +++++++++++++++++++++++++------------------- 2 files changed, 56 insertions(+), 44 deletions(-) diff --git a/train_ms.py b/train_ms.py index 08d6af9..9fbc991 100644 --- a/train_ms.py +++ b/train_ms.py @@ -487,17 +487,20 @@ def run(): for_infer=True, ) if hps.repo_id is not None: - api.upload_folder( - repo_id=hps.repo_id, - folder_path=model_dir, - path_in_repo=f"Data/{config.model_name}/models", - delete_patterns="*.pth", - ) - api.upload_folder( - repo_id=hps.repo_id, - folder_path=config.out_dir, - path_in_repo=f"model_assets/{config.model_name}", - ) + try: + api.upload_folder( + repo_id=hps.repo_id, + folder_path=model_dir, + path_in_repo=f"Data/{config.model_name}/models", + delete_patterns="*.pth", + ) + api.upload_folder( + repo_id=hps.repo_id, + folder_path=config.out_dir, + path_in_repo=f"model_assets/{config.model_name}", + ) + except Exception as e: + logger.warning(e) if pbar is not None: pbar.close() @@ -781,17 +784,20 @@ def train_and_evaluate( for_infer=True, ) if hps.repo_id is not None: - api.upload_folder( - repo_id=hps.repo_id, - folder_path=hps.model_dir, - path_in_repo=f"Data/{config.model_name}/models", - delete_patterns="*.pth", - ) - api.upload_folder( - repo_id=hps.repo_id, - folder_path=config.out_dir, - path_in_repo=f"model_assets/{config.model_name}", - ) + try: + api.upload_folder( + repo_id=hps.repo_id, + folder_path=hps.model_dir, + path_in_repo=f"Data/{config.model_name}/models", + delete_patterns="*.pth", + ) + api.upload_folder( + repo_id=hps.repo_id, + folder_path=config.out_dir, + path_in_repo=f"model_assets/{config.model_name}", + ) + except Exception as e: + logger.warning(e) global_step += 1 if pbar is not None: diff --git a/train_ms_jp_extra.py b/train_ms_jp_extra.py index 9c116ab..ac16661 100644 --- a/train_ms_jp_extra.py +++ b/train_ms_jp_extra.py @@ -575,17 +575,20 @@ def run(): for_infer=True, ) if hps.repo_id is not None: - api.upload_folder( - repo_id=hps.repo_id, - folder_path=model_dir, - path_in_repo=f"Data/{config.model_name}/models", - delete_patterns="*.pth", - ) - api.upload_folder( - repo_id=hps.repo_id, - folder_path=config.out_dir, - path_in_repo=f"model_assets/{config.model_name}", - ) + try: + api.upload_folder( + repo_id=hps.repo_id, + folder_path=model_dir, + path_in_repo=f"Data/{config.model_name}/models", + delete_patterns="*.pth", + ) + api.upload_folder( + repo_id=hps.repo_id, + folder_path=config.out_dir, + path_in_repo=f"model_assets/{config.model_name}", + ) + except Exception as e: + logger.warning(e) if pbar is not None: pbar.close() @@ -938,17 +941,20 @@ def train_and_evaluate( for_infer=True, ) if hps.repo_id is not None: - api.upload_folder( - repo_id=hps.repo_id, - folder_path=hps.model_dir, - path_in_repo=f"Data/{config.model_name}/models", - delete_patterns="*.pth", - ) - api.upload_folder( - repo_id=hps.repo_id, - folder_path=config.out_dir, - path_in_repo=f"model_assets/{config.model_name}", - ) + try: + api.upload_folder( + repo_id=hps.repo_id, + folder_path=hps.model_dir, + path_in_repo=f"Data/{config.model_name}/models", + delete_patterns="*.pth", + ) + api.upload_folder( + repo_id=hps.repo_id, + folder_path=config.out_dir, + path_in_repo=f"model_assets/{config.model_name}", + ) + except Exception as e: + logger.warning(e) global_step += 1 if pbar is not None: From 9305d108a82551b270c5757d9cebcd88185d0096 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Sun, 18 Feb 2024 18:25:47 +0900 Subject: [PATCH 08/51] Add Dockerfile (currently for paperspace) --- Dockerfile | 117 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f5c64a8 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,117 @@ +# For paperspace gradient +# Based on https://github.com/gradient-ai/base-container +# Style-Bert-VITS2 are NOT included in this image, only for environment setup + +# ================================================================== +# Initial setup +# ------------------------------------------------------------------ + +# Ubuntu 22.04 as base image +FROM ubuntu:22.04 +# RUN yes| unminimize + +RUN sed -i 's@archive.ubuntu.com@ftp.jaist.ac.jp/pub/Linux@g' /etc/apt/sources.list + +# Set ENV variables +ENV LANG C.UTF-8 +ENV SHELL=/bin/bash +ENV DEBIAN_FRONTEND=noninteractive + +ENV APT_INSTALL="apt-get install -y --no-install-recommends" +ENV PIP_INSTALL="python3 -m pip --no-cache-dir install --upgrade" +ENV GIT_CLONE="git clone --depth 10" + + +# ================================================================== +# Tools +# ------------------------------------------------------------------ + +RUN apt-get update && \ + $APT_INSTALL \ + build-essential \ + ca-certificates \ + wget \ + git \ + curl \ + unzip \ + zip \ + nano \ + ffmpeg \ + sudo \ + software-properties-common \ + gnupg \ + python3 \ + python3-pip \ + python3-dev + +# ================================================================== +# Git-lfs +# ------------------------------------------------------------------ + +RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash && \ + $APT_INSTALL git-lfs + +# Add symlink so python and python3 commands use same python3.9 executable +RUN ln -s /usr/bin/python3 /usr/local/bin/python + + +# ================================================================== +# Installing CUDA packages (CUDA Toolkit 12.1 and CUDNN 8.9.7) +# ------------------------------------------------------------------ +RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-ubuntu2204.pin && \ + mv cuda-ubuntu2204.pin /etc/apt/preferences.d/cuda-repository-pin-600 && \ + wget https://developer.download.nvidia.com/compute/cuda/12.1.0/local_installers/cuda-repo-ubuntu2204-12-1-local_12.1.0-530.30.02-1_amd64.deb && \ + dpkg -i cuda-repo-ubuntu2204-12-1-local_12.1.0-530.30.02-1_amd64.deb && \ + cp /var/cuda-repo-ubuntu2204-12-1-local/cuda-*-keyring.gpg /usr/share/keyrings/ && \ + apt-get update && \ + $APT_INSTALL cuda && \ + rm cuda-repo-ubuntu2204-12-1-local_12.1.0-530.30.02-1_amd64.deb + +# Installing CUDNN +RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/3bf863cc.pub && \ + add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/ /" && \ + apt-get update && \ + $APT_INSTALL libcudnn8=8.9.7.29-1+cuda12.2 \ + libcudnn8-dev=8.9.7.29-1+cuda12.2 + + +ENV PATH=$PATH:/usr/local/cuda/bin +ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH + + +# ================================================================== +# PyTorch +# ------------------------------------------------------------------ + +# Based on https://pytorch.org/get-started/locally/ + +RUN $PIP_INSTALL torch==2.1.2 torchvision==0.16.2 torchaudio==2.1.2 --index-url https://download.pytorch.org/whl/cu121 + +# ================================================================== +# Jupyter Lab +# ------------------------------------------------------------------ + +RUN $PIP_INSTALL jupyterlab + +# ================================================================== +# huggingface_cli +# ------------------------------------------------------------------ + +RUN $PIP_INSTALL "huggingface_hub[cli]" + +# ================================================================== +# Other Python packages +# ------------------------------------------------------------------ + +COPY requirements.txt /tmp/requirements.txt + +RUN $PIP_INSTALL -r /tmp/requirements.txt && \ + rm /tmp/requirements.txt + +# ================================================================== +# Startup +# ------------------------------------------------------------------ + +EXPOSE 8888 6006 + +CMD jupyter lab --allow-root --ip=0.0.0.0 --no-browser --ServerApp.trust_xheaders=True --ServerApp.disable_check_xsrf=False --ServerApp.allow_remote_access=True --ServerApp.allow_origin='*' --ServerApp.allow_credentials=True \ No newline at end of file From 16ab5ac3a7976a587a7f251354a3e4285f939a00 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Mon, 19 Feb 2024 14:22:24 +0900 Subject: [PATCH 09/51] Improve docs --- Dockerfile | 10 ++++++++++ app.py | 10 +++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index f5c64a8..863425d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,16 @@ # Based on https://github.com/gradient-ai/base-container # Style-Bert-VITS2 are NOT included in this image, only for environment setup +# ================================================================== +# Details +# ------------------------------------------------------------------ +# Ubuntu 22.04, Python 3.10 +# CUDA Toolkit 12.1, CUDNN 8.9.7 +# PyTorch 2.1.2 (cuda 12.1) +# Jupyter Lab +# Huggingface CLI +# Other Python packages in requirements.txt + # ================================================================== # Initial setup # ------------------------------------------------------------------ diff --git a/app.py b/app.py index d2e9c19..3f0f91e 100644 --- a/app.py +++ b/app.py @@ -311,18 +311,18 @@ if __name__ == "__main__": load_button = gr.Button("ロード", scale=1, variant="primary") text_input = gr.TextArea(label="テキスト", value=initial_text) pitch_scale = gr.Slider( - minimum=0, - maximum=2, + minimum=0.8, + maximum=1.5, value=1, - step=0.1, - label="音程", + step=0.05, + label="音程(1以外では音質劣化)", ) intonation_scale = gr.Slider( minimum=0, maximum=2, value=1, step=0.1, - label="抑揚", + label="抑揚(1以外では音質劣化)", ) line_split = gr.Checkbox( From f49755689a5ae147820b8260a12de3ed5143fcf6 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Mon, 19 Feb 2024 15:21:17 +0900 Subject: [PATCH 10/51] Fix counting lines --- preprocess_text.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/preprocess_text.py b/preprocess_text.py index 4f49100..75e61dc 100644 --- a/preprocess_text.py +++ b/preprocess_text.py @@ -16,8 +16,8 @@ preprocess_text_config = config.preprocess_text_config # Count lines for tqdm -def count_lines(file_path): - with file_path.open("r", encoding="utf-8") as file: +def count_lines(file_path: str): + with open(file_path, "r", encoding="utf-8") as file: return sum(1 for _ in file) From d26336bf0ae985af920104b2e8f8e57a669d6c2a Mon Sep 17 00:00:00 2001 From: litagin02 Date: Mon, 19 Feb 2024 15:42:08 +0900 Subject: [PATCH 11/51] Upload to hug as future --- train_ms.py | 55 +++++++++++++++++++++++--------------------- train_ms_jp_extra.py | 55 +++++++++++++++++++++++--------------------- 2 files changed, 58 insertions(+), 52 deletions(-) diff --git a/train_ms.py b/train_ms.py index 9fbc991..c16dda7 100644 --- a/train_ms.py +++ b/train_ms.py @@ -487,20 +487,24 @@ def run(): for_infer=True, ) if hps.repo_id is not None: + future1 = api.upload_folder( + repo_id=hps.repo_id, + folder_path=model_dir, + path_in_repo=f"Data/{config.model_name}/models", + delete_patterns="*.pth", + run_as_future=True, + ) + future2 = api.upload_folder( + repo_id=hps.repo_id, + folder_path=config.out_dir, + path_in_repo=f"model_assets/{config.model_name}", + run_as_future=True, + ) try: - api.upload_folder( - repo_id=hps.repo_id, - folder_path=model_dir, - path_in_repo=f"Data/{config.model_name}/models", - delete_patterns="*.pth", - ) - api.upload_folder( - repo_id=hps.repo_id, - folder_path=config.out_dir, - path_in_repo=f"model_assets/{config.model_name}", - ) + future1.result() + future2.result() except Exception as e: - logger.warning(e) + logger.error(e) if pbar is not None: pbar.close() @@ -784,20 +788,19 @@ def train_and_evaluate( for_infer=True, ) if hps.repo_id is not None: - try: - api.upload_folder( - repo_id=hps.repo_id, - folder_path=hps.model_dir, - path_in_repo=f"Data/{config.model_name}/models", - delete_patterns="*.pth", - ) - api.upload_folder( - repo_id=hps.repo_id, - folder_path=config.out_dir, - path_in_repo=f"model_assets/{config.model_name}", - ) - except Exception as e: - logger.warning(e) + api.upload_folder( + repo_id=hps.repo_id, + folder_path=hps.model_dir, + path_in_repo=f"Data/{config.model_name}/models", + delete_patterns="*.pth", + run_as_future=True, + ) + api.upload_folder( + repo_id=hps.repo_id, + folder_path=config.out_dir, + path_in_repo=f"model_assets/{config.model_name}", + run_as_future=True, + ) global_step += 1 if pbar is not None: diff --git a/train_ms_jp_extra.py b/train_ms_jp_extra.py index ac16661..75e72c5 100644 --- a/train_ms_jp_extra.py +++ b/train_ms_jp_extra.py @@ -575,20 +575,24 @@ def run(): for_infer=True, ) if hps.repo_id is not None: + future1 = api.upload_folder( + repo_id=hps.repo_id, + folder_path=model_dir, + path_in_repo=f"Data/{config.model_name}/models", + delete_patterns="*.pth", + run_as_future=True, + ) + future2 = api.upload_folder( + repo_id=hps.repo_id, + folder_path=config.out_dir, + path_in_repo=f"model_assets/{config.model_name}", + run_as_future=True, + ) try: - api.upload_folder( - repo_id=hps.repo_id, - folder_path=model_dir, - path_in_repo=f"Data/{config.model_name}/models", - delete_patterns="*.pth", - ) - api.upload_folder( - repo_id=hps.repo_id, - folder_path=config.out_dir, - path_in_repo=f"model_assets/{config.model_name}", - ) + future1.result() + future2.result() except Exception as e: - logger.warning(e) + logger.error(e) if pbar is not None: pbar.close() @@ -941,20 +945,19 @@ def train_and_evaluate( for_infer=True, ) if hps.repo_id is not None: - try: - api.upload_folder( - repo_id=hps.repo_id, - folder_path=hps.model_dir, - path_in_repo=f"Data/{config.model_name}/models", - delete_patterns="*.pth", - ) - api.upload_folder( - repo_id=hps.repo_id, - folder_path=config.out_dir, - path_in_repo=f"model_assets/{config.model_name}", - ) - except Exception as e: - logger.warning(e) + api.upload_folder( + repo_id=hps.repo_id, + folder_path=hps.model_dir, + path_in_repo=f"Data/{config.model_name}/models", + delete_patterns="*.pth", + run_as_future=True, + ) + api.upload_folder( + repo_id=hps.repo_id, + folder_path=config.out_dir, + path_in_repo=f"model_assets/{config.model_name}", + run_as_future=True, + ) global_step += 1 if pbar is not None: From 036cea17f3181514c1dc5c5f269869a82ae19da5 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Mon, 19 Feb 2024 17:22:40 +0900 Subject: [PATCH 12/51] Add preprocess_all.py and optimize cli settings --- gen_yaml.py | 16 ++++---- initialize.py | 33 +++++++++++++++- preprocess_all.py | 96 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 135 insertions(+), 10 deletions(-) create mode 100644 preprocess_all.py diff --git a/gen_yaml.py b/gen_yaml.py index 8650dc0..91301ac 100644 --- a/gen_yaml.py +++ b/gen_yaml.py @@ -3,22 +3,20 @@ import shutil import yaml import argparse -parser = argparse.ArgumentParser(description="config.ymlの生成。あらかじめ前準備をしたデータをバッチファイルなどで連続で学習する時にtrain_ms.pyより前に使用する。") -# そうしないと最後の前準備したデータで学習してしまう -parser.add_argument( - "--model_name", - type=str, - help="Model name", - required=True +parser = argparse.ArgumentParser( + description="config.ymlの生成。あらかじめ前準備をしたデータをバッチファイルなどで連続で学習する時にtrain_ms.pyより前に使用する。" ) +# そうしないと最後の前準備したデータで学習してしまう +parser.add_argument("--model_name", type=str, help="Model name", required=True) parser.add_argument( "--dataset_path", type=str, help="Dataset path(example: Data\\your_model_name)", - required=True + required=True, ) args = parser.parse_args() + def gen_yaml(model_name, dataset_path): if not os.path.exists("config.yml"): shutil.copy(src="default_config.yml", dst="config.yml") @@ -29,6 +27,6 @@ def gen_yaml(model_name, dataset_path): with open("config.yml", "w", encoding="utf-8") as f: yaml.dump(yml_data, f, allow_unicode=True) + if __name__ == "__main__": gen_yaml(args.model_name, args.dataset_path) - diff --git a/initialize.py b/initialize.py index c163ef9..5e35061 100644 --- a/initialize.py +++ b/initialize.py @@ -2,6 +2,7 @@ import argparse import json from pathlib import Path +import yaml from huggingface_hub import hf_hub_download from common.log import logger @@ -90,9 +91,21 @@ def download_jvnv_models(): ) -if __name__ == "__main__": +def main(): parser = argparse.ArgumentParser() parser.add_argument("--skip_jvnv", action="store_true") + parser.add_argument( + "--dataset_root", + type=str, + help="Dataset root path (default: Data)", + default=None, + ) + parser.add_argument( + "--assets_root", + type=str, + help="Assets root path (default: model_assets)", + default=None, + ) args = parser.parse_args() download_bert_models() @@ -105,3 +118,21 @@ if __name__ == "__main__": if not args.skip_jvnv: download_jvnv_models() + + if args.dataset_root is None and args.assets_root is None: + return + + # Change default paths if necessary + paths_yml = Path("configs/paths.yml") + with open(paths_yml, "r", encoding="utf-8") as f: + yml_data = yaml.safe_load(f) + if args.assets_root is not None: + yml_data["assets_root"] = args.assets_root + if args.dataset_root is not None: + yml_data["dataset_root"] = args.dataset_root + with open(paths_yml, "w", encoding="utf-8") as f: + yaml.dump(yml_data, f, allow_unicode=True) + + +if __name__ == "__main__": + main() diff --git a/preprocess_all.py b/preprocess_all.py new file mode 100644 index 0000000..b83e93b --- /dev/null +++ b/preprocess_all.py @@ -0,0 +1,96 @@ +import argparse +from webui_train import preprocess_all +from multiprocessing import cpu_count + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--model_name", "-m", type=str, help="Model name", required=True + ) + parser.add_argument("--batch_size", "-b", type=int, help="Batch size", default=2) + parser.add_argument("--epochs", "-e", type=int, help="Epochs", default=100) + parser.add_argument( + "--save_every_steps", + "-s", + type=int, + help="Save every steps", + default=1000, + ) + parser.add_argument( + "--num_processes", + type=int, + help="Number of processes", + default=cpu_count() // 2, + ) + parser.add_argument( + "--normalize", + action="store_true", + help="Loudness normalize audio", + ) + parser.add_argument( + "--trim", + action="store_true", + help="Trim silence", + ) + parser.add_argument( + "--freeze_EN_bert", + action="store_true", + help="Freeze English BERT", + ) + parser.add_argument( + "--freeze_JP_bert", + action="store_true", + help="Freeze Japanese BERT", + ) + parser.add_argument( + "--freeze_ZH_bert", + action="store_true", + help="Freeze Chinese BERT", + ) + parser.add_argument( + "--freeze_style", + action="store_true", + help="Freeze style vector", + ) + parser.add_argument( + "--use_jp_extra", + action="store_true", + help="Use JP-Extra pretrained model", + ) + parser.add_argument( + "--val_per_lang", + type=int, + help="Validation per language", + default=0, + ) + parser.add_argument( + "--log_interval", + type=int, + help="Log interval", + default=200, + ) + parser.add_argument( + "--skip_invalid", + action="store_true", + help="Skip invalid", + ) + + args = parser.parse_args() + + preprocess_all( + model_name=args.model_name, + batch_size=args.batch_size, + epochs=args.epochs, + save_every_steps=args.save_every_steps, + num_processes=args.num_processes, + normalize=args.normalize, + trim=args.trim, + freeze_EN_bert=args.freeze_EN_bert, + freeze_JP_bert=args.freeze_JP_bert, + freeze_ZH_bert=args.freeze_ZH_bert, + freeze_style=args.freeze_style, + use_jp_extra=args.use_jp_extra, + val_per_lang=args.val_per_lang, + log_interval=args.log_interval, + skip_invalid=args.skip_invalid, + ) From 4c8ea82959890b30aa1c01cab065b949436f57ed Mon Sep 17 00:00:00 2001 From: litagin02 Date: Mon, 19 Feb 2024 18:35:22 +0900 Subject: [PATCH 13/51] Improve: log error for preprocessing text --- preprocess_all.py | 2 +- preprocess_text.py | 37 +++++++++++++++++++------------------ 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/preprocess_all.py b/preprocess_all.py index b83e93b..8ff4e34 100644 --- a/preprocess_all.py +++ b/preprocess_all.py @@ -55,7 +55,7 @@ if __name__ == "__main__": parser.add_argument( "--use_jp_extra", action="store_true", - help="Use JP-Extra pretrained model", + help="Use JP-Extra model", ) parser.add_argument( "--val_per_lang", diff --git a/preprocess_text.py b/preprocess_text.py index 75e61dc..68534ea 100644 --- a/preprocess_text.py +++ b/preprocess_text.py @@ -40,7 +40,6 @@ def count_lines(file_path: str): @click.option("--clean/--no-clean", default=preprocess_text_config.clean) @click.option("-y", "--yml_config") @click.option("--use_jp_extra", is_flag=True) -@click.option("--skip_invalid", is_flag=True) def preprocess( transcription_path: str, cleaned_path: Optional[str], @@ -52,11 +51,13 @@ def preprocess( clean: bool, yml_config: str, # 这个不要删 use_jp_extra: bool, - skip_invalid: bool = False, ): if cleaned_path == "" or cleaned_path is None: cleaned_path = transcription_path + ".cleaned" + error_log_path = os.path.join(os.path.dirname(cleaned_path), "error.log") + error_count = 0 + if clean: total_lines = count_lines(transcription_path) with open(cleaned_path, "w", encoding="utf-8") as out_file: @@ -79,18 +80,10 @@ def preprocess( ) ) except Exception as e: - if skip_invalid: - logger.warning( - f"An error occurred while generating the training set and validation set, at line:\n{line}\nDetails:\n{e}" - ) - logger.warning( - "This line will be skipped from the training set and validation set." - ) - else: - logger.error( - f"An error occurred while generating the training set and validation set, at line:\n{line}\nDetails:\n{e}" - ) - raise + logger.error(f"An error occurred at line:\n{line.strip()}\n{e}") + with open(error_log_path, "a", encoding="utf-8") as error_log: + error_log.write(f"{line.strip()}\n{e}\n\n") + error_count += 1 transcription_path = cleaned_path spk_utt_map = defaultdict(list) @@ -118,9 +111,10 @@ def preprocess( if spk not in spk_id_map.keys(): spk_id_map[spk] = current_sid current_sid += 1 - logger.info( - f"Total repeated audios: {countSame}, Total number of audio not found: {countNotFound}" - ) + if countSame > 0 or countNotFound > 0: + logger.warning( + f"Total repeated audios: {countSame}, Total number of audio not found: {countNotFound}" + ) train_list = [] val_list = [] @@ -156,7 +150,14 @@ def preprocess( ) with open(config_path, "w", encoding="utf-8") as f: json.dump(json_config, f, indent=2, ensure_ascii=False) - logger.info("Training set and validation set generation from texts is complete!") + if error_count > 0: + logger.warning( + f"An error occurred in {error_count} lines. Please check {error_log_path} for details. You can proceed with lines that do not have errors." + ) + else: + logger.info( + "Training set and validation set generation from texts is complete!" + ) if __name__ == "__main__": From 79f20fafbb94d54956c5bedc78bc5365ece803c6 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Mon, 19 Feb 2024 19:16:41 +0900 Subject: [PATCH 14/51] Change args from output_dir to model_name --- slice.py | 27 +++++++++++++++++++-------- webui_dataset.py | 5 ++--- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/slice.py b/slice.py index 3b6e2d7..b20fe9c 100644 --- a/slice.py +++ b/slice.py @@ -5,6 +5,7 @@ from pathlib import Path import soundfile as sf import torch +import yaml from tqdm import tqdm from common.log import logger @@ -76,6 +77,7 @@ def split_wav( os.makedirs(target_dir, exist_ok=True) total_time_ms = 0 + count = 0 # タイムスタンプに従って分割し、ファイルに保存 for i, ts in enumerate(speech_timestamps): @@ -88,8 +90,9 @@ def split_wav( sf.write(os.path.join(target_dir, f"{file_name}-{i}.wav"), segment, sr) total_time_ms += end_ms - start_ms + count += 1 - return total_time_ms / 1000 + return total_time_ms / 1000, count if __name__ == "__main__": @@ -108,11 +111,11 @@ if __name__ == "__main__": help="Directory of input wav files", ) parser.add_argument( - "--output_dir", - "-o", + "--model_name", + "-m", type=str, - default="raw", - help="Directory of output wav files", + required=True, + help="The result will be in Data/{model_name}/raw/ (if Data is dataset_root in configs/paths.yml)", ) parser.add_argument( "--min_silence_dur_ms", @@ -123,8 +126,12 @@ if __name__ == "__main__": ) args = parser.parse_args() + with open(os.path.join("configs", "paths.yml"), "r", encoding="utf-8") as f: + path_config: dict[str, str] = yaml.safe_load(f.read()) + dataset_root = path_config["dataset_root"] + input_dir = args.input_dir - output_dir = args.output_dir + output_dir = os.path.join(dataset_root, args.model_name, "raw") min_sec = args.min_sec max_sec = args.max_sec min_silence_dur_ms = args.min_silence_dur_ms @@ -137,8 +144,9 @@ if __name__ == "__main__": shutil.rmtree(output_dir) total_sec = 0 + total_count = 0 for wav_file in tqdm(wav_files, file=SAFE_STDOUT): - time_sec = split_wav( + time_sec, count = split_wav( audio_file=str(wav_file), target_dir=output_dir, min_sec=min_sec, @@ -146,5 +154,8 @@ if __name__ == "__main__": min_silence_dur_ms=min_silence_dur_ms, ) total_sec += time_sec + total_count += count - logger.info(f"Slice done! Total time: {total_sec / 60:.2f} min.") + logger.info( + f"Slice done! Total time: {total_sec / 60:.2f} min, {total_count} files." + ) diff --git a/webui_dataset.py b/webui_dataset.py index 2885e27..02fb15d 100644 --- a/webui_dataset.py +++ b/webui_dataset.py @@ -25,11 +25,10 @@ def do_slice( if model_name == "": return "Error: モデル名を入力してください。" logger.info("Start slicing...") - output_dir = os.path.join(dataset_root, model_name, "raw") cmd = [ "slice.py", - "--output_dir", - output_dir, + "--model_name", + model_name, "--min_sec", str(min_sec), "--max_sec", From 2053ddd101056f855621ee0b662dac24f88d7ef9 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Mon, 19 Feb 2024 19:19:09 +0900 Subject: [PATCH 15/51] Fix conflict -m --- slice.py | 1 - 1 file changed, 1 deletion(-) diff --git a/slice.py b/slice.py index b20fe9c..2d56427 100644 --- a/slice.py +++ b/slice.py @@ -112,7 +112,6 @@ if __name__ == "__main__": ) parser.add_argument( "--model_name", - "-m", type=str, required=True, help="The result will be in Data/{model_name}/raw/ (if Data is dataset_root in configs/paths.yml)", From b7ffc37ba98d10f5415fa4d027c5fc9fedb53ea8 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Mon, 19 Feb 2024 19:36:14 +0900 Subject: [PATCH 16/51] Change args from input output to model_name --- transcribe.py | 21 +++++++++++++-------- webui_dataset.py | 17 +++-------------- 2 files changed, 16 insertions(+), 22 deletions(-) diff --git a/transcribe.py b/transcribe.py index d9fb98b..0ec9e99 100644 --- a/transcribe.py +++ b/transcribe.py @@ -2,6 +2,7 @@ import argparse import os import sys +import yaml from faster_whisper import WhisperModel from tqdm import tqdm @@ -20,25 +21,29 @@ def transcribe(wav_path, initial_prompt=None, language="ja"): if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("--input_dir", "-i", type=str, default="raw") - parser.add_argument("--output_file", "-o", type=str, default="esd.list") + parser.add_argument("--model_name", type=str, required=True) parser.add_argument( - "--initial_prompt", type=str, default="こんにちは。元気、ですかー？ふふっ、私は……ちゃんと元気だよ！" + "--initial_prompt", + type=str, + default="こんにちは。元気、ですかー？ふふっ、私は……ちゃんと元気だよ！", ) parser.add_argument( "--language", type=str, default="ja", choices=["ja", "en", "zh"] ) - parser.add_argument("--speaker_name", type=str, required=True) parser.add_argument("--model", type=str, default="large-v3") parser.add_argument("--device", type=str, default="cuda") parser.add_argument("--compute_type", type=str, default="bfloat16") args = parser.parse_args() - speaker_name = args.speaker_name + with open(os.path.join("configs", "paths.yml"), "r", encoding="utf-8") as f: + path_config: dict[str, str] = yaml.safe_load(f.read()) + dataset_root = path_config["dataset_root"] - input_dir = args.input_dir - output_file = args.output_file + model_name = args.model_name + + input_dir = os.path.join(dataset_root, model_name, "raw") + output_file = os.path.join(dataset_root, model_name, "esd.list") initial_prompt = args.initial_prompt language = args.language device = args.device @@ -79,6 +84,6 @@ if __name__ == "__main__": text = transcribe( wav_file, initial_prompt=initial_prompt, language=language ) - f.write(f"{file_name}|{speaker_name}|{language_id}|{text}\n") + f.write(f"{file_name}|{model_name}|{language_id}|{text}\n") f.flush() sys.exit(0) diff --git a/webui_dataset.py b/webui_dataset.py index 02fb15d..a795575 100644 --- a/webui_dataset.py +++ b/webui_dataset.py @@ -46,24 +46,17 @@ def do_slice( def do_transcribe( - model_name, whisper_model, compute_type, language, initial_prompt, input_dir, device + model_name, whisper_model, compute_type, language, initial_prompt, device ): if model_name == "": return "Error: モデル名を入力してください。" if initial_prompt == "": initial_prompt = "こんにちは。元気、ですかー？私は……ふふっ、ちゃんと元気だよ！" - # logger.debug(f"initial_prompt: {initial_prompt}") - if input_dir == "": - input_dir = os.path.join(dataset_root, model_name, "raw") - output_file = os.path.join(dataset_root, model_name, "esd.list") + success, message = run_script_with_log( [ "transcribe.py", - "--input_dir", - input_dir, - "--output_file", - output_file, - "--speaker_name", + "--model_name", model_name, "--model", whisper_model, @@ -153,9 +146,6 @@ with gr.Blocks(theme=GRADIO_THEME) as app: result1 = gr.Textbox(label="結果") with gr.Row(): with gr.Column(): - raw_dir = gr.Textbox( - label="書き起こしたい音声ファイルが入っているフォルダ（スライスした場合など、`Data/{モデル名}/raw`の場合は省略可", - ) whisper_model = gr.Dropdown( ["tiny", "base", "small", "medium", "large", "large-v2", "large-v3"], label="Whisperモデル", @@ -197,7 +187,6 @@ with gr.Blocks(theme=GRADIO_THEME) as app: compute_type, language, initial_prompt, - raw_dir, device, ], outputs=[result2], From 6ee8125dffcf6cc2d3bb0e554d875a84cc136c5d Mon Sep 17 00:00:00 2001 From: litagin02 Date: Mon, 19 Feb 2024 19:44:12 +0900 Subject: [PATCH 17/51] Fix invalid option --- webui_train.py | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/webui_train.py b/webui_train.py index 6ca1216..2fe53cc 100644 --- a/webui_train.py +++ b/webui_train.py @@ -150,7 +150,7 @@ def resample(model_name, normalize, trim, num_processes): return True, "Step 2, Success: 音声ファイルの前処理が完了しました" -def preprocess_text(model_name, use_jp_extra, val_per_lang, skip_invalid=False): +def preprocess_text(model_name, use_jp_extra, val_per_lang): logger.info("Step 3: start preprocessing text...") dataset_path, lbl_path, train_path, val_path, config_path = get_path(model_name) try: @@ -180,8 +180,6 @@ def preprocess_text(model_name, use_jp_extra, val_per_lang, skip_invalid=False): ] if use_jp_extra: cmd.append("--use_jp_extra") - if skip_invalid: - cmd.append("--skip_invalid") success, message = run_script_with_log(cmd) if not success: logger.error(f"Step 3: preprocessing text failed.") @@ -267,7 +265,6 @@ def preprocess_all( use_jp_extra, val_per_lang, log_interval, - skip_invalid=False, ): if model_name == "": return False, "Error: モデル名を入力してください" @@ -288,9 +285,7 @@ def preprocess_all( success, message = resample(model_name, normalize, trim, num_processes) if not success: return False, message - success, message = preprocess_text( - model_name, use_jp_extra, val_per_lang, skip_invalid - ) + success, message = preprocess_text(model_name, use_jp_extra, val_per_lang) if not success: return False, message success, message = bert_gen(model_name) # bert_genは重いのでプロセス数いじらない @@ -495,10 +490,6 @@ if __name__ == "__main__": maximum=1000, step=10, ) - skip_invalid = gr.Checkbox( - label="日本語処理でエラーが出たファイルは学習に使わない", - value=False, - ) gr.Markdown("学習時に特定の部分を凍結させるかどうか") freeze_EN_bert = gr.Checkbox( label="英語bert部分を凍結", @@ -608,10 +599,6 @@ if __name__ == "__main__": maximum=100, step=1, ) - skip_invalid_manual = gr.Checkbox( - label="日本語処理でエラーが出たファイルは学習に使わない", - value=False, - ) with gr.Column(): preprocess_text_btn = gr.Button(value="実行", variant="primary") info_preprocess_text = gr.Textbox(label="状況") @@ -674,7 +661,6 @@ if __name__ == "__main__": use_jp_extra, val_per_lang, log_interval, - skip_invalid, ], outputs=[info_all], ) @@ -712,7 +698,6 @@ if __name__ == "__main__": model_name, use_jp_extra_manual, val_per_lang_manual, - skip_invalid_manual, ], outputs=[info_preprocess_text], ) From 442ad6ac276ec2622334b8b490a5604211619517 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Mon, 19 Feb 2024 20:15:20 +0900 Subject: [PATCH 18/51] Docs: CLI doc and paperspace guide --- Dockerfile | 127 ---------------------------------- README.md | 3 + colab.ipynb | 2 +- common/constants.py | 2 +- configs/config.json | 2 +- configs/configs_jp_extra.json | 2 +- docs/CLI.md | 92 ++++++++++++++++-------- docs/paperspace.md | 72 +++++++++++++++++++ preprocess_all.py | 5 -- requirements.txt | 3 +- 10 files changed, 146 insertions(+), 164 deletions(-) delete mode 100644 Dockerfile create mode 100644 docs/paperspace.md diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 863425d..0000000 --- a/Dockerfile +++ /dev/null @@ -1,127 +0,0 @@ -# For paperspace gradient -# Based on https://github.com/gradient-ai/base-container -# Style-Bert-VITS2 are NOT included in this image, only for environment setup - -# ================================================================== -# Details -# ------------------------------------------------------------------ -# Ubuntu 22.04, Python 3.10 -# CUDA Toolkit 12.1, CUDNN 8.9.7 -# PyTorch 2.1.2 (cuda 12.1) -# Jupyter Lab -# Huggingface CLI -# Other Python packages in requirements.txt - -# ================================================================== -# Initial setup -# ------------------------------------------------------------------ - -# Ubuntu 22.04 as base image -FROM ubuntu:22.04 -# RUN yes| unminimize - -RUN sed -i 's@archive.ubuntu.com@ftp.jaist.ac.jp/pub/Linux@g' /etc/apt/sources.list - -# Set ENV variables -ENV LANG C.UTF-8 -ENV SHELL=/bin/bash -ENV DEBIAN_FRONTEND=noninteractive - -ENV APT_INSTALL="apt-get install -y --no-install-recommends" -ENV PIP_INSTALL="python3 -m pip --no-cache-dir install --upgrade" -ENV GIT_CLONE="git clone --depth 10" - - -# ================================================================== -# Tools -# ------------------------------------------------------------------ - -RUN apt-get update && \ - $APT_INSTALL \ - build-essential \ - ca-certificates \ - wget \ - git \ - curl \ - unzip \ - zip \ - nano \ - ffmpeg \ - sudo \ - software-properties-common \ - gnupg \ - python3 \ - python3-pip \ - python3-dev - -# ================================================================== -# Git-lfs -# ------------------------------------------------------------------ - -RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash && \ - $APT_INSTALL git-lfs - -# Add symlink so python and python3 commands use same python3.9 executable -RUN ln -s /usr/bin/python3 /usr/local/bin/python - - -# ================================================================== -# Installing CUDA packages (CUDA Toolkit 12.1 and CUDNN 8.9.7) -# ------------------------------------------------------------------ -RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-ubuntu2204.pin && \ - mv cuda-ubuntu2204.pin /etc/apt/preferences.d/cuda-repository-pin-600 && \ - wget https://developer.download.nvidia.com/compute/cuda/12.1.0/local_installers/cuda-repo-ubuntu2204-12-1-local_12.1.0-530.30.02-1_amd64.deb && \ - dpkg -i cuda-repo-ubuntu2204-12-1-local_12.1.0-530.30.02-1_amd64.deb && \ - cp /var/cuda-repo-ubuntu2204-12-1-local/cuda-*-keyring.gpg /usr/share/keyrings/ && \ - apt-get update && \ - $APT_INSTALL cuda && \ - rm cuda-repo-ubuntu2204-12-1-local_12.1.0-530.30.02-1_amd64.deb - -# Installing CUDNN -RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/3bf863cc.pub && \ - add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/ /" && \ - apt-get update && \ - $APT_INSTALL libcudnn8=8.9.7.29-1+cuda12.2 \ - libcudnn8-dev=8.9.7.29-1+cuda12.2 - - -ENV PATH=$PATH:/usr/local/cuda/bin -ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH - - -# ================================================================== -# PyTorch -# ------------------------------------------------------------------ - -# Based on https://pytorch.org/get-started/locally/ - -RUN $PIP_INSTALL torch==2.1.2 torchvision==0.16.2 torchaudio==2.1.2 --index-url https://download.pytorch.org/whl/cu121 - -# ================================================================== -# Jupyter Lab -# ------------------------------------------------------------------ - -RUN $PIP_INSTALL jupyterlab - -# ================================================================== -# huggingface_cli -# ------------------------------------------------------------------ - -RUN $PIP_INSTALL "huggingface_hub[cli]" - -# ================================================================== -# Other Python packages -# ------------------------------------------------------------------ - -COPY requirements.txt /tmp/requirements.txt - -RUN $PIP_INSTALL -r /tmp/requirements.txt && \ - rm /tmp/requirements.txt - -# ================================================================== -# Startup -# ------------------------------------------------------------------ - -EXPOSE 8888 6006 - -CMD jupyter lab --allow-root --ip=0.0.0.0 --no-browser --ServerApp.trust_xheaders=True --ServerApp.disable_check_xsrf=False --ServerApp.allow_remote_access=True --ServerApp.allow_origin='*' --ServerApp.allow_credentials=True \ No newline at end of file diff --git a/README.md b/README.md index bf2a05f..edd5ba5 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,9 @@ model_assets ### 学習 +- CLIでの学習の詳細は[こちら](docs/CLI.md)を参照してください。 +- paperspace上での学習の詳細は[こちら](docs/paperspace.md)を参照してください。 + 学習には2-14秒程度の音声ファイルが複数と、それらの書き起こしデータが必要です。 - 既存コーパスなどですでに分割された音声ファイルと書き起こしデータがある場合はそのまま（必要に応じて書き起こしファイルを修正して）使えます。下の「学習WebUI」を参照してください。 diff --git a/colab.ipynb b/colab.ipynb index f5cab3e..1a5369a 100644 --- a/colab.ipynb +++ b/colab.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Style-Bert-VITS2 (ver 2.2) のGoogle Colabでの学習\n", + "# Style-Bert-VITS2 (ver 2.3) のGoogle Colabでの学習\n", "\n", "Google Colab上でStyle-Bert-VITS2の学習を行うことができます。\n", "\n", diff --git a/common/constants.py b/common/constants.py index d2d6352..cd118e2 100644 --- a/common/constants.py +++ b/common/constants.py @@ -4,7 +4,7 @@ import enum # See https://huggingface.co/spaces/gradio/theme-gallery for more themes GRADIO_THEME: str = "NoCrypt/miku" -LATEST_VERSION: str = "2.2" +LATEST_VERSION: str = "2.3" DEFAULT_STYLE: str = "Neutral" DEFAULT_STYLE_WEIGHT: float = 5.0 diff --git a/configs/config.json b/configs/config.json index 6aa647c..5196633 100644 --- a/configs/config.json +++ b/configs/config.json @@ -67,5 +67,5 @@ "use_spectral_norm": false, "gin_channels": 256 }, - "version": "2.2" + "version": "2.3" } diff --git a/configs/configs_jp_extra.json b/configs/configs_jp_extra.json index 7e2698e..490f4d6 100644 --- a/configs/configs_jp_extra.json +++ b/configs/configs_jp_extra.json @@ -74,5 +74,5 @@ "initial_channel": 64 } }, - "version": "2.2-JP-Extra" + "version": "2.3-JP-Extra" } diff --git a/docs/CLI.md b/docs/CLI.md index ca955e1..3873475 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -1,35 +1,49 @@ # CLI -**WIP** - -## Dataset - -`Dataset.bat` webui (`python webui_dataset.py`) consists of **slice audio** and **transcribe wavs**. - -### Slice audio +## 0. Install and global paths settings ```bash -python slice.py -i -o -m -M +git clone https://github.com/litagin02/Style-Bert-VITS2.git +cd Style-Bert-VITS2 +python -m venv venv +venv\Scripts\activate +pip install torch==2.1.2 torchvision==0.16.2 torchaudio==2.1.2 --index-url https://download.pytorch.org/whl/cu118 +pip install -r requirements.txt ``` -Required: -- `input_dir`: Path to the directory containing the audio files to slice. -- `output_dir`: Path to the directory where the sliced audio files will be saved. +Then download the necessary models and the default TTS model, and set the global paths. +```bash +python initialize.py [--skip_jvnv] [--dataset_root ] [--assets_root ] +``` Optional: -- `min_sec`: Minimum duration of the sliced audio files in seconds (default 2). -- `max_sec`: Maximum duration of the sliced audio files in seconds (default 12). +- `--skip_jvnv`: Skip downloading the default JVNV voice models (use this if you only have to train your own models). +- `--dataset_root`: Default: `Data`. Root directory of the training dataset. The training dataset of `{model_name}` should be placed in `{dataset_root}/{model_name}`. +- `--assets_root`: Default: `model_assets`. Root directory of the model assets (for inference). In training, the model assets will be saved to `{assets_root}/{model_name}`, and in inference, we load all the models from `{assets_root}`. -### Transcribe wavs +## 1. Dataset preparation + +### 1.1. Slice wavs ```bash -python transcribe.py -i -o --speaker_name +python slice.py --model_name [-i ] [-m ] [-M ] ``` Required: -- `input_dir`: Path to the directory containing the audio files to transcribe. -- `output_file`: Path to the file where the transcriptions will be saved. -- `speaker_name`: Name of the speaker. +- `model_name`: Name of the speaker (to be used as the name of the trained model). + +Optional: +- `input_dir`: Path to the directory containing the audio files to slice (default: `inputs`) +- `min_sec`: Minimum duration of the sliced audio files in seconds (default: 2). +- `max_sec`: Maximum duration of the sliced audio files in seconds (default: 12). + +### 1.2. Transcribe wavs + +```bash +python transcribe.py --model_name +``` +Required: +- `model_name`: Name of the speaker (to be used as the name of the trained model). Optional - `--initial_prompt`: Initial prompt to use for the transcription (default value is specific to Japanese). @@ -38,19 +52,43 @@ Optional - `--model`: Whisper model, default: `large-v3` - `--compute_type`: default: `bfloat16` -## Train +## 2. Preprocess -`Train.bat` webui (`python webui_train.py`) consists of the following. - -### Preprocess audio ```bash -python resample.py -i -o [--normalize] [--trim] +python preprocess_all.py -m [--use_jp_extra] [-b ] [-e ] [-s ] [--num_processes ] [--normalize] [--trim] [--val_per_lang ] [--log_interval ] [--freeze_EN_bert] [--freeze_JP_bert] [--freeze_ZH_bert] [--freeze_style] ``` Required: -- `input_dir`: Path to the directory containing the audio files to preprocess. -- `output_dir`: Path to the directory where the preprocessed audio files will be saved. +- `model_name`: Name of the speaker (to be used as the name of the trained model). -TO BE WRITTEN (WIP) +Optional: +- `--batch_size`, `-b`: Batch size (default: 2). +- `--epochs`, `-e`: Number of epochs (default: 100). +- `--save_every_steps`, `-s`: Save every steps (default: 1000). +- `--num_processes`: Number of processes (default: half of the number of CPU cores). +- `--normalize`: Loudness normalize audio. +- `--trim`: Trim silence. +- `--freeze_EN_bert`: Freeze English BERT. +- `--freeze_JP_bert`: Freeze Japanese BERT. +- `--freeze_ZH_bert`: Freeze Chinese BERT. +- `--freeze_style`: Freeze style vector. +- `--use_jp_extra`: Use JP-Extra model. +- `--val_per_lang`: Validation data per language (default: 0). +- `--log_interval`: Log interval (default: 200). -これいる？ +## 3. Train + +Training settings are automatically loaded from the above process. + +If NOT using JP-Extra model: +```bash +python train_ms.py [--repo_id /] +``` + +If using JP-Extra model: +```bash +python train_ms_jp_extra.py [--repo_id /] +``` + +Optional: +- `--repo_id`: Hugging Face repository ID to upload the trained model to. You should have logged in using `huggingface-cli login` before running this command. diff --git a/docs/paperspace.md b/docs/paperspace.md new file mode 100644 index 0000000..d67253e --- /dev/null +++ b/docs/paperspace.md @@ -0,0 +1,72 @@ +# Paperspace gradient で学習する + +詳しいコマンドの叩き方は[こちら](CLI.md)を参照してください。 + +## 事前準備 +- Paperspace のアカウントを作成し必要なら課金する +- Projectを作る +- NotebookはStart from Scratchを選択して空いてるGPUマシンを選ぶ + +## 使い方 + +以下では次のような方針でやっています。 + +- `/storage/`は永続ストレージなので、事前学習モデルとかを含めてリポジトリをクローンするとよい。 +- `/notebooks/`は一時ストレージなので、データセットやその結果を保存する。 +- hugging faceアカウントを作り、（プライベートな）リポジトリを作って、学習元データを置いたり、学習結果を随時アップロードする。 + +### 1. 環境を作る + +まずは永続ストレージにgit clone +```bash +mkdir -p /storage/sbv2 +cd /storage/sbv2 +git clone https://github.com/litagin02/Style-Bert-VITS2.git +``` +環境構築（デフォルトはPyTorch 1.x系、Python 3.9の模様） +```bash +pip install torch==2.1.2 torchvision==0.16.2 torchaudio==2.1.2 --index-url https://download.pytorch.org/whl/cu118 +pip install -r requirements.txt +``` +事前学習済みモデル等のダウンロード、またパスを`/notebooks/`以下のものに設定 +```bash +python initialize.py --skip_jvnv --dataset_root /notebooks/Data --assets_root /notebooks/model_assets +``` + +### 2. データセットの準備 +以下では`username/voices`というデータセットリポジトリにある`Foo.zip`というデータセットを使うことを想定しています。 +```bash +cd /nodtebooks +huggingface-cli login # 事前にトークンが必要 +huggingface-cli download username/voices Foo.zip --repo-type dataset --local-dir . +``` + +- zipファイル中身が既に`raw`と`esd.list`があるデータ（スライス・書き起こし済み）の場合 +```bash +mkdir -p Data/Foo +unzip Foo.zip -d Data/Foo +rm Foo.zip +cd /storage/sbv2/Style-Bert-VITS2 +``` + +- zipファイルが音声ファイルのみの場合 +```bash +mkdir inputs +unzip Foo.zip -d inputs +cd /storage/sbv2/Style-Bert-VITS2 +python slice.py --model_name Foo -i /notebooks/inputs +python transcribe.py --model_name Foo +``` + +それが終わったら、以下のコマンドで一括前処理を行う（パラメータは各自お好み、バッチサイズ6でVRAM 16GBギリくらい）。 +```bash +python preprocess_all.py --model_name Foo -b 6 -e 300 --use_jp_extra +``` + +### 3. 学習 + +Hugging faceの`username/sbv2-private`というモデルリポジトリに学習済みモデルをアップロードすることを想定しています。事前に`huggingface-cli login`でログインしておくこと。 +```bash +python train_ms_jp_extra.py --repo_id username/sbv2-private +``` +(JP-Extraでない場合は`train_ms.py`を使う) diff --git a/preprocess_all.py b/preprocess_all.py index 8ff4e34..954af0b 100644 --- a/preprocess_all.py +++ b/preprocess_all.py @@ -69,11 +69,6 @@ if __name__ == "__main__": help="Log interval", default=200, ) - parser.add_argument( - "--skip_invalid", - action="store_true", - help="Skip invalid", - ) args = parser.parse_args() diff --git a/requirements.txt b/requirements.txt index 62b4d60..f5325d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,8 @@ numpy psutil pyannote.audio>=3.1.0 pyloudnorm -pyopenjtalk-prebuilt +# pyopenjtalk-prebuilt # Should be manually uninstalled +pyopenjtalk-dict pypinyin pyworld PyYAML From ea28ec90589109af71c415025a74cd69d9dccb24 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Mon, 19 Feb 2024 20:17:31 +0900 Subject: [PATCH 19/51] Fix typo --- docs/paperspace.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/paperspace.md b/docs/paperspace.md index d67253e..f5d3770 100644 --- a/docs/paperspace.md +++ b/docs/paperspace.md @@ -36,7 +36,7 @@ python initialize.py --skip_jvnv --dataset_root /notebooks/Data --assets_root /n ### 2. データセットの準備 以下では`username/voices`というデータセットリポジトリにある`Foo.zip`というデータセットを使うことを想定しています。 ```bash -cd /nodtebooks +cd /notebooks huggingface-cli login # 事前にトークンが必要 huggingface-cli download username/voices Foo.zip --repo-type dataset --local-dir . ``` From a04e569c86cfd73898142c1c0a4f6509b35090fe Mon Sep 17 00:00:00 2001 From: litagin02 Date: Tue, 20 Feb 2024 00:42:00 +0900 Subject: [PATCH 20/51] Fix and upload all Data/ to hub --- preprocess_all.py | 1 - train_ms_jp_extra.py | 18 ++++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/preprocess_all.py b/preprocess_all.py index 954af0b..679abb2 100644 --- a/preprocess_all.py +++ b/preprocess_all.py @@ -87,5 +87,4 @@ if __name__ == "__main__": use_jp_extra=args.use_jp_extra, val_per_lang=args.val_per_lang, log_interval=args.log_interval, - skip_invalid=args.skip_invalid, ) diff --git a/train_ms_jp_extra.py b/train_ms_jp_extra.py index 75e72c5..f53d401 100644 --- a/train_ms_jp_extra.py +++ b/train_ms_jp_extra.py @@ -175,6 +175,16 @@ def run(): os.path.join(config.out_dir, "style_vectors.npy"), ) + if args.repo_id is not None: + # Upload Dataset + api.upload_folder( + repo_id=hps.repo_id, + folder_path=config.dataset_path, + path_in_repo=f"Data/{config.model_name}", + delete_patterns="*.pth", + run_as_future=True, + ) + torch.manual_seed(hps.train.seed) torch.cuda.set_device(local_rank) @@ -577,8 +587,8 @@ def run(): if hps.repo_id is not None: future1 = api.upload_folder( repo_id=hps.repo_id, - folder_path=model_dir, - path_in_repo=f"Data/{config.model_name}/models", + folder_path=config.dataset_path, + path_in_repo=f"Data/{config.model_name}", delete_patterns="*.pth", run_as_future=True, ) @@ -947,8 +957,8 @@ def train_and_evaluate( if hps.repo_id is not None: api.upload_folder( repo_id=hps.repo_id, - folder_path=hps.model_dir, - path_in_repo=f"Data/{config.model_name}/models", + folder_path=config.dataset_path, + path_in_repo=f"Data/{config.model_name}", delete_patterns="*.pth", run_as_future=True, ) From 5b60bf705a03d3881c96a0913b1e52628c042949 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Tue, 20 Feb 2024 00:52:12 +0900 Subject: [PATCH 21/51] Upload all Data/ to hub --- train_ms.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/train_ms.py b/train_ms.py index c16dda7..51e8d61 100644 --- a/train_ms.py +++ b/train_ms.py @@ -171,6 +171,16 @@ def run(): os.path.join(config.out_dir, "style_vectors.npy"), ) + if args.repo_id is not None: + # Upload Dataset + api.upload_folder( + repo_id=hps.repo_id, + folder_path=config.dataset_path, + path_in_repo=f"Data/{config.model_name}", + delete_patterns="*.pth", + run_as_future=True, + ) + torch.manual_seed(hps.train.seed) torch.cuda.set_device(local_rank) @@ -489,8 +499,8 @@ def run(): if hps.repo_id is not None: future1 = api.upload_folder( repo_id=hps.repo_id, - folder_path=model_dir, - path_in_repo=f"Data/{config.model_name}/models", + folder_path=config.dataset_path, + path_in_repo=f"Data/{config.model_name}", delete_patterns="*.pth", run_as_future=True, ) @@ -790,8 +800,8 @@ def train_and_evaluate( if hps.repo_id is not None: api.upload_folder( repo_id=hps.repo_id, - folder_path=hps.model_dir, - path_in_repo=f"Data/{config.model_name}/models", + folder_path=config.dataset_path, + path_in_repo=f"Data/{config.model_name}", delete_patterns="*.pth", run_as_future=True, ) From f8a1d98c6f4c1397da66d4d33b97f3b1d2bce308 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Tue, 20 Feb 2024 21:06:01 +0900 Subject: [PATCH 22/51] Change text error log filename --- preprocess_text.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/preprocess_text.py b/preprocess_text.py index 68534ea..7ba2310 100644 --- a/preprocess_text.py +++ b/preprocess_text.py @@ -55,7 +55,7 @@ def preprocess( if cleaned_path == "" or cleaned_path is None: cleaned_path = transcription_path + ".cleaned" - error_log_path = os.path.join(os.path.dirname(cleaned_path), "error.log") + error_log_path = os.path.join(os.path.dirname(cleaned_path), "text_error.log") error_count = 0 if clean: From c8778e98f18d839472bbcc748bd4455885dca6ed Mon Sep 17 00:00:00 2001 From: Serick Date: Wed, 21 Feb 2024 23:45:48 +0900 Subject: [PATCH 23/51] =?UTF-8?q?speech=5Fmos.py=E3=81=8C=E5=87=BA?= =?UTF-8?q?=E5=8A=9B=E3=81=99=E3=82=8BCSV=E3=81=ABBOM=E3=82=92=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- speech_mos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/speech_mos.py b/speech_mos.py index 8464223..d69a23a 100644 --- a/speech_mos.py +++ b/speech_mos.py @@ -99,7 +99,7 @@ for model_file, step, scores in results: logger.info(f"{model_file}: {scores[-1]}") with open( - mos_result_dir / f"mos_{model_name}.csv", "w", encoding="utf-8", newline="" + mos_result_dir / f"mos_{model_name}.csv", "w", encoding="utf_8_sig", newline="" ) as f: writer = csv.writer(f) writer.writerow(["model_path"] + ["step"] + test_texts + ["mean"]) From a6a7a30fc64654e13ccaddfeed111cfb919faa5c Mon Sep 17 00:00:00 2001 From: litagin02 Date: Thu, 22 Feb 2024 09:42:42 +0900 Subject: [PATCH 24/51] First check repo-id is valid --- train_ms.py | 38 ++++++++++++++++++++++++++------------ train_ms_jp_extra.py | 40 ++++++++++++++++++++++++++++------------ 2 files changed, 54 insertions(+), 24 deletions(-) diff --git a/train_ms.py b/train_ms.py index 51e8d61..fb7fa9c 100644 --- a/train_ms.py +++ b/train_ms.py @@ -159,6 +159,30 @@ def run(): config.out_dir: The directory for model assets of this model (for inference). default: `model_assets/{model_name}`. """ + + if args.repo_id is not None: + # First try to upload config.json to check if the repo exists + try: + api.upload_file( + path_or_fileobj=args.config, + path_in_repo=f"Data/{config.model_name}/config.json", + repo_id=hps.repo_id, + ) + except Exception as e: + logger.error(e) + logger.error( + f"Failed to upload files to the repo {hps.repo_id}. Please check if the repo exists and you have logged in using `huggingface-cli login`." + ) + raise e + # Upload Data dir for resuming training + api.upload_folder( + repo_id=hps.repo_id, + folder_path=config.dataset_path, + path_in_repo=f"Data/{config.model_name}", + delete_patterns="*.pth", # Only keep the latest checkpoint + run_as_future=True, + ) + os.makedirs(config.out_dir, exist_ok=True) if not args.skip_default_style: @@ -171,16 +195,6 @@ def run(): os.path.join(config.out_dir, "style_vectors.npy"), ) - if args.repo_id is not None: - # Upload Dataset - api.upload_folder( - repo_id=hps.repo_id, - folder_path=config.dataset_path, - path_in_repo=f"Data/{config.model_name}", - delete_patterns="*.pth", - run_as_future=True, - ) - torch.manual_seed(hps.train.seed) torch.cuda.set_device(local_rank) @@ -501,7 +515,7 @@ def run(): repo_id=hps.repo_id, folder_path=config.dataset_path, path_in_repo=f"Data/{config.model_name}", - delete_patterns="*.pth", + delete_patterns="*.pth", # Only keep the latest checkpoint run_as_future=True, ) future2 = api.upload_folder( @@ -802,7 +816,7 @@ def train_and_evaluate( repo_id=hps.repo_id, folder_path=config.dataset_path, path_in_repo=f"Data/{config.model_name}", - delete_patterns="*.pth", + delete_patterns="*.pth", # Only keep the latest checkpoint run_as_future=True, ) api.upload_folder( diff --git a/train_ms_jp_extra.py b/train_ms_jp_extra.py index f53d401..0cc9d2a 100644 --- a/train_ms_jp_extra.py +++ b/train_ms_jp_extra.py @@ -163,6 +163,30 @@ def run(): config.out_dir: The directory for model assets of this model (for inference). default: `model_assets/{model_name}`. """ + + if args.repo_id is not None: + # First try to upload config.json to check if the repo exists + try: + api.upload_file( + path_or_fileobj=args.config, + path_in_repo=f"Data/{config.model_name}/config.json", + repo_id=hps.repo_id, + ) + except Exception as e: + logger.error(e) + logger.error( + f"Failed to upload files to the repo {hps.repo_id}. Please check if the repo exists and you have logged in using `huggingface-cli login`." + ) + raise e + # Upload Data dir for resuming training + api.upload_folder( + repo_id=hps.repo_id, + folder_path=config.dataset_path, + path_in_repo=f"Data/{config.model_name}", + delete_patterns="*.pth", # Only keep the latest checkpoint + ignore_patterns=f"{config.dataset_path}/raw", # Ignore raw data + run_as_future=True, + ) os.makedirs(config.out_dir, exist_ok=True) if not args.skip_default_style: @@ -175,16 +199,6 @@ def run(): os.path.join(config.out_dir, "style_vectors.npy"), ) - if args.repo_id is not None: - # Upload Dataset - api.upload_folder( - repo_id=hps.repo_id, - folder_path=config.dataset_path, - path_in_repo=f"Data/{config.model_name}", - delete_patterns="*.pth", - run_as_future=True, - ) - torch.manual_seed(hps.train.seed) torch.cuda.set_device(local_rank) @@ -589,7 +603,8 @@ def run(): repo_id=hps.repo_id, folder_path=config.dataset_path, path_in_repo=f"Data/{config.model_name}", - delete_patterns="*.pth", + delete_patterns="*.pth", # Only keep the latest checkpoint + ignore_patterns=f"{config.dataset_path}/raw", # Ignore raw data run_as_future=True, ) future2 = api.upload_folder( @@ -959,7 +974,8 @@ def train_and_evaluate( repo_id=hps.repo_id, folder_path=config.dataset_path, path_in_repo=f"Data/{config.model_name}", - delete_patterns="*.pth", + delete_patterns="*.pth", # Only keep the latest checkpoint + ignore_patterns=f"{config.dataset_path}/raw", # Ignore raw data run_as_future=True, ) api.upload_folder( From 66ed7ce60d6bcb3df6edfe9b8c486a35aab5ffd4 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Thu, 22 Feb 2024 09:52:20 +0900 Subject: [PATCH 25/51] Avoid Japanese message in bat file --- scripts/Install-Style-Bert-VITS2-CPU.bat | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/Install-Style-Bert-VITS2-CPU.bat b/scripts/Install-Style-Bert-VITS2-CPU.bat index e5d106f..aaf3536 100644 --- a/scripts/Install-Style-Bert-VITS2-CPU.bat +++ b/scripts/Install-Style-Bert-VITS2-CPU.bat @@ -33,7 +33,7 @@ xcopy /QSY .\Style-Bert-VITS2-master\ .\Style-Bert-VITS2\ rmdir /s /q Style-Bert-VITS2-master echo ---------------------------------------- -echo Python環境の構築を開始します。 +echo Setup Python and Virtual Environment echo ---------------------------------------- @REM Pythonと仮想環境のセットアップを呼び出す（仮想環境が有効化されて戻ってくる） @@ -45,7 +45,7 @@ pip install -r Style-Bert-VITS2\requirements.txt if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) echo ---------------------------------------- -echo 環境構築が完了しました。モデルのダウンロードを開始します。 +echo Environment setup is complete. Start downloading the model. echo ---------------------------------------- @REM Style-Bert-VITS2フォルダに移動 @@ -55,8 +55,7 @@ pushd Style-Bert-VITS2 python initialize.py echo ---------------------------------------- -echo モデルのダウンロードが完了し、インストールが完了しました！ -echo 音声合成のWebUIを起動します。 +echo Model download is complete. Start the WebUI of the voice synthesis. echo ---------------------------------------- @REM 音声合成WebUIの起動 From fc32ec26cdb564bcdf4962c5f87b7518f314193a Mon Sep 17 00:00:00 2001 From: litagin02 Date: Thu, 22 Feb 2024 09:54:32 +0900 Subject: [PATCH 26/51] Update about paperspace --- Dockerfile | 97 ++++++++++++++++++++++++++++++++++++++++++++++ docs/CLI.md | 3 +- docs/paperspace.md | 16 +++++++- 3 files changed, 113 insertions(+), 3 deletions(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..790c9c2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,97 @@ +# ================================================================== +# Initial setup +# ------------------------------------------------------------------ + +# Ubuntu 22.04 as base image +FROM ubuntu:22.04 +# RUN yes| unminimize + +# Set ENV variables +ENV LANG C.UTF-8 +ENV SHELL=/bin/bash +ENV DEBIAN_FRONTEND=noninteractive + +ENV APT_INSTALL="apt-get install -y --no-install-recommends" +ENV PIP_INSTALL="python3 -m pip --no-cache-dir install --upgrade" +ENV GIT_CLONE="git clone --depth 10" + +# ================================================================== +# Tools +# ------------------------------------------------------------------ + +RUN apt-get update && \ + $APT_INSTALL \ + sudo \ + build-essential \ + ca-certificates \ + wget \ + curl \ + git \ + zip \ + unzip \ + nano \ + ffmpeg \ + software-properties-common \ + gnupg \ + python3 \ + python3-pip \ + python3-dev + +# ================================================================== +# Git-lfs +# ------------------------------------------------------------------ + +RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash && \ + $APT_INSTALL git-lfs + + +# Add symlink so python and python3 commands use same python3.9 executable +RUN ln -s /usr/bin/python3 /usr/local/bin/python + +# ================================================================== +# Installing CUDA packages (CUDA Toolkit 12.0 and CUDNN 8.9.7) +# ------------------------------------------------------------------ +RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-ubuntu2204.pin && \ + mv cuda-ubuntu2204.pin /etc/apt/preferences.d/cuda-repository-pin-600 && \ + wget https://developer.download.nvidia.com/compute/cuda/12.0.0/local_installers/cuda-repo-ubuntu2204-12-0-local_12.0.0-525.60.13-1_amd64.deb && \ + dpkg -i cuda-repo-ubuntu2204-12-0-local_12.0.0-525.60.13-1_amd64.deb && \ + cp /var/cuda-repo-ubuntu2204-12-0-local/cuda-*-keyring.gpg /usr/share/keyrings/ && \ + apt-get update && \ + $APT_INSTALL cuda && \ + rm cuda-repo-ubuntu2204-12-0-local_12.0.0-525.60.13-1_amd64.deb + +# Installing CUDNN +RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/3bf863cc.pub && \ + add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/ /" && \ + apt-get update && \ + $APT_INSTALL libcudnn8=8.9.7.29-1+cuda12.2 \ + libcudnn8-dev=8.9.7.29-1+cuda12.2 + + +ENV PATH=$PATH:/usr/local/cuda/bin +ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH + + +# ================================================================== +# PyTorch +# ------------------------------------------------------------------ + +# Based on https://pytorch.org/get-started/locally/ + +RUN $PIP_INSTALL torch==2.1.2 torchvision==0.16.2 torchaudio==2.1.2 --index-url https://download.pytorch.org/whl/cu118 + + +RUN $PIP_INSTALL jupyterlab + +# Install requirements.txt from the project +COPY requirements.txt /tmp/requirements.txt +RUN $PIP_INSTALL -r /tmp/requirements.txt +RUN rm /tmp/requirements.txt + +# ================================================================== +# Startup +# ------------------------------------------------------------------ + +EXPOSE 8888 6006 + +CMD jupyter lab --allow-root --ip=0.0.0.0 --no-browser --ServerApp.trust_xheaders=True --ServerApp.disable_check_xsrf=False --ServerApp.allow_remote_access=True --ServerApp.allow_origin='*' --ServerApp.allow_credentials=True \ No newline at end of file diff --git a/docs/CLI.md b/docs/CLI.md index 3873475..88ad5fd 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -87,8 +87,9 @@ python train_ms.py [--repo_id /] If using JP-Extra model: ```bash -python train_ms_jp_extra.py [--repo_id /] +python train_ms_jp_extra.py [--repo_id /] [--skip_default_style] ``` Optional: - `--repo_id`: Hugging Face repository ID to upload the trained model to. You should have logged in using `huggingface-cli login` before running this command. +- `--skip_default_style`: Skip making the default style vector. Use this if you want to resume training (since the default style vector is already made). diff --git a/docs/paperspace.md b/docs/paperspace.md index f5d3770..924d3b3 100644 --- a/docs/paperspace.md +++ b/docs/paperspace.md @@ -25,8 +25,8 @@ git clone https://github.com/litagin02/Style-Bert-VITS2.git ``` 環境構築（デフォルトはPyTorch 1.x系、Python 3.9の模様） ```bash -pip install torch==2.1.2 torchvision==0.16.2 torchaudio==2.1.2 --index-url https://download.pytorch.org/whl/cu118 -pip install -r requirements.txt +cd /storage/sbv2/Style-Bert-VITS2 +pip install torch==2.1.2 torchvision==0.16.2 torchaudio==2.1.2 --index-url https://download.pytorch.org/whl/cu118 && pip install -r requirements.txt ``` 事前学習済みモデル等のダウンロード、またパスを`/notebooks/`以下のものに設定 ```bash @@ -70,3 +70,15 @@ Hugging faceの`username/sbv2-private`というモデルリポジトリに学習 python train_ms_jp_extra.py --repo_id username/sbv2-private ``` (JP-Extraでない場合は`train_ms.py`を使う) + +### 4. 学習再開 + +Notebooksの時間制限が切れてから別Notebooksで同じモデルを学習を再開する場合（環境構築は必要）。 +```bash +huggingface-cli login +cd /notebooks +huggingface-cli download username/sbv2-private --include "Data/Foo/*" --local-dir . +cd /storage/sbv2/Style-Bert-VITS2 +python train_ms_jp_extra.py --repo_id username/sbv2-private --skip_default_style +``` +前回の設定が残っているので特に前処理等は不要。 \ No newline at end of file From 476e06c48b1f8409bf49ed8796f380c02fa7d44a Mon Sep 17 00:00:00 2001 From: litagin02 Date: Thu, 22 Feb 2024 09:55:55 +0900 Subject: [PATCH 27/51] Improve doc --- webui_train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webui_train.py b/webui_train.py index 2fe53cc..c3a876e 100644 --- a/webui_train.py +++ b/webui_train.py @@ -87,7 +87,7 @@ def initialize( model_path = os.path.join(dataset_path, "models") if os.path.exists(model_path): - logger.warning(f"Step 1: {model_path} already exists, so copy it to backup.") + logger.warning(f"Step 1: {model_path} already exists, so copy it to backup to {model_path}_backup") shutil.copytree( src=model_path, dst=os.path.join(dataset_path, "models_backup"), From de539e74f912a19425a5b1709c88b357cade2c60 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Fri, 23 Feb 2024 09:06:37 +0900 Subject: [PATCH 28/51] Improve initial prompt docs --- colab.ipynb | 5 ++++- webui_dataset.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/colab.ipynb b/colab.ipynb index 1a5369a..37b708f 100644 --- a/colab.ipynb +++ b/colab.ipynb @@ -115,8 +115,11 @@ "# モデル名（話者名）を入力\n", "model_name = \"your_model_name\"\n", "\n", + "# こういうふうに書き起こして欲しいという例文（句読点の入れ方・笑い方や固有名詞等）\n", + "initial_prompt = \"こんにちは。元気、ですかー？ふふっ、私は……ちゃんと元気だよ！\"\n", + "\n", "!python slice.py -i {input_dir} -o {dataset_root}/{model_name}/raw\n", - "!python transcribe.py -i {dataset_root}/{model_name}/raw -o {dataset_root}/{model_name}/esd.list --speaker_name {model_name} --compute_type float16" + "!python transcribe.py -i {dataset_root}/{model_name}/raw -o {dataset_root}/{model_name}/esd.list --speaker_name {model_name} --compute_type float16 --initial_prompt {initial_prompt}" ] }, { diff --git a/webui_dataset.py b/webui_dataset.py index a795575..59c7270 100644 --- a/webui_dataset.py +++ b/webui_dataset.py @@ -169,8 +169,8 @@ with gr.Blocks(theme=GRADIO_THEME) as app: language = gr.Dropdown(["ja", "en", "zh"], value="ja", label="言語") initial_prompt = gr.Textbox( label="初期プロンプト", - placeholder="こんにちは。元気、ですかー？ふふっ、私は……ちゃんと元気だよ！", - info="このように書き起こしてほしいという例文、日本語なら省略可、英語等なら書いてください", + value="こんにちは。元気、ですかー？ふふっ、私は……ちゃんと元気だよ！", + info="このように書き起こしてほしいという例文（句読点の入れ方・笑い方・固有名詞等）", ) transcribe_button = gr.Button("音声の文字起こし") result2 = gr.Textbox(label="結果") From e16cdad6e9d154a11993f0eaa721b9bfdb008d46 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Fri, 23 Feb 2024 13:21:14 +0900 Subject: [PATCH 29/51] Feat: add editor server backend --- .gitignore | 4 + app.py | 3 +- common/constants.py | 4 + common/tts_model.py | 110 ++++++++++------ infer.py | 15 ++- server_editor.py | 285 ++++++++++++++++++++++++++++++++++++++++++ text/__init__.py | 19 ++- text/cleaner.py | 6 +- text/japanese.py | 25 +++- text/japanese_bert.py | 6 +- 10 files changed, 423 insertions(+), 54 deletions(-) create mode 100644 server_editor.py diff --git a/.gitignore b/.gitignore index 31602b0..5ebe1d8 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,7 @@ venv/ safetensors.ipynb *.wav +/Web/ + +# pyopenjtalk's dictionary +*.dic diff --git a/app.py b/app.py index 3f0f91e..d99229b 100644 --- a/app.py +++ b/app.py @@ -3,6 +3,7 @@ import datetime import json import os import sys +from pathlib import Path from typing import Optional import gradio as gr @@ -271,7 +272,7 @@ if __name__ == "__main__": help="Do not launch app automatically", ) args = parser.parse_args() - model_dir = args.dir + model_dir = Path(args.dir) if args.cpu: device = "cpu" diff --git a/common/constants.py b/common/constants.py index cd118e2..5fe2e0a 100644 --- a/common/constants.py +++ b/common/constants.py @@ -5,6 +5,10 @@ import enum GRADIO_THEME: str = "NoCrypt/miku" LATEST_VERSION: str = "2.3" + +USER_DICT_CSV_PATH: str = "dict/user_dict.csv" +USER_DICT_PATH: str = "dict/user.dic" + DEFAULT_STYLE: str = "Neutral" DEFAULT_STYLE_WEIGHT: float = 5.0 diff --git a/common/tts_model.py b/common/tts_model.py index 0482096..ccde9ac 100644 --- a/common/tts_model.py +++ b/common/tts_model.py @@ -1,6 +1,7 @@ import os import warnings -from typing import Dict, List, Optional, Union +from pathlib import Path +from typing import Optional, Union import gradio as gr import numpy as np @@ -29,12 +30,13 @@ from .log import logger def adjust_voice(fs, wave, pitch_scale, intonation_scale): if pitch_scale == 1.0 and intonation_scale == 1.0: + # 初期値の場合は、音質劣化を避けるためにそのまま返す return fs, wave + # pyworldでf0を加工して合成 # pyworldよりもよいのがあるかもしれないが…… wave = wave.astype(np.double) - logger.debug(f"wave: shape={wave.shape}, max={max(wave)}, min={min(wave)}") f0, t = pyworld.harvest(wave, fs) # 質が高そうだしとりあえずharvestにしておく @@ -43,7 +45,6 @@ def adjust_voice(fs, wave, pitch_scale, intonation_scale): non_zero_f0 = [f for f in f0 if f != 0] f0_mean = sum(non_zero_f0) / len(non_zero_f0) - logger.debug(f"f0: shape={f0.shape}, mean={f0_mean}, max={max(f0)}") for i, f in enumerate(f0): if f == 0: @@ -56,21 +57,21 @@ def adjust_voice(fs, wave, pitch_scale, intonation_scale): class Model: def __init__( - self, model_path: str, config_path: str, style_vec_path: str, device: str + self, model_path: Path, config_path: Path, style_vec_path: Path, device: str ): - self.model_path: str = model_path - self.config_path: str = config_path + self.model_path: Path = model_path + self.config_path: Path = config_path + self.style_vec_path: Path = style_vec_path self.device: str = device - self.style_vec_path: str = style_vec_path self.hps: utils.HParams = utils.get_hparams_from_file(self.config_path) - self.spk2id: Dict[str, int] = self.hps.data.spk2id - self.id2spk: Dict[int, str] = {v: k for k, v in self.spk2id.items()} + self.spk2id: dict[str, int] = self.hps.data.spk2id + self.id2spk: dict[int, str] = {v: k for k, v in self.spk2id.items()} self.num_styles: int = self.hps.data.num_styles if hasattr(self.hps.data, "style2id"): - self.style2id: Dict[str, int] = self.hps.data.style2id + self.style2id: dict[str, int] = self.hps.data.style2id else: - self.style2id: Dict[str, int] = {str(i): i for i in range(self.num_styles)} + self.style2id: dict[str, int] = {str(i): i for i in range(self.num_styles)} if len(self.style2id) != self.num_styles: raise ValueError( f"Number of styles ({self.num_styles}) does not match the number of style2id ({len(self.style2id)})" @@ -86,7 +87,7 @@ class Model: def load_net_g(self): self.net_g = get_net_g( - model_path=self.model_path, + model_path=str(self.model_path), version=self.hps.version, device=self.device, hps=self.hps, @@ -128,6 +129,7 @@ class Model: given_tone: Optional[list[int]] = None, pitch_scale: float = 1.0, intonation_scale: float = 1.0, + ignore_unknown: bool = False, ) -> tuple[int, np.ndarray]: logger.info(f"Start generating audio data from text:\n{text}") if language != "JP" and self.hps.version.endswith("JP-Extra"): @@ -165,6 +167,7 @@ class Model: assist_text_weight=assist_text_weight, style_vec=style_vector, given_tone=given_tone, + ignore_unknown=ignore_unknown, ) else: texts = text.split("\n") @@ -187,14 +190,12 @@ class Model: assist_text=assist_text, assist_text_weight=assist_text_weight, style_vec=style_vector, + ignore_unknown=ignore_unknown, ) ) if i != len(texts) - 1: audios.append(np.zeros(int(44100 * split_interval))) audio = np.concatenate(audios) - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - audio = convert_to_16_bit_wav(audio) logger.info("Audio data generated successfully") fs, audio = adjust_voice( fs=self.hps.data.sampling_rate, @@ -202,46 +203,79 @@ class Model: pitch_scale=pitch_scale, intonation_scale=intonation_scale, ) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + audio = convert_to_16_bit_wav(audio) return (fs, audio) class ModelHolder: - def __init__(self, root_dir: str, device: str): - self.root_dir: str = root_dir + def __init__(self, root_dir: Path, device: str): + self.root_dir: Path = root_dir self.device: str = device - self.model_files_dict: Dict[str, List[str]] = {} + self.model_files_dict: dict[str, list[Path]] = {} self.current_model: Optional[Model] = None - self.model_names: List[str] = [] - self.models: List[Model] = [] + self.model_names: list[str] = [] + self.models: list[Model] = [] self.refresh() def refresh(self): self.model_files_dict = {} self.model_names = [] self.current_model = None - model_dirs = [ - d - for d in os.listdir(self.root_dir) - if os.path.isdir(os.path.join(self.root_dir, d)) - ] - for model_name in model_dirs: - model_dir = os.path.join(self.root_dir, model_name) + + model_dirs = [d for d in self.root_dir.iterdir() if d.is_dir()] + for model_dir in model_dirs: model_files = [ - os.path.join(model_dir, f) - for f in os.listdir(model_dir) - if f.endswith(".pth") or f.endswith(".pt") or f.endswith(".safetensors") + f + for f in model_dir.iterdir() + if f.suffix in [".pth", ".pt", ".safetensors"] ] if len(model_files) == 0: - logger.warning( - f"No model files found in {self.root_dir}/{model_name}, so skip it" - ) + logger.warning(f"No model files found in {model_dir}, so skip it") continue - self.model_files_dict[model_name] = model_files - self.model_names.append(model_name) + self.model_files_dict[model_dir.name] = model_files + self.model_names.append(model_dir.name) + + def models_info(self): + if hasattr(self, "_models_info"): + return self._models_info + result = [] + for name, files in self.model_files_dict.items(): + # Get styles + config_path = self.root_dir / name / "config.json" + hps = utils.get_hparams_from_file(config_path) + style2id: dict[str, int] = hps.data.style2id + styles = list(style2id.keys()) + result.append( + { + "name": name, + "files": [str(f) for f in files], + "styles": styles, + } + ) + self._models_info = result + return result + + def load_model(self, model_name: str, model_path_str: str): + model_path = Path(model_path_str) + if model_name not in self.model_files_dict: + raise ValueError(f"Model `{model_name}` is not found") + if model_path not in self.model_files_dict[model_name]: + raise ValueError(f"Model file `{model_path}` is not found") + if self.current_model is None or self.current_model.model_path != model_path: + self.current_model = Model( + model_path=model_path, + config_path=self.root_dir / model_name / "config.json", + style_vec_path=self.root_dir / model_name / "style_vectors.npy", + device=self.device, + ) + return self.current_model def load_model_gr( - self, model_name: str, model_path: str + self, model_name: str, model_path_str: str ) -> tuple[gr.Dropdown, gr.Button, gr.Dropdown]: + model_path = Path(model_path_str) if model_name not in self.model_files_dict: raise ValueError(f"Model `{model_name}` is not found") if model_path not in self.model_files_dict[model_name]: @@ -260,8 +294,8 @@ class ModelHolder: ) self.current_model = Model( model_path=model_path, - config_path=os.path.join(self.root_dir, model_name, "config.json"), - style_vec_path=os.path.join(self.root_dir, model_name, "style_vectors.npy"), + config_path=self.root_dir / model_name / "config.json", + style_vec_path=self.root_dir / model_name / "style_vectors.npy", device=self.device, ) speakers = list(self.current_model.spk2id.keys()) diff --git a/infer.py b/infer.py index 6febc08..914a355 100644 --- a/infer.py +++ b/infer.py @@ -52,9 +52,12 @@ def get_text( assist_text=None, assist_text_weight=0.7, given_tone=None, + ignore_unknown=False, ): use_jp_extra = hps.version.endswith("JP-Extra") - norm_text, phone, tone, word2ph = clean_text(text, language_str, use_jp_extra) + norm_text, phone, tone, word2ph = clean_text( + text, language_str, use_jp_extra, ignore_unknown=ignore_unknown + ) if given_tone is not None: if len(given_tone) != len(phone): raise InvalidToneError( @@ -71,7 +74,13 @@ def get_text( word2ph[i] = word2ph[i] * 2 word2ph[0] += 1 bert_ori = get_bert( - norm_text, word2ph, language_str, device, assist_text, assist_text_weight + norm_text, + word2ph, + language_str, + device, + assist_text, + assist_text_weight, + ignore_unknown, ) del word2ph assert bert_ori.shape[-1] == len(phone), phone @@ -118,6 +127,7 @@ def infer( assist_text=None, assist_text_weight=0.7, given_tone=None, + ignore_unknown=False, ): is_jp_extra = hps.version.endswith("JP-Extra") bert, ja_bert, en_bert, phones, tones, lang_ids = get_text( @@ -128,6 +138,7 @@ def infer( assist_text=assist_text, assist_text_weight=assist_text_weight, given_tone=given_tone, + ignore_unknown=ignore_unknown, ) if skip_start: phones = phones[3:] diff --git a/server_editor.py b/server_editor.py new file mode 100644 index 0000000..1aebe6e --- /dev/null +++ b/server_editor.py @@ -0,0 +1,285 @@ +import csv +import os +import sys +from io import BytesIO +from pathlib import Path + +import numpy as np +import pyopenjtalk +import uvicorn +from fastapi import FastAPI, HTTPException, status +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse, Response +from fastapi.staticfiles import StaticFiles +from pydantic import BaseModel +from scipy.io import wavfile + +from common.constants import ( + DEFAULT_ASSIST_TEXT_WEIGHT, + DEFAULT_LENGTH, + DEFAULT_NOISE, + DEFAULT_NOISEW, + DEFAULT_SDP_RATIO, + DEFAULT_STYLE, + DEFAULT_STYLE_WEIGHT, + USER_DICT_CSV_PATH, + USER_DICT_PATH, + Languages, + LATEST_VERSION, +) +from common.log import logger +from common.tts_model import ModelHolder +from text.japanese import g2kata_tone, kata_tone2phone_tone + + +class AudioResponse(Response): + media_type = "audio/wav" + + +origins = [ + "http://localhost:3000", + "http://localhost:8000", + "http://127.0.0.1:3000", + "http://127.0.0.1:8000", +] + +device = "cuda" +model_dir = Path("model_assets/server_test") +model_holder = ModelHolder(model_dir, device) +if len(model_holder.model_names) == 0: + logger.error(f"Models not found in {model_dir}.") + sys.exit(1) + +app = FastAPI() + + +app.add_middleware( + CORSMiddleware, + allow_origins=origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + + +@app.get("/version") +def version() -> str: + return LATEST_VERSION + + +class MoraTone(BaseModel): + mora: str + tone: int + + +class G2PRequest(BaseModel): + text: str + + +@app.post("/g2p") +async def read_item(item: G2PRequest): + try: + kata_tone_list = g2kata_tone(item.text, ignore_unknown=True) + except Exception as e: + raise HTTPException( + status_code=400, + detail=f"Failed to convert {item.text} to katakana and tone, {e}", + ) + return [MoraTone(mora=kata, tone=tone) for kata, tone in kata_tone_list] + + +@app.get("/models_info") +def models_info(): + return model_holder.models_info() + + +class SynthesisRequest(BaseModel): + model: str + modelFile: str + text: str + moraToneList: list[MoraTone] + style: str = DEFAULT_STYLE + styleWeight: float = DEFAULT_STYLE_WEIGHT + assistText: str = "" + assistTextWeight: float = DEFAULT_ASSIST_TEXT_WEIGHT + speed: float = 1.0 + noise: float = DEFAULT_NOISE + noisew: float = DEFAULT_NOISEW + sdpRatio: float = DEFAULT_SDP_RATIO + language: Languages = Languages.JP + silenceAfter: float = 0.5 + pitchScale: float = 1.0 + intonationScale: float = 1.0 + + +@app.post("/synthesis", response_class=AudioResponse) +def synthesis(request: SynthesisRequest): + try: + model = model_holder.load_model( + model_name=request.model, model_path_str=request.modelFile + ) + except Exception as e: + logger.error(e) + raise HTTPException( + status_code=500, + detail=f"Failed to load model {request.model} from {request.modelFile}, {e}", + ) + text = request.text + kata_tone_list = [ + (mora_tone.mora, mora_tone.tone) for mora_tone in request.moraToneList + ] + phone_tone = kata_tone2phone_tone(kata_tone_list) + tone = [t for _, t in phone_tone] + sr, audio = model.infer( + text=text, + language=request.language.value, + sdp_ratio=request.sdpRatio, + noise=request.noise, + noisew=request.noisew, + length=1 / request.speed, + given_tone=tone, + style=request.style, + style_weight=request.styleWeight, + assist_text=request.assistText, + assist_text_weight=request.assistTextWeight, + use_assist_text=bool(request.assistText), + line_split=False, + ignore_unknown=True, + pitch_scale=request.pitchScale, + intonation_scale=request.intonationScale, + ) + + with BytesIO() as wavContent: + wavfile.write(wavContent, sr, audio) + return Response(content=wavContent.getvalue(), media_type="audio/wav") + + +class MultiSynthesisRequest(BaseModel): + lines: list[SynthesisRequest] + + +@app.post("/multi-synthesis", response_class=AudioResponse) +def multi_synthesis(request: MultiSynthesisRequest): + lines = request.lines + audios = [] + for i, req in enumerate(lines): + # Loade model + try: + model = model_holder.load_model( + model_name=req.model, model_path_str=req.modelFile + ) + except Exception as e: + logger.error(e) + raise HTTPException( + status_code=500, + detail=f"Failed to load model {req.model} from {req.modelFile}, {e}", + ) + text = req.text + kata_tone_list = [ + (mora_tone.mora, mora_tone.tone) for mora_tone in req.moraToneList + ] + phone_tone = kata_tone2phone_tone(kata_tone_list) + tone = [t for _, t in phone_tone] + sr, audio = model.infer( + text=text, + language=req.language.value, + sdp_ratio=req.sdpRatio, + noise=req.noise, + noisew=req.noisew, + length=1 / req.speed, + given_tone=tone, + style=req.style, + style_weight=req.styleWeight, + assist_text=req.assistText, + assist_text_weight=req.assistTextWeight, + use_assist_text=bool(req.assistText), + line_split=False, + ignore_unknown=True, + pitch_scale=req.pitchScale, + intonation_scale=req.intonationScale, + ) + audios.append(audio) + if i < len(lines) - 1: + silence = int(sr * req.silenceAfter) + audios.append(np.zeros(silence, dtype=np.int16)) + audio = np.concatenate(audios) + logger.debug(audio.dtype) + + with BytesIO() as wavContent: + wavfile.write(wavContent, sr, audio) + return Response(content=wavContent.getvalue(), media_type="audio/wav") + + +class AddUserDictWordRequest(BaseModel): + surface: str + pronunciation: str + accentType: str # アクセント核位置/モーラ数、例：1/3, アクセント核位置は1から始まる + priority: int = 5 + + +# コストの値は以下の値を参考にしている +# https://github.com/VOICEVOX/voicevox_engine/blob/master/voicevox_engine/user_dict/part_of_speech_data.py +COST_CANDIDATES = [-988, 3488, 4768, 6048, 7328, 8609, 8734, 8859, 8984, 9110, 14176] + + +@app.post("/user_dict_word") +def add_user_dict_word(request: AddUserDictWordRequest): + if request.surface == "" or request.pronunciation == "": + return JSONResponse( + status_code=status.HTTP_400_BAD_REQUEST, + content={"message": "単語か読みが空です。"}, + ) + user_dict_file = Path(USER_DICT_CSV_PATH) + user_dict_file.parent.mkdir(parents=True, exist_ok=True) + + # 新規追加または更新する単語のCSV行 + new_csv_row = f"{request.surface},,,{COST_CANDIDATES[request.priority]},名詞,固有名詞,一般,*,*,*,{request.surface},{request.pronunciation},{request.pronunciation},{request.accentType},*\n" + found = False + updated = False + + # ユーザー辞書ファイルが存在する場合、既存の単語をチェックし、必要に応じて更新する + if user_dict_file.exists(): + with user_dict_file.open(encoding="utf-8") as f: + lines = f.readlines() + + with user_dict_file.open("w", encoding="utf-8") as f: + for line in lines: + if line.split(",")[0] == request.surface: + found = True + if line.strip() != new_csv_row.strip(): + f.write(new_csv_row) + updated = True + else: + f.write(line) + else: + f.write(line) + # 単語が新しい場合、新規に追加 + if not found: + with user_dict_file.open("a", encoding="utf-8") as f: + f.write(new_csv_row) + + pyopenjtalk.unset_user_dict() + pyopenjtalk.mecab_dict_index(USER_DICT_CSV_PATH, USER_DICT_PATH) + pyopenjtalk.update_global_jtalk_with_user_dict(USER_DICT_PATH) + + if updated: + return JSONResponse( + status_code=status.HTTP_200_OK, + content={"message": "単語が既に存在し、更新されました。"}, + ) + elif found: + return JSONResponse( + status_code=status.HTTP_200_OK, + content={"message": "既に登録されている単語です。更新はありません。"}, + ) + else: + return JSONResponse( + status_code=status.HTTP_201_CREATED, + content={"message": "単語が新規追加されました。"}, + ) + + +app.mount("/", StaticFiles(directory="Web"), name="static") + +if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/text/__init__.py b/text/__init__.py index 495e57b..0554884 100644 --- a/text/__init__.py +++ b/text/__init__.py @@ -19,14 +19,25 @@ def cleaned_text_to_sequence(cleaned_text, tones, language): def get_bert( - norm_text, word2ph, language, device, assist_text=None, assist_text_weight=0.7 + text, + word2ph, + language, + device, + assist_text=None, + assist_text_weight=0.7, + ignore_unknown=False, ): from .chinese_bert import get_bert_feature as zh_bert from .english_bert_mock import get_bert_feature as en_bert from .japanese_bert import get_bert_feature as jp_bert lang_bert_func_map = {"ZH": zh_bert, "EN": en_bert, "JP": jp_bert} - bert = lang_bert_func_map[language]( - norm_text, word2ph, device, assist_text, assist_text_weight - ) + if language == "JP": + bert = lang_bert_func_map[language]( + text, word2ph, device, assist_text, assist_text_weight, ignore_unknown + ) + else: + bert = lang_bert_func_map[language]( + text, word2ph, device, assist_text, assist_text_weight + ) return bert diff --git a/text/cleaner.py b/text/cleaner.py index ab59b4e..759c6cc 100644 --- a/text/cleaner.py +++ b/text/cleaner.py @@ -4,11 +4,13 @@ from text import chinese, japanese, english, cleaned_text_to_sequence language_module_map = {"ZH": chinese, "JP": japanese, "EN": english} -def clean_text(text, language, use_jp_extra=True): +def clean_text(text, language, use_jp_extra=True, ignore_unknown=False): language_module = language_module_map[language] norm_text = language_module.text_normalize(text) if language == "JP": - phones, tones, word2ph = language_module.g2p(norm_text, use_jp_extra) + phones, tones, word2ph = language_module.g2p( + norm_text, use_jp_extra, ignore_unknown=ignore_unknown + ) else: phones, tones, word2ph = language_module.g2p(norm_text) return norm_text, phones, tones, word2ph diff --git a/text/japanese.py b/text/japanese.py index 2b0f24b..39c8c6a 100644 --- a/text/japanese.py +++ b/text/japanese.py @@ -2,11 +2,13 @@ # compatible with Julius https://github.com/julius-speech/segmentation-kit import re import unicodedata +from pathlib import Path import pyopenjtalk from num2words import num2words from transformers import AutoTokenizer +from common.constants import USER_DICT_PATH, USER_DICT_CSV_PATH from common.log import logger from text import punctuation from text.japanese_mora_list import ( @@ -14,6 +16,11 @@ from text.japanese_mora_list import ( mora_phonemes_to_mora_kata, ) +if Path(USER_DICT_CSV_PATH).exists(): + pyopenjtalk.mecab_dict_index(USER_DICT_CSV_PATH, USER_DICT_PATH) +if Path(USER_DICT_PATH).exists(): + pyopenjtalk.update_global_jtalk_with_user_dict(USER_DICT_PATH) + # 子音の集合 COSONANTS = set( [ @@ -160,7 +167,7 @@ def japanese_convert_numbers_to_words(text: str) -> str: def g2p( - norm_text: str, use_jp_extra: bool = True + norm_text: str, use_jp_extra: bool = True, ignore_unknown: bool = False ) -> tuple[list[str], list[int], list[int]]: """ 他で使われるメインの関数。`text_normalize()`で正規化された`norm_text`を受け取り、 @@ -182,7 +189,7 @@ def g2p( # sep_text: 単語単位の単語のリスト # sep_kata: 単語単位の単語のカタカナ読みのリスト - sep_text, sep_kata = text2sep_kata(norm_text) + sep_text, sep_kata = text2sep_kata(norm_text, ignore_unknown=ignore_unknown) # sep_phonemes: 各単語ごとの音素のリストのリスト sep_phonemes = handle_long([kata2phoneme_list(i) for i in sep_kata]) @@ -231,8 +238,8 @@ def g2p( return phones, tones, word2ph -def g2kata_tone(norm_text: str) -> list[tuple[str, int]]: - phones, tones, _ = g2p(norm_text, use_jp_extra=True) +def g2kata_tone(norm_text: str, ignore_unknown: bool = False) -> list[tuple[str, int]]: + phones, tones, _ = g2p(norm_text, use_jp_extra=True, ignore_unknown=ignore_unknown) return phone_tone2kata_tone(list(zip(phones, tones))) @@ -325,7 +332,9 @@ def g2phone_tone_wo_punct(text: str) -> list[tuple[str, int]]: return result -def text2sep_kata(norm_text: str) -> tuple[list[str], list[str]]: +def text2sep_kata( + norm_text: str, ignore_unknown: bool = False +) -> tuple[list[str], list[str]]: """ `text_normalize`で正規化済みの`norm_text`を受け取り、それを単語分割し、 分割された単語リストとその読み（カタカナor記号1文字）のリストのタプルを返す。 @@ -361,6 +370,9 @@ def text2sep_kata(norm_text: str) -> tuple[list[str], list[str]]: # wordは正規化されているので、`.`, `,`, `!`, `'`, `-`, `--` のいずれか if not set(word).issubset(set(punctuation)): # 記号繰り返しか判定 # ここはpyopenjtalkが読めない文字等のときに起こる + if ignore_unknown: + logger.error(f"Ignoring unknown: {word} in:\n{norm_text}") + continue raise ValueError(f"Cannot read: {word} in:\n{norm_text}") # yomiは元の記号のままに変更 yomi = word @@ -500,6 +512,9 @@ def handle_long(sep_phonemes: list[list[str]]) -> list[list[str]]: おそらく長音記号とダッシュを勘違いしていると思われるので、ダッシュに対応する音素`-`に変換する。 """ for i in range(len(sep_phonemes)): + if len(sep_phonemes[i]) == 0: + # 空白文字等でリストが空の場合 + continue if sep_phonemes[i][0] == "ー": if i != 0: prev_phoneme = sep_phonemes[i - 1][-1] diff --git a/text/japanese_bert.py b/text/japanese_bert.py index 2974c76..9efe7a4 100644 --- a/text/japanese_bert.py +++ b/text/japanese_bert.py @@ -4,7 +4,7 @@ import torch from transformers import AutoModelForMaskedLM, AutoTokenizer from config import config -from text.japanese import text2sep_kata +from text.japanese import text2sep_kata, text_normalize LOCAL_PATH = "./bert/deberta-v2-large-japanese-char-wwm" @@ -19,8 +19,10 @@ def get_bert_feature( device=config.bert_gen_config.device, assist_text=None, assist_text_weight=0.7, + ignore_unknown=False, ): - text = "".join(text2sep_kata(text)[0]) + text = "".join(text2sep_kata(text, ignore_unknown=ignore_unknown)[0]) + # text = text_normalize(text) if assist_text: assist_text = "".join(text2sep_kata(assist_text)[0]) if ( From deb9190e091afcb16d2b5b439958368f345021c7 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Fri, 23 Feb 2024 16:18:33 +0900 Subject: [PATCH 30/51] Add config.json existing check --- common/tts_model.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/common/tts_model.py b/common/tts_model.py index ccde9ac..4f3932a 100644 --- a/common/tts_model.py +++ b/common/tts_model.py @@ -234,6 +234,12 @@ class ModelHolder: if len(model_files) == 0: logger.warning(f"No model files found in {model_dir}, so skip it") continue + config_path = model_dir / "config.json" + if not config_path.exists(): + logger.warning( + f"Config file {config_path} not found, so skip {model_dir}" + ) + continue self.model_files_dict[model_dir.name] = model_files self.model_names.append(model_dir.name) From 337ca71ec2b3b25583c826d55e272c4e045ce962 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Fri, 23 Feb 2024 16:19:15 +0900 Subject: [PATCH 31/51] Add downloading front build zip --- server_editor.py | 114 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 104 insertions(+), 10 deletions(-) diff --git a/server_editor.py b/server_editor.py index 1aebe6e..18eb71c 100644 --- a/server_editor.py +++ b/server_editor.py @@ -1,11 +1,25 @@ -import csv -import os +""" +Style-Bert-VITS2-Editor用のサーバー。 +次のリポジトリ +https://github.com/litagin02/Style-Bert-VITS2-Editor +をビルドしてできあがったファイルをWebフォルダに入れて実行する。 + +TODO: リファクタリングやドキュメンテーションやAPI整理、辞書周りの改善などが必要。 +""" + +import argparse +import io +import shutil import sys +import webbrowser +import zipfile +from datetime import datetime from io import BytesIO from pathlib import Path import numpy as np import pyopenjtalk +import requests import uvicorn from fastapi import FastAPI, HTTPException, status from fastapi.middleware.cors import CORSMiddleware @@ -16,21 +30,95 @@ from scipy.io import wavfile from common.constants import ( DEFAULT_ASSIST_TEXT_WEIGHT, - DEFAULT_LENGTH, DEFAULT_NOISE, DEFAULT_NOISEW, DEFAULT_SDP_RATIO, DEFAULT_STYLE, DEFAULT_STYLE_WEIGHT, + LATEST_VERSION, USER_DICT_CSV_PATH, USER_DICT_PATH, Languages, - LATEST_VERSION, ) from common.log import logger from common.tts_model import ModelHolder from text.japanese import g2kata_tone, kata_tone2phone_tone +STATIC_DIR = Path("Web") # ビルドした静的ファイルを配置するディレクトリ +LAST_DOWNLOAD_FILE = STATIC_DIR / "last_download.txt" + + +def download_static_files(user, repo, asset_name): + """Style-Bert-VITS2エディターの最新のビルドzipをダウンロードして展開する。""" + + logger.info("Checking for new release...") + latest_release = get_latest_release(user, repo) + if latest_release is None: + logger.warning( + "Failed to fetch the latest release. Proceeding without static files." + ) + return + + if not new_release_available(latest_release): + logger.info("No new release available. Proceeding with existing static files.") + return + + # asset_name = "out.zip" + asset_url = get_asset_url(latest_release, asset_name) + if asset_url: + if STATIC_DIR.exists(): + shutil.rmtree(STATIC_DIR) + STATIC_DIR.mkdir(parents=True, exist_ok=True) + download_and_extract(asset_url, STATIC_DIR) + save_last_download(latest_release) + else: + logger.warning("Asset not found. Proceeding without static files.") + + +def get_latest_release(user, repo): + url = f"https://api.github.com/repos/{user}/{repo}/releases/latest" + try: + response = requests.get(url) + response.raise_for_status() + return response.json() + except requests.RequestException: + return None + + +def get_asset_url(release, asset_name): + for asset in release["assets"]: + if asset["name"] == asset_name: + return asset["browser_download_url"] + return None + + +def download_and_extract(url, extract_to): + response = requests.get(url) + response.raise_for_status() + with zipfile.ZipFile(io.BytesIO(response.content)) as zip_ref: + zip_ref.extractall(extract_to) + + +def new_release_available(latest_release): + if LAST_DOWNLOAD_FILE.exists(): + with open(LAST_DOWNLOAD_FILE, "r") as file: + last_download_str = file.read().strip() + # 'Z'を除去して日時オブジェクトに変換 + last_download_str = last_download_str.replace("Z", "+00:00") + last_download = datetime.fromisoformat(last_download_str) + return ( + datetime.fromisoformat( + latest_release["published_at"].replace("Z", "+00:00") + ) + > last_download + ) + return True + + +def save_last_download(latest_release): + with open(LAST_DOWNLOAD_FILE, "w") as file: + file.write(latest_release["published_at"]) + class AudioResponse(Response): media_type = "audio/wav" @@ -43,8 +131,14 @@ origins = [ "http://127.0.0.1:8000", ] -device = "cuda" -model_dir = Path("model_assets/server_test") +parser = argparse.ArgumentParser() +parser.add_argument("--model_dir", type=str, default="model_assets/") +parser.add_argument("--device", type=str, default="cuda") + +args = parser.parse_args() +device = args.device +model_dir = Path(args.model_dir) + model_holder = ModelHolder(model_dir, device) if len(model_holder.model_names) == 0: logger.error(f"Models not found in {model_dir}.") @@ -158,7 +252,7 @@ class MultiSynthesisRequest(BaseModel): lines: list[SynthesisRequest] -@app.post("/multi-synthesis", response_class=AudioResponse) +@app.post("/multi_synthesis", response_class=AudioResponse) def multi_synthesis(request: MultiSynthesisRequest): lines = request.lines audios = [] @@ -203,7 +297,6 @@ def multi_synthesis(request: MultiSynthesisRequest): silence = int(sr * req.silenceAfter) audios.append(np.zeros(silence, dtype=np.int16)) audio = np.concatenate(audios) - logger.debug(audio.dtype) with BytesIO() as wavContent: wavfile.write(wavContent, sr, audio) @@ -279,7 +372,8 @@ def add_user_dict_word(request: AddUserDictWordRequest): ) -app.mount("/", StaticFiles(directory="Web"), name="static") - if __name__ == "__main__": + download_static_files("litagin02", "Style-Bert-VITS2-Editor", "out.zip") + app.mount("/", StaticFiles(directory=STATIC_DIR, html=True), name="Web") + webbrowser.open("http://localhost:8000") uvicorn.run(app, host="0.0.0.0", port=8000) From 0a6e4f1203b55dd5d059ec35b9da22b497d23348 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Fri, 23 Feb 2024 16:19:29 +0900 Subject: [PATCH 32/51] Fix to use Path obj --- server_fastapi.py | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/server_fastapi.py b/server_fastapi.py index 40877fd..ce6ed04 100644 --- a/server_fastapi.py +++ b/server_fastapi.py @@ -1,10 +1,13 @@ """ API server for TTS +TODO: server_editor.pyと統合する? """ + import argparse import os import sys from io import BytesIO +from pathlib import Path from typing import Dict, Optional, Union from urllib.parse import unquote @@ -53,10 +56,8 @@ def load_models(model_holder: ModelHolder): for model_name, model_paths in model_holder.model_files_dict.items(): model = Model( model_path=model_paths[0], - config_path=os.path.join(model_holder.root_dir, model_name, "config.json"), - style_vec_path=os.path.join( - model_holder.root_dir, model_name, "style_vectors.npy" - ), + config_path=model_holder.root_dir / model_name / "config.json", + style_vec_path=model_holder.root_dir / model_name / "style_vectors.npy", device=model_holder.device, ) model.load_net_g() @@ -76,7 +77,7 @@ if __name__ == "__main__": else: device = "cuda" if torch.cuda.is_available() else "cpu" - model_dir = args.dir + model_dir = Path(args.dir) model_holder = ModelHolder(model_dir, device) if len(model_holder.model_names) == 0: logger.error(f"Models not found in {model_dir}.") @@ -105,9 +106,12 @@ if __name__ == "__main__": request: Request, text: str = Query(..., min_length=1, max_length=limit, description=f"セリフ"), encoding: str = Query(None, description="textをURLデコードする(ex, `utf-8`)"), - model_id: int = Query(0, description="モデルID。`GET /models/info`のkeyの値を指定ください"), + model_id: int = Query( + 0, description="モデルID。`GET /models/info`のkeyの値を指定ください" + ), speaker_name: str = Query( - None, description="話者名(speaker_idより優先)。esd.listの2列目の文字列を指定" + None, + description="話者名(speaker_idより優先)。esd.listの2列目の文字列を指定", ), speaker_id: int = Query( 0, description="話者ID。model_assets>[model]>config.json内のspk2idを確認" @@ -116,12 +120,17 @@ if __name__ == "__main__": DEFAULT_SDP_RATIO, description="SDP(Stochastic Duration Predictor)/DP混合比。比率が高くなるほどトーンのばらつきが大きくなる", ), - noise: float = Query(DEFAULT_NOISE, description="サンプルノイズの割合。大きくするほどランダム性が高まる"), + noise: float = Query( + DEFAULT_NOISE, + description="サンプルノイズの割合。大きくするほどランダム性が高まる", + ), noisew: float = Query( - DEFAULT_NOISEW, description="SDPノイズ。大きくするほど発音の間隔にばらつきが出やすくなる" + DEFAULT_NOISEW, + description="SDPノイズ。大きくするほど発音の間隔にばらつきが出やすくなる", ), length: float = Query( - DEFAULT_LENGTH, description="話速。基準は1で大きくするほど音声は長くなり読み上げが遅まる" + DEFAULT_LENGTH, + description="話速。基準は1で大きくするほど音声は長くなり読み上げが遅まる", ), language: Languages = Query(ln, description=f"textの言語"), auto_split: bool = Query(DEFAULT_LINE_SPLIT, description="改行で分けて生成"), @@ -129,20 +138,25 @@ if __name__ == "__main__": DEFAULT_SPLIT_INTERVAL, description="分けた場合に挟む無音の長さ（秒）" ), assist_text: Optional[str] = Query( - None, description="このテキストの読み上げと似た声音・感情になりやすくなる。ただし抑揚やテンポ等が犠牲になる傾向がある" + None, + description="このテキストの読み上げと似た声音・感情になりやすくなる。ただし抑揚やテンポ等が犠牲になる傾向がある", ), assist_text_weight: float = Query( DEFAULT_ASSIST_TEXT_WEIGHT, description="assist_textの強さ" ), style: Optional[Union[int, str]] = Query(DEFAULT_STYLE, description="スタイル"), style_weight: float = Query(DEFAULT_STYLE_WEIGHT, description="スタイルの強さ"), - reference_audio_path: Optional[str] = Query(None, description="スタイルを音声ファイルで行う"), + reference_audio_path: Optional[str] = Query( + None, description="スタイルを音声ファイルで行う" + ), ): """Infer text to speech(テキストから感情付き音声を生成する)""" logger.info( f"{request.client.host}:{request.client.port}/voice { unquote(str(request.query_params) )}" ) - if model_id >= len(model_holder.models): # /models/refresh があるためQuery(le)で表現不可 + if model_id >= len( + model_holder.models + ): # /models/refresh があるためQuery(le)で表現不可 raise_validation_error(f"model_id={model_id} not found", "model_id") model = model_holder.models[model_id] From cb842faaf72d8e1b71b59a15bbde6e66cf30ec01 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Fri, 23 Feb 2024 16:20:08 +0900 Subject: [PATCH 33/51] Improve to import only necesary lang --- text/__init__.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/text/__init__.py b/text/__init__.py index 0554884..3c7d6cd 100644 --- a/text/__init__.py +++ b/text/__init__.py @@ -27,17 +27,24 @@ def get_bert( assist_text_weight=0.7, ignore_unknown=False, ): - from .chinese_bert import get_bert_feature as zh_bert - from .english_bert_mock import get_bert_feature as en_bert - from .japanese_bert import get_bert_feature as jp_bert + # from .chinese_bert import get_bert_feature as zh_bert + # from .english_bert_mock import get_bert_feature as en_bert + # from .japanese_bert import get_bert_feature as jp_bert - lang_bert_func_map = {"ZH": zh_bert, "EN": en_bert, "JP": jp_bert} - if language == "JP": - bert = lang_bert_func_map[language]( + # Change logic to import inside if condition to avoid unnecessary import + if language == "ZH": + from .chinese_bert import get_bert_feature as zh_bert + + return zh_bert(text, word2ph, device, assist_text, assist_text_weight) + elif language == "EN": + from .english_bert_mock import get_bert_feature as en_bert + + return en_bert(text, word2ph, device, assist_text, assist_text_weight) + elif language == "JP": + from .japanese_bert import get_bert_feature as jp_bert + + return jp_bert( text, word2ph, device, assist_text, assist_text_weight, ignore_unknown ) else: - bert = lang_bert_func_map[language]( - text, word2ph, device, assist_text, assist_text_weight - ) - return bert + raise ValueError(f"Language {language} not supported") From 049b0fc1d570e442166ff7bfbd6fd37dd9ffb00a Mon Sep 17 00:00:00 2001 From: litagin02 Date: Fri, 23 Feb 2024 16:49:11 +0900 Subject: [PATCH 34/51] Change to import inside if block --- text/__init__.py | 5 ----- text/cleaner.py | 39 ++++++++++++++++----------------------- 2 files changed, 16 insertions(+), 28 deletions(-) diff --git a/text/__init__.py b/text/__init__.py index 3c7d6cd..2151caf 100644 --- a/text/__init__.py +++ b/text/__init__.py @@ -27,11 +27,6 @@ def get_bert( assist_text_weight=0.7, ignore_unknown=False, ): - # from .chinese_bert import get_bert_feature as zh_bert - # from .english_bert_mock import get_bert_feature as en_bert - # from .japanese_bert import get_bert_feature as jp_bert - - # Change logic to import inside if condition to avoid unnecessary import if language == "ZH": from .chinese_bert import get_bert_feature as zh_bert diff --git a/text/cleaner.py b/text/cleaner.py index 759c6cc..8da4c7b 100644 --- a/text/cleaner.py +++ b/text/cleaner.py @@ -1,33 +1,26 @@ -from text import chinese, japanese, english, cleaned_text_to_sequence - - -language_module_map = {"ZH": chinese, "JP": japanese, "EN": english} - - def clean_text(text, language, use_jp_extra=True, ignore_unknown=False): - language_module = language_module_map[language] - norm_text = language_module.text_normalize(text) - if language == "JP": + # Changed to import inside if condition to avoid unnecessary import + if language == "ZH": + from . import chinese as language_module + + norm_text = language_module.text_normalize(text) + phones, tones, word2ph = language_module.g2p(norm_text) + elif language == "EN": + from . import english as language_module + + norm_text = language_module.text_normalize(text) + phones, tones, word2ph = language_module.g2p(norm_text) + elif language == "JP": + from . import japanese as language_module + + norm_text = language_module.text_normalize(text) phones, tones, word2ph = language_module.g2p( norm_text, use_jp_extra, ignore_unknown=ignore_unknown ) else: - phones, tones, word2ph = language_module.g2p(norm_text) + raise ValueError(f"Language {language} not supported") return norm_text, phones, tones, word2ph -def clean_text_bert(text, language): - language_module = language_module_map[language] - norm_text = language_module.text_normalize(text) - phones, tones, word2ph = language_module.g2p(norm_text) - bert = language_module.get_bert_feature(norm_text, word2ph) - return phones, tones, bert - - -def text_to_sequence(text, language): - norm_text, phones, tones, word2ph = clean_text(text, language) - return cleaned_text_to_sequence(phones, tones, language) - - if __name__ == "__main__": pass From 039d23d56078dcbb814a176b0a833bf7da142caf Mon Sep 17 00:00:00 2001 From: litagin02 Date: Fri, 23 Feb 2024 19:57:23 +0900 Subject: [PATCH 35/51] Update Dockerfile --- .dockerignore | 42 ++++++++++++++++++++++++++++++++++ .gitignore | 2 +- Dockerfile.deploy | 22 ++++++++++++++++++ Dockerfile => Dockerfile.train | 12 ++++++++++ 4 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 .dockerignore create mode 100644 Dockerfile.deploy rename Dockerfile => Dockerfile.train (89%) diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..95ef322 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,42 @@ +# Dockerfile.deploy用 + +*.pyc +*.pyo +*.pyd +__pycache__ +*.pyc + +venv/ +.vscode/ + +.ipynb_checkpoints/ +*.ipynb + +.git/ +.gitignore + +Dockerfile* +.dockerignore +*.md +*.bat +LICENSE + +*.wav +*.zip +*.csv + +# 中国語と英語が必要な場合はコメントアウト +/bert/chinese-roberta-wwm-ext-large/ +/bert/deberta-v3-large/ + +Data/ +dict/ +docs/ +inputs/ +mos_results/ +pretrained/ +pretrained_jp_extra/ +scripts/ +slm/ +static/ +tools/ diff --git a/.gitignore b/.gitignore index 5ebe1d8..b556dff 100644 --- a/.gitignore +++ b/.gitignore @@ -28,7 +28,7 @@ venv/ safetensors.ipynb *.wav -/Web/ +/static/ # pyopenjtalk's dictionary *.dic diff --git a/Dockerfile.deploy b/Dockerfile.deploy new file mode 100644 index 0000000..f6cab39 --- /dev/null +++ b/Dockerfile.deploy @@ -0,0 +1,22 @@ +# Hugging face spaces (CPU) でエディタ (server_editor.py) のデプロイ用 + +# See https://huggingface.co/docs/hub/spaces-sdks-docker-first-demo + +FROM python:3.10 + +RUN useradd -m -u 1000 user + +USER user + +ENV HOME=/home/user \ + PATH=/home/user/.local/bin:$PATH + +WORKDIR $HOME/app + +RUN pip install --no-cache-dir --upgrade pip + +COPY --chown=user . $HOME/app + +RUN pip install --no-cache-dir -r $HOME/app/requirements.txt + +CMD ["python", "server_editor.py"] diff --git a/Dockerfile b/Dockerfile.train similarity index 89% rename from Dockerfile rename to Dockerfile.train index 790c9c2..333fd46 100644 --- a/Dockerfile +++ b/Dockerfile.train @@ -1,3 +1,15 @@ +# PaperspaceのGradient環境での学習環境構築用Dockerfileです。 +# 環境のみ構築するため、イメージには学習用のコードは含まれていません。 +# 以下を参照しました。 +# https://github.com/gradient-ai/base-container/tree/main/pt211-tf215-cudatk120-py311 + +# 主なバージョン等 +# Ubuntu 22.04 +# Python 3.10 +# PyTorch 2.1.2 (CUDA 11.8) +# CUDA Toolkit 12.0, CUDNN 8.9.7 + + # ================================================================== # Initial setup # ------------------------------------------------------------------ From 20b3326b79a815fdfe50fd150bb459126e07697e Mon Sep 17 00:00:00 2001 From: litagin02 Date: Fri, 23 Feb 2024 20:34:16 +0900 Subject: [PATCH 36/51] Update editor server --- server_editor.py | 51 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/server_editor.py b/server_editor.py index 18eb71c..7e1991c 100644 --- a/server_editor.py +++ b/server_editor.py @@ -20,8 +20,9 @@ from pathlib import Path import numpy as np import pyopenjtalk import requests +import torch import uvicorn -from fastapi import FastAPI, HTTPException, status +from fastapi import APIRouter, FastAPI, HTTPException, status from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse, Response from fastapi.staticfiles import StaticFiles @@ -44,7 +45,10 @@ from common.log import logger from common.tts_model import ModelHolder from text.japanese import g2kata_tone, kata_tone2phone_tone -STATIC_DIR = Path("Web") # ビルドした静的ファイルを配置するディレクトリ + +# エディターのビルドファイルを配置するディレクトリ +STATIC_DIR = Path("static") +# エディターの最新のビルドファイルのダウンロード日時を記録するファイル LAST_DOWNLOAD_FILE = STATIC_DIR / "last_download.txt" @@ -63,7 +67,7 @@ def download_static_files(user, repo, asset_name): logger.info("No new release available. Proceeding with existing static files.") return - # asset_name = "out.zip" + logger.info("New release available. Downloading static files...") asset_url = get_asset_url(latest_release, asset_name) if asset_url: if STATIC_DIR.exists(): @@ -92,12 +96,23 @@ def get_asset_url(release, asset_name): return None -def download_and_extract(url, extract_to): +def download_and_extract(url, extract_to: Path): response = requests.get(url) response.raise_for_status() with zipfile.ZipFile(io.BytesIO(response.content)) as zip_ref: zip_ref.extractall(extract_to) + # 展開先が1つのディレクトリだけの場合、その中身を直下に移動する + extracted_dirs = list(extract_to.iterdir()) + if len(extracted_dirs) == 1 and extracted_dirs[0].is_dir(): + for file in extracted_dirs[0].iterdir(): + file.rename(extract_to / file.name) + extracted_dirs[0].rmdir() + + # index.htmlが存在するかチェック + if not (extract_to / "index.html").exists(): + logger.warning("index.html not found in the extracted files.") + def new_release_available(latest_release): if LAST_DOWNLOAD_FILE.exists(): @@ -134,10 +149,15 @@ origins = [ parser = argparse.ArgumentParser() parser.add_argument("--model_dir", type=str, default="model_assets/") parser.add_argument("--device", type=str, default="cuda") +parser.add_argument("--port", type=int, default=8000) +parser.add_argument("--inbrowser", action="store_true") args = parser.parse_args() device = args.device +if device == "cuda" and not torch.cuda.is_available(): + device = "cpu" model_dir = Path(args.model_dir) +port = int(args.port) model_holder = ModelHolder(model_dir, device) if len(model_holder.model_names) == 0: @@ -155,8 +175,10 @@ app.add_middleware( allow_headers=["*"], ) +router = APIRouter() -@app.get("/version") + +@router.get("/version") def version() -> str: return LATEST_VERSION @@ -170,7 +192,7 @@ class G2PRequest(BaseModel): text: str -@app.post("/g2p") +@router.post("/g2p") async def read_item(item: G2PRequest): try: kata_tone_list = g2kata_tone(item.text, ignore_unknown=True) @@ -182,7 +204,7 @@ async def read_item(item: G2PRequest): return [MoraTone(mora=kata, tone=tone) for kata, tone in kata_tone_list] -@app.get("/models_info") +@router.get("/models_info") def models_info(): return model_holder.models_info() @@ -206,7 +228,7 @@ class SynthesisRequest(BaseModel): intonationScale: float = 1.0 -@app.post("/synthesis", response_class=AudioResponse) +@router.post("/synthesis", response_class=AudioResponse) def synthesis(request: SynthesisRequest): try: model = model_holder.load_model( @@ -252,7 +274,7 @@ class MultiSynthesisRequest(BaseModel): lines: list[SynthesisRequest] -@app.post("/multi_synthesis", response_class=AudioResponse) +@router.post("/multi_synthesis", response_class=AudioResponse) def multi_synthesis(request: MultiSynthesisRequest): lines = request.lines audios = [] @@ -315,7 +337,7 @@ class AddUserDictWordRequest(BaseModel): COST_CANDIDATES = [-988, 3488, 4768, 6048, 7328, 8609, 8734, 8859, 8984, 9110, 14176] -@app.post("/user_dict_word") +@router.post("/user_dict_word") def add_user_dict_word(request: AddUserDictWordRequest): if request.surface == "" or request.pronunciation == "": return JSONResponse( @@ -372,8 +394,11 @@ def add_user_dict_word(request: AddUserDictWordRequest): ) +app.include_router(router, prefix="/api") + if __name__ == "__main__": download_static_files("litagin02", "Style-Bert-VITS2-Editor", "out.zip") - app.mount("/", StaticFiles(directory=STATIC_DIR, html=True), name="Web") - webbrowser.open("http://localhost:8000") - uvicorn.run(app, host="0.0.0.0", port=8000) + app.mount("/", StaticFiles(directory=STATIC_DIR, html=True), name="static") + if args.inbrowser: + webbrowser.open(f"http://localhost:{port}") + uvicorn.run(app, host="0.0.0.0", port=port) From ffdbb8e0d18cc0c6ab84b4d36592e7abe3eb082b Mon Sep 17 00:00:00 2001 From: litagin02 Date: Fri, 23 Feb 2024 20:57:54 +0900 Subject: [PATCH 37/51] Feat: freezing decoder option --- colab.ipynb | 1 + configs/config.json | 3 +- configs/configs_jp_extra.json | 3 +- docs/CLI.md | 3 +- preprocess_all.py | 6 +++ scripts/Update-to-Dict-Editor.bat | 66 +++++++++++++++++++++++++++++++ train_ms.py | 5 +++ train_ms_jp_extra.py | 5 +++ webui_train.py | 57 ++++++++++++++++++-------- 9 files changed, 130 insertions(+), 19 deletions(-) create mode 100644 scripts/Update-to-Dict-Editor.bat diff --git a/colab.ipynb b/colab.ipynb index 37b708f..f4e932b 100644 --- a/colab.ipynb +++ b/colab.ipynb @@ -265,6 +265,7 @@ " freeze_JP_bert=False,\n", " freeze_ZH_bert=False,\n", " freeze_style=False,\n", + " freeze_decoder=False, # ここをTrueにするともしかしたら違う結果になるかもしれません。\n", " use_jp_extra=use_jp_extra,\n", " val_per_lang=0,\n", " log_interval=200,\n", diff --git a/configs/config.json b/configs/config.json index 5196633..25e86db 100644 --- a/configs/config.json +++ b/configs/config.json @@ -20,7 +20,8 @@ "freeze_ZH_bert": false, "freeze_JP_bert": false, "freeze_EN_bert": false, - "freeze_style": false + "freeze_style": false, + "freeze_encoder": false }, "data": { "training_files": "Data/your_model_name/filelists/train.list", diff --git a/configs/configs_jp_extra.json b/configs/configs_jp_extra.json index 490f4d6..616d1d3 100644 --- a/configs/configs_jp_extra.json +++ b/configs/configs_jp_extra.json @@ -22,7 +22,8 @@ "freeze_JP_bert": false, "freeze_EN_bert": false, "freeze_emo": false, - "freeze_style": false + "freeze_style": false, + "freeze_decoder": false }, "data": { "use_jp_extra": true, diff --git a/docs/CLI.md b/docs/CLI.md index 88ad5fd..08e2fd0 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -55,7 +55,7 @@ Optional ## 2. Preprocess ```bash -python preprocess_all.py -m [--use_jp_extra] [-b ] [-e ] [-s ] [--num_processes ] [--normalize] [--trim] [--val_per_lang ] [--log_interval ] [--freeze_EN_bert] [--freeze_JP_bert] [--freeze_ZH_bert] [--freeze_style] +python preprocess_all.py -m [--use_jp_extra] [-b ] [-e ] [-s ] [--num_processes ] [--normalize] [--trim] [--val_per_lang ] [--log_interval ] [--freeze_EN_bert] [--freeze_JP_bert] [--freeze_ZH_bert] [--freeze_style] [--freeze_decoder] ``` Required: @@ -72,6 +72,7 @@ Optional: - `--freeze_JP_bert`: Freeze Japanese BERT. - `--freeze_ZH_bert`: Freeze Chinese BERT. - `--freeze_style`: Freeze style vector. +- `--freeze_decoder`: Freeze decoder. - `--use_jp_extra`: Use JP-Extra model. - `--val_per_lang`: Validation data per language (default: 0). - `--log_interval`: Log interval (default: 200). diff --git a/preprocess_all.py b/preprocess_all.py index 679abb2..82a3c20 100644 --- a/preprocess_all.py +++ b/preprocess_all.py @@ -52,6 +52,11 @@ if __name__ == "__main__": action="store_true", help="Freeze style vector", ) + parser.add_argument( + "--freeze_decoder", + action="store_true", + help="Freeze decoder", + ) parser.add_argument( "--use_jp_extra", action="store_true", @@ -84,6 +89,7 @@ if __name__ == "__main__": freeze_JP_bert=args.freeze_JP_bert, freeze_ZH_bert=args.freeze_ZH_bert, freeze_style=args.freeze_style, + freeze_decoder=args.freeze_decoder, use_jp_extra=args.use_jp_extra, val_per_lang=args.val_per_lang, log_interval=args.log_interval, diff --git a/scripts/Update-to-Dict-Editor.bat b/scripts/Update-to-Dict-Editor.bat new file mode 100644 index 0000000..254b060 --- /dev/null +++ b/scripts/Update-to-Dict-Editor.bat @@ -0,0 +1,66 @@ +chcp 65001 > NUL +@echo off + +pushd %~dp0 +set PS_CMD=PowerShell -Version 5.1 -ExecutionPolicy Bypass + +set CURL_CMD=C:\Windows\System32\curl.exe +if not exist %CURL_CMD% ( + echo [ERROR] %CURL_CMD% が見つかりません。 + pause & popd & exit /b 1 +) + +@REM Style-Bert-VITS2.zip をGitHubのmasterの最新のものをダウンロード +%CURL_CMD% -Lo Style-Bert-VITS2.zip^ + https://github.com/litagin02/Style-Bert-VITS2/archive/refs/heads/master.zip +if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) + +@REM Style-Bert-VITS2.zip を解凍（フォルダ名前がBert-VITS2-masterになる） +%PS_CMD% Expand-Archive -Path Style-Bert-VITS2.zip -DestinationPath . -Force +if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) + +@REM 元のzipを削除 +del Style-Bert-VITS2.zip +if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) + +@REM Bert-VITS2-masterの中身をStyle-Bert-VITS2に上書き移動 +xcopy /QSY .\Style-Bert-VITS2-master\ .\Style-Bert-VITS2\ +rmdir /s /q Style-Bert-VITS2-master +if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) + +@REM 仮想環境のpip requirements.txtを更新 + +echo call .\Style-Bert-VITS2\scripts\activate.bat +call .\Style-Bert-VITS2\venv\Scripts\activate.bat +if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) + +@REM pyopenjtalk-prebuiltやpyopenjtalkが入っていたら削除 +echo pip uninstall -y pyopenjtalk-prebuilt pyopenjtalk +pip uninstall -y pyopenjtalk-prebuilt pyopenjtalk +if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) + +@REM pyopenjtalk-dictをインストール +echo pip install -U pyopenjtalk-dict +pip install -U pyopenjtalk-dict + +@REM その他のrequirements.txtも一応更新 +pip install -U -r Style-Bert-VITS2\requirements.txt +if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) + +pushd Style-Bert-VITS2 + +echo Update completed. Running Style-Bert-VITS2 Editor... + +@REM Style-Bert-VITS2 Editorを起動 +python server_editor.py + +pause + +popd + + +pause + +popd + +popd \ No newline at end of file diff --git a/train_ms.py b/train_ms.py index fb7fa9c..ea4b65f 100644 --- a/train_ms.py +++ b/train_ms.py @@ -306,6 +306,11 @@ def run(): for param in net_g.enc_p.style_proj.parameters(): param.requires_grad = False + if getattr(hps.train, "freeze_decoder", False): + logger.info("Freezing decoder !!!") + for param in net_g.dec.parameters(): + param.requires_grad = False + net_d = MultiPeriodDiscriminator(hps.model.use_spectral_norm).cuda(local_rank) optim_g = torch.optim.AdamW( filter(lambda p: p.requires_grad, net_g.parameters()), diff --git a/train_ms_jp_extra.py b/train_ms_jp_extra.py index 0cc9d2a..bae922d 100644 --- a/train_ms_jp_extra.py +++ b/train_ms_jp_extra.py @@ -310,6 +310,11 @@ def run(): for param in net_g.enc_p.style_proj.parameters(): param.requires_grad = False + if getattr(hps.train, "freeze_decoder", False): + logger.info("Freezing decoder !!!") + for param in net_g.dec.parameters(): + param.requires_grad = False + net_d = MultiPeriodDiscriminator(hps.model.use_spectral_norm).cuda(local_rank) optim_g = torch.optim.AdamW( filter(lambda p: p.requires_grad, net_g.parameters()), diff --git a/webui_train.py b/webui_train.py index c3a876e..eccfeaf 100644 --- a/webui_train.py +++ b/webui_train.py @@ -47,6 +47,7 @@ def initialize( freeze_JP_bert, freeze_ZH_bert, freeze_style, + freeze_decoder, use_jp_extra, log_interval, ): @@ -61,7 +62,7 @@ def initialize( logger_handler = logger.add(os.path.join(dataset_path, file_name)) logger.info( - f"Step 1: start initialization...\nmodel_name: {model_name}, batch_size: {batch_size}, epochs: {epochs}, save_every_steps: {save_every_steps}, freeze_ZH_bert: {freeze_ZH_bert}, freeze_JP_bert: {freeze_JP_bert}, freeze_EN_bert: {freeze_EN_bert}, freeze_style: {freeze_style}, use_jp_extra: {use_jp_extra}" + f"Step 1: start initialization...\nmodel_name: {model_name}, batch_size: {batch_size}, epochs: {epochs}, save_every_steps: {save_every_steps}, freeze_ZH_bert: {freeze_ZH_bert}, freeze_JP_bert: {freeze_JP_bert}, freeze_EN_bert: {freeze_EN_bert}, freeze_style: {freeze_style}, freeze_decoder: {freeze_decoder}, use_jp_extra: {use_jp_extra}" ) default_config_path = ( @@ -82,12 +83,15 @@ def initialize( config["train"]["freeze_JP_bert"] = freeze_JP_bert config["train"]["freeze_ZH_bert"] = freeze_ZH_bert config["train"]["freeze_style"] = freeze_style + config["train"]["freeze_decoder"] = freeze_decoder config["train"]["bf16_run"] = False # デフォルトでFalseのはずだが念のため model_path = os.path.join(dataset_path, "models") if os.path.exists(model_path): - logger.warning(f"Step 1: {model_path} already exists, so copy it to backup to {model_path}_backup") + logger.warning( + f"Step 1: {model_path} already exists, so copy it to backup to {model_path}_backup" + ) shutil.copytree( src=model_path, dst=os.path.join(dataset_path, "models_backup"), @@ -262,6 +266,7 @@ def preprocess_all( freeze_JP_bert, freeze_ZH_bert, freeze_style, + freeze_decoder, use_jp_extra, val_per_lang, log_interval, @@ -269,29 +274,39 @@ def preprocess_all( if model_name == "": return False, "Error: モデル名を入力してください" success, message = initialize( - model_name, - batch_size, - epochs, - save_every_steps, - freeze_EN_bert, - freeze_JP_bert, - freeze_ZH_bert, - freeze_style, - use_jp_extra, - log_interval, + model_name=model_name, + batch_size=batch_size, + epochs=epochs, + save_every_steps=save_every_steps, + freeze_EN_bert=freeze_EN_bert, + freeze_JP_bert=freeze_JP_bert, + freeze_ZH_bert=freeze_ZH_bert, + freeze_style=freeze_style, + freeze_decoder=freeze_decoder, + use_jp_extra=use_jp_extra, + log_interval=log_interval, ) if not success: return False, message - success, message = resample(model_name, normalize, trim, num_processes) + success, message = resample( + model_name=model_name, + normalize=normalize, + trim=trim, + num_processes=num_processes, + ) if not success: return False, message - success, message = preprocess_text(model_name, use_jp_extra, val_per_lang) + success, message = preprocess_text( + model_name=model_name, use_jp_extra=use_jp_extra, val_per_lang=val_per_lang + ) if not success: return False, message - success, message = bert_gen(model_name) # bert_genは重いのでプロセス数いじらない + success, message = bert_gen( + model_name=model_name + ) # bert_genは重いのでプロセス数いじらない if not success: return False, message - success, message = style_gen(model_name, num_processes) + success, message = style_gen(model_name=model_name, num_processes=num_processes) if not success: return False, message logger.success("Success: All preprocess finished!") @@ -507,6 +522,10 @@ if __name__ == "__main__": label="スタイル部分を凍結", value=False, ) + freeze_decoder = gr.Checkbox( + label="デコーダ部分を凍結", + value=False, + ) with gr.Column(): preprocess_button = gr.Button( @@ -565,6 +584,10 @@ if __name__ == "__main__": label="スタイル部分を凍結", value=False, ) + freeze_decoder_manual = gr.Checkbox( + label="デコーダ部分を凍結", + value=False, + ) with gr.Column(): generate_config_btn = gr.Button(value="実行", variant="primary") info_init = gr.Textbox(label="状況") @@ -658,6 +681,7 @@ if __name__ == "__main__": freeze_JP_bert, freeze_ZH_bert, freeze_style, + freeze_decoder, use_jp_extra, val_per_lang, log_interval, @@ -677,6 +701,7 @@ if __name__ == "__main__": freeze_JP_bert_manual, freeze_ZH_bert_manual, freeze_style_manual, + freeze_decoder_manual, use_jp_extra_manual, log_interval_manual, ], From 1dac495be14f667c948a6a52a2aca51b79cb276d Mon Sep 17 00:00:00 2001 From: litagin02 Date: Fri, 23 Feb 2024 21:39:21 +0900 Subject: [PATCH 38/51] Add line length count limit option --- server_editor.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/server_editor.py b/server_editor.py index 7e1991c..f9420cf 100644 --- a/server_editor.py +++ b/server_editor.py @@ -151,6 +151,8 @@ parser.add_argument("--model_dir", type=str, default="model_assets/") parser.add_argument("--device", type=str, default="cuda") parser.add_argument("--port", type=int, default=8000) parser.add_argument("--inbrowser", action="store_true") +parser.add_argument("--line_length", type=int, default=None) +parser.add_argument("--line_count", type=int, default=None) args = parser.parse_args() device = args.device @@ -230,6 +232,11 @@ class SynthesisRequest(BaseModel): @router.post("/synthesis", response_class=AudioResponse) def synthesis(request: SynthesisRequest): + if args.line_length is not None and len(request.text) > args.line_length: + raise HTTPException( + status_code=400, + detail=f"1行の文字数は{args.line_length}文字以下にしてください。", + ) try: model = model_holder.load_model( model_name=request.model, model_path_str=request.modelFile @@ -277,6 +284,11 @@ class MultiSynthesisRequest(BaseModel): @router.post("/multi_synthesis", response_class=AudioResponse) def multi_synthesis(request: MultiSynthesisRequest): lines = request.lines + if args.line_count is not None and len(lines) > args.line_count: + raise HTTPException( + status_code=400, + detail=f"行数は{args.line_count}行以下にしてください。", + ) audios = [] for i, req in enumerate(lines): # Loade model From 760f6c268021b88ae6727123c0d669a1b155e998 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Sat, 24 Feb 2024 12:01:06 +0900 Subject: [PATCH 39/51] Enable empty initial prompt --- webui_dataset.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/webui_dataset.py b/webui_dataset.py index 59c7270..fec7a9a 100644 --- a/webui_dataset.py +++ b/webui_dataset.py @@ -50,8 +50,6 @@ def do_transcribe( ): if model_name == "": return "Error: モデル名を入力してください。" - if initial_prompt == "": - initial_prompt = "こんにちは。元気、ですかー？私は……ふふっ、ちゃんと元気だよ！" success, message = run_script_with_log( [ From 357a3330c899617fca7a3d149960ee0ae25a5892 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Sun, 25 Feb 2024 13:14:28 +0900 Subject: [PATCH 40/51] Add User dict module from VOICEVOX --- dict_data/.gitignore | 3 + dict_data/default.csv | 3 + text/user_dict/LGPL_LICENSE | 165 +++++++++ text/user_dict/README.md | 16 + text/user_dict/__init__.py | 462 ++++++++++++++++++++++++++ text/user_dict/part_of_speech_data.py | 144 ++++++++ text/user_dict/word_model.py | 124 +++++++ 7 files changed, 917 insertions(+) create mode 100644 dict_data/.gitignore create mode 100644 dict_data/default.csv create mode 100644 text/user_dict/LGPL_LICENSE create mode 100644 text/user_dict/README.md create mode 100644 text/user_dict/__init__.py create mode 100644 text/user_dict/part_of_speech_data.py create mode 100644 text/user_dict/word_model.py diff --git a/dict_data/.gitignore b/dict_data/.gitignore new file mode 100644 index 0000000..0b60cb6 --- /dev/null +++ b/dict_data/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore +!default.csv diff --git a/dict_data/default.csv b/dict_data/default.csv new file mode 100644 index 0000000..8869f97 --- /dev/null +++ b/dict_data/default.csv @@ -0,0 +1,3 @@ +Ｂｅｒｔ,,,8609,名詞,固有名詞,一般,*,*,*,Ｂｅｒｔ,バアト,バアト,0/3,* +ＶＩＴＳ,,,8609,名詞,固有名詞,一般,*,*,*,ＶＩＴＳ,ビッツ,ビッツ,0/3,* +ＶＩＴＳ２,,,8609,名詞,固有名詞,一般,*,*,*,ＶＩＴＳ２,ビッツツウ,ビッツツウ,4/5,* diff --git a/text/user_dict/LGPL_LICENSE b/text/user_dict/LGPL_LICENSE new file mode 100644 index 0000000..153d416 --- /dev/null +++ b/text/user_dict/LGPL_LICENSE @@ -0,0 +1,165 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. \ No newline at end of file diff --git a/text/user_dict/README.md b/text/user_dict/README.md new file mode 100644 index 0000000..c7190ea --- /dev/null +++ b/text/user_dict/README.md @@ -0,0 +1,16 @@ +このモジュールは、[voicevox_engine](https://github.com/VOICEVOX/voicevox_engine)を使わせていただいています。 + +引用元: + +https://github.com/VOICEVOX/voicevox_engine/tree/709527be089c0410c08e989df95a4a1d78439423/voicevox_engine/user_dict + +改変部分は以下のとおりです。 +- ファイル名の書き換え、それに伴うimport文の書き換え +- VOICEVOX固有の他のモジュールへの依存のコメントアウト +- mutexを使用している部分のコメントアウト +- 参照しているpyopenjtalkの違いによるメソッド名書き換え +- UserDictWordのmora_countのデフォルト値をNoneに指定 + +ライセンス: LGPL + +[LGPL_LICENSE](LGPL_LICENSE)を参照してください。 diff --git a/text/user_dict/__init__.py b/text/user_dict/__init__.py new file mode 100644 index 0000000..d258359 --- /dev/null +++ b/text/user_dict/__init__.py @@ -0,0 +1,462 @@ +import json +import sys +import threading +import traceback +from pathlib import Path +from typing import Dict, List, Optional +from uuid import UUID, uuid4 + +import numpy as np +import pyopenjtalk +from fastapi import HTTPException + +from .word_model import UserDictWord, WordTypes + +# from ..utility.mutex_utility import mutex_wrapper +# from ..utility.path_utility import engine_root, get_save_dir +from .part_of_speech_data import MAX_PRIORITY, MIN_PRIORITY, part_of_speech_data +from common.constants import USER_DICT_DIR + +# root_dir = engine_root() +# save_dir = get_save_dir() +root_dir = Path(USER_DICT_DIR) +save_dir = Path(USER_DICT_DIR) + + +if not save_dir.is_dir(): + save_dir.mkdir(parents=True) + +default_dict_path = root_dir / "default.csv" # VOICEVOXデフォルト辞書ファイルのパス +user_dict_path = save_dir / "user_dict.json" # ユーザー辞書ファイルのパス +compiled_dict_path = save_dir / "user.dic" # コンパイル済み辞書ファイルのパス + + +# # 同時書き込みの制御 +# mutex_user_dict = threading.Lock() +# mutex_openjtalk_dict = threading.Lock() + + +# @mutex_wrapper(mutex_user_dict) +def _write_to_json(user_dict: Dict[str, UserDictWord], user_dict_path: Path) -> None: + """ + ユーザー辞書ファイルへのユーザー辞書データ書き込み + Parameters + ---------- + user_dict : Dict[str, UserDictWord] + ユーザー辞書データ + user_dict_path : Path + ユーザー辞書ファイルのパス + """ + converted_user_dict = {} + for word_uuid, word in user_dict.items(): + word_dict = word.dict() + word_dict["cost"] = _priority2cost( + word_dict["context_id"], word_dict["priority"] + ) + del word_dict["priority"] + converted_user_dict[word_uuid] = word_dict + # 予めjsonに変換できることを確かめる + user_dict_json = json.dumps(converted_user_dict, ensure_ascii=False) + + # ユーザー辞書ファイルへの書き込み + user_dict_path.write_text(user_dict_json, encoding="utf-8") + + +# @mutex_wrapper(mutex_openjtalk_dict) +def update_dict( + default_dict_path: Path = default_dict_path, + user_dict_path: Path = user_dict_path, + compiled_dict_path: Path = compiled_dict_path, +) -> None: + """ + 辞書の更新 + Parameters + ---------- + default_dict_path : Path + デフォルト辞書ファイルのパス + user_dict_path : Path + ユーザー辞書ファイルのパス + compiled_dict_path : Path + コンパイル済み辞書ファイルのパス + """ + random_string = uuid4() + tmp_csv_path = compiled_dict_path.with_suffix( + f".dict_csv-{random_string}.tmp" + ) # csv形式辞書データの一時保存ファイル + tmp_compiled_path = compiled_dict_path.with_suffix( + f".dict_compiled-{random_string}.tmp" + ) # コンパイル済み辞書データの一時保存ファイル + + try: + # 辞書.csvを作成 + csv_text = "" + + # デフォルト辞書データの追加 + if not default_dict_path.is_file(): + print("Warning: Cannot find default dictionary.", file=sys.stderr) + return + default_dict = default_dict_path.read_text(encoding="utf-8") + if default_dict == default_dict.rstrip(): + default_dict += "\n" + csv_text += default_dict + + # ユーザー辞書データの追加 + user_dict = read_dict(user_dict_path=user_dict_path) + for word_uuid in user_dict: + word = user_dict[word_uuid] + csv_text += ( + "{surface},{context_id},{context_id},{cost},{part_of_speech}," + + "{part_of_speech_detail_1},{part_of_speech_detail_2}," + + "{part_of_speech_detail_3},{inflectional_type}," + + "{inflectional_form},{stem},{yomi},{pronunciation}," + + "{accent_type}/{mora_count},{accent_associative_rule}\n" + ).format( + surface=word.surface, + context_id=word.context_id, + cost=_priority2cost(word.context_id, word.priority), + part_of_speech=word.part_of_speech, + part_of_speech_detail_1=word.part_of_speech_detail_1, + part_of_speech_detail_2=word.part_of_speech_detail_2, + part_of_speech_detail_3=word.part_of_speech_detail_3, + inflectional_type=word.inflectional_type, + inflectional_form=word.inflectional_form, + stem=word.stem, + yomi=word.yomi, + pronunciation=word.pronunciation, + accent_type=word.accent_type, + mora_count=word.mora_count, + accent_associative_rule=word.accent_associative_rule, + ) + # 辞書データを辞書.csv へ一時保存 + tmp_csv_path.write_text(csv_text, encoding="utf-8") + + # 辞書.csvをOpenJTalk用にコンパイル + # pyopenjtalk.create_user_dict(str(tmp_csv_path), str(tmp_compiled_path)) + pyopenjtalk.mecab_dict_index(str(tmp_csv_path), str(tmp_compiled_path)) + if not tmp_compiled_path.is_file(): + raise RuntimeError("辞書のコンパイル時にエラーが発生しました。") + + # コンパイル済み辞書の置き換え・読み込み + pyopenjtalk.unset_user_dict() + tmp_compiled_path.replace(compiled_dict_path) + if compiled_dict_path.is_file(): + # pyopenjtalk.set_user_dict(str(compiled_dict_path.resolve(strict=True))) + pyopenjtalk.update_global_jtalk_with_user_dict(str(compiled_dict_path)) + + except Exception as e: + print("Error: Failed to update dictionary.", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + raise e + + finally: + # 後処理 + if tmp_csv_path.exists(): + tmp_csv_path.unlink() + if tmp_compiled_path.exists(): + tmp_compiled_path.unlink() + + +# @mutex_wrapper(mutex_user_dict) +def read_dict(user_dict_path: Path = user_dict_path) -> Dict[str, UserDictWord]: + """ + ユーザー辞書の読み出し + Parameters + ---------- + user_dict_path : Path + ユーザー辞書ファイルのパス + Returns + ------- + result : Dict[str, UserDictWord] + ユーザー辞書 + """ + # 指定ユーザー辞書が存在しない場合、空辞書を返す + if not user_dict_path.is_file(): + return {} + + with user_dict_path.open(encoding="utf-8") as f: + result: Dict[str, UserDictWord] = {} + for word_uuid, word in json.load(f).items(): + # cost2priorityで変換を行う際にcontext_idが必要となるが、 + # 0.12以前の辞書は、context_idがハードコーディングされていたためにユーザー辞書内に保管されていない + # ハードコーディングされていたcontext_idは固有名詞を意味するものなので、固有名詞のcontext_idを補完する + if word.get("context_id") is None: + word["context_id"] = part_of_speech_data[ + WordTypes.PROPER_NOUN + ].context_id + word["priority"] = _cost2priority(word["context_id"], word["cost"]) + del word["cost"] + result[str(UUID(word_uuid))] = UserDictWord(**word) + + return result + + +def _create_word( + surface: str, + pronunciation: str, + accent_type: int, + word_type: Optional[WordTypes] = None, + priority: Optional[int] = None, +) -> UserDictWord: + """ + 単語オブジェクトの生成 + Parameters + ---------- + surface : str + 単語情報 + pronunciation : str + 単語情報 + accent_type : int + 単語情報 + word_type : Optional[WordTypes] + 品詞 + priority : Optional[int] + 優先度 + Returns + ------- + : UserDictWord + 単語オブジェクト + """ + if word_type is None: + word_type = WordTypes.PROPER_NOUN + if word_type not in part_of_speech_data.keys(): + raise HTTPException(status_code=422, detail="不明な品詞です") + if priority is None: + priority = 5 + if not MIN_PRIORITY <= priority <= MAX_PRIORITY: + raise HTTPException(status_code=422, detail="優先度の値が無効です") + pos_detail = part_of_speech_data[word_type] + return UserDictWord( + surface=surface, + context_id=pos_detail.context_id, + priority=priority, + part_of_speech=pos_detail.part_of_speech, + part_of_speech_detail_1=pos_detail.part_of_speech_detail_1, + part_of_speech_detail_2=pos_detail.part_of_speech_detail_2, + part_of_speech_detail_3=pos_detail.part_of_speech_detail_3, + inflectional_type="*", + inflectional_form="*", + stem="*", + yomi=pronunciation, + pronunciation=pronunciation, + accent_type=accent_type, + accent_associative_rule="*", + ) + + +def apply_word( + surface: str, + pronunciation: str, + accent_type: int, + word_type: Optional[WordTypes] = None, + priority: Optional[int] = None, + user_dict_path: Path = user_dict_path, + compiled_dict_path: Path = compiled_dict_path, +) -> str: + """ + 新規単語の追加 + Parameters + ---------- + surface : str + 単語情報 + pronunciation : str + 単語情報 + accent_type : int + 単語情報 + word_type : Optional[WordTypes] + 品詞 + priority : Optional[int] + 優先度 + user_dict_path : Path + ユーザー辞書ファイルのパス + compiled_dict_path : Path + コンパイル済み辞書ファイルのパス + Returns + ------- + word_uuid : UserDictWord + 追加された単語に発行されたUUID + """ + # 新規単語の追加による辞書データの更新 + word = _create_word( + surface=surface, + pronunciation=pronunciation, + accent_type=accent_type, + word_type=word_type, + priority=priority, + ) + user_dict = read_dict(user_dict_path=user_dict_path) + word_uuid = str(uuid4()) + user_dict[word_uuid] = word + + # 更新された辞書データの保存と適用 + _write_to_json(user_dict, user_dict_path) + update_dict(user_dict_path=user_dict_path, compiled_dict_path=compiled_dict_path) + + return word_uuid + + +def rewrite_word( + word_uuid: str, + surface: str, + pronunciation: str, + accent_type: int, + word_type: Optional[WordTypes] = None, + priority: Optional[int] = None, + user_dict_path: Path = user_dict_path, + compiled_dict_path: Path = compiled_dict_path, +) -> None: + """ + 既存単語の上書き更新 + Parameters + ---------- + word_uuid : str + 単語UUID + surface : str + 単語情報 + pronunciation : str + 単語情報 + accent_type : int + 単語情報 + word_type : Optional[WordTypes] + 品詞 + priority : Optional[int] + 優先度 + user_dict_path : Path + ユーザー辞書ファイルのパス + compiled_dict_path : Path + コンパイル済み辞書ファイルのパス + """ + word = _create_word( + surface=surface, + pronunciation=pronunciation, + accent_type=accent_type, + word_type=word_type, + priority=priority, + ) + + # 既存単語の上書きによる辞書データの更新 + user_dict = read_dict(user_dict_path=user_dict_path) + if word_uuid not in user_dict: + raise HTTPException( + status_code=422, detail="UUIDに該当するワードが見つかりませんでした" + ) + user_dict[word_uuid] = word + + # 更新された辞書データの保存と適用 + _write_to_json(user_dict, user_dict_path) + update_dict(user_dict_path=user_dict_path, compiled_dict_path=compiled_dict_path) + + +def delete_word( + word_uuid: str, + user_dict_path: Path = user_dict_path, + compiled_dict_path: Path = compiled_dict_path, +) -> None: + """ + 単語の削除 + Parameters + ---------- + word_uuid : str + 単語UUID + user_dict_path : Path + ユーザー辞書ファイルのパス + compiled_dict_path : Path + コンパイル済み辞書ファイルのパス + """ + # 既存単語の削除による辞書データの更新 + user_dict = read_dict(user_dict_path=user_dict_path) + if word_uuid not in user_dict: + raise HTTPException( + status_code=422, detail="IDに該当するワードが見つかりませんでした" + ) + del user_dict[word_uuid] + + # 更新された辞書データの保存と適用 + _write_to_json(user_dict, user_dict_path) + update_dict(user_dict_path=user_dict_path, compiled_dict_path=compiled_dict_path) + + +def import_user_dict( + dict_data: Dict[str, UserDictWord], + override: bool = False, + user_dict_path: Path = user_dict_path, + default_dict_path: Path = default_dict_path, + compiled_dict_path: Path = compiled_dict_path, +) -> None: + """ + ユーザー辞書のインポート + Parameters + ---------- + dict_data : Dict[str, UserDictWord] + インポートするユーザー辞書のデータ + override : bool + 重複したエントリがあった場合、上書きするかどうか + user_dict_path : Path + ユーザー辞書ファイルのパス + default_dict_path : Path + デフォルト辞書ファイルのパス + compiled_dict_path : Path + コンパイル済み辞書ファイルのパス + """ + # インポートする辞書データのバリデーション + for word_uuid, word in dict_data.items(): + UUID(word_uuid) + assert isinstance(word, UserDictWord) + for pos_detail in part_of_speech_data.values(): + if word.context_id == pos_detail.context_id: + assert word.part_of_speech == pos_detail.part_of_speech + assert ( + word.part_of_speech_detail_1 == pos_detail.part_of_speech_detail_1 + ) + assert ( + word.part_of_speech_detail_2 == pos_detail.part_of_speech_detail_2 + ) + assert ( + word.part_of_speech_detail_3 == pos_detail.part_of_speech_detail_3 + ) + assert ( + word.accent_associative_rule in pos_detail.accent_associative_rules + ) + break + else: + raise ValueError("対応していない品詞です") + + # 既存辞書の読み出し + old_dict = read_dict(user_dict_path=user_dict_path) + + # 辞書データの更新 + # 重複エントリの上書き + if override: + new_dict = {**old_dict, **dict_data} + # 重複エントリの保持 + else: + new_dict = {**dict_data, **old_dict} + + # 更新された辞書データの保存と適用 + _write_to_json(user_dict=new_dict, user_dict_path=user_dict_path) + update_dict( + default_dict_path=default_dict_path, + user_dict_path=user_dict_path, + compiled_dict_path=compiled_dict_path, + ) + + +def _search_cost_candidates(context_id: int) -> List[int]: + for value in part_of_speech_data.values(): + if value.context_id == context_id: + return value.cost_candidates + raise HTTPException(status_code=422, detail="品詞IDが不正です") + + +def _cost2priority(context_id: int, cost: int) -> int: + assert -32768 <= cost <= 32767 + cost_candidates = _search_cost_candidates(context_id) + # cost_candidatesの中にある値で最も近い値を元にpriorityを返す + # 参考: https://qiita.com/Krypf/items/2eada91c37161d17621d + # この関数とpriority2cost関数によって、辞書ファイルのcostを操作しても最も近いpriorityのcostに上書きされる + return MAX_PRIORITY - np.argmin(np.abs(np.array(cost_candidates) - cost)).item() + + +def _priority2cost(context_id: int, priority: int) -> int: + assert MIN_PRIORITY <= priority <= MAX_PRIORITY + cost_candidates = _search_cost_candidates(context_id) + return cost_candidates[MAX_PRIORITY - priority] diff --git a/text/user_dict/part_of_speech_data.py b/text/user_dict/part_of_speech_data.py new file mode 100644 index 0000000..3e80115 --- /dev/null +++ b/text/user_dict/part_of_speech_data.py @@ -0,0 +1,144 @@ +from typing import Dict + +from .word_model import ( + USER_DICT_MAX_PRIORITY, + USER_DICT_MIN_PRIORITY, + PartOfSpeechDetail, + WordTypes, +) + +MIN_PRIORITY = USER_DICT_MIN_PRIORITY +MAX_PRIORITY = USER_DICT_MAX_PRIORITY + +part_of_speech_data: Dict[WordTypes, PartOfSpeechDetail] = { + WordTypes.PROPER_NOUN: PartOfSpeechDetail( + part_of_speech="名詞", + part_of_speech_detail_1="固有名詞", + part_of_speech_detail_2="一般", + part_of_speech_detail_3="*", + context_id=1348, + cost_candidates=[ + -988, + 3488, + 4768, + 6048, + 7328, + 8609, + 8734, + 8859, + 8984, + 9110, + 14176, + ], + accent_associative_rules=[ + "*", + "C1", + "C2", + "C3", + "C4", + "C5", + ], + ), + WordTypes.COMMON_NOUN: PartOfSpeechDetail( + part_of_speech="名詞", + part_of_speech_detail_1="一般", + part_of_speech_detail_2="*", + part_of_speech_detail_3="*", + context_id=1345, + cost_candidates=[ + -4445, + 49, + 1473, + 2897, + 4321, + 5746, + 6554, + 7362, + 8170, + 8979, + 15001, + ], + accent_associative_rules=[ + "*", + "C1", + "C2", + "C3", + "C4", + "C5", + ], + ), + WordTypes.VERB: PartOfSpeechDetail( + part_of_speech="動詞", + part_of_speech_detail_1="自立", + part_of_speech_detail_2="*", + part_of_speech_detail_3="*", + context_id=642, + cost_candidates=[ + 3100, + 6160, + 6360, + 6561, + 6761, + 6962, + 7414, + 7866, + 8318, + 8771, + 13433, + ], + accent_associative_rules=[ + "*", + ], + ), + WordTypes.ADJECTIVE: PartOfSpeechDetail( + part_of_speech="形容詞", + part_of_speech_detail_1="自立", + part_of_speech_detail_2="*", + part_of_speech_detail_3="*", + context_id=20, + cost_candidates=[ + 1527, + 3266, + 3561, + 3857, + 4153, + 4449, + 5149, + 5849, + 6549, + 7250, + 10001, + ], + accent_associative_rules=[ + "*", + ], + ), + WordTypes.SUFFIX: PartOfSpeechDetail( + part_of_speech="名詞", + part_of_speech_detail_1="接尾", + part_of_speech_detail_2="一般", + part_of_speech_detail_3="*", + context_id=1358, + cost_candidates=[ + 4399, + 5373, + 6041, + 6710, + 7378, + 8047, + 9440, + 10834, + 12228, + 13622, + 15847, + ], + accent_associative_rules=[ + "*", + "C1", + "C2", + "C3", + "C4", + "C5", + ], + ), +} diff --git a/text/user_dict/word_model.py b/text/user_dict/word_model.py new file mode 100644 index 0000000..fda5576 --- /dev/null +++ b/text/user_dict/word_model.py @@ -0,0 +1,124 @@ +from enum import Enum +from re import findall, fullmatch +from typing import List, Optional + +from pydantic import BaseModel, Field, validator + +USER_DICT_MIN_PRIORITY = 0 +USER_DICT_MAX_PRIORITY = 10 + + +class UserDictWord(BaseModel): + """ + 辞書のコンパイルに使われる情報 + """ + + surface: str = Field(title="表層形") + priority: int = Field( + title="優先度", ge=USER_DICT_MIN_PRIORITY, le=USER_DICT_MAX_PRIORITY + ) + context_id: int = Field(title="文脈ID", default=1348) + part_of_speech: str = Field(title="品詞") + part_of_speech_detail_1: str = Field(title="品詞細分類1") + part_of_speech_detail_2: str = Field(title="品詞細分類2") + part_of_speech_detail_3: str = Field(title="品詞細分類3") + inflectional_type: str = Field(title="活用型") + inflectional_form: str = Field(title="活用形") + stem: str = Field(title="原形") + yomi: str = Field(title="読み") + pronunciation: str = Field(title="発音") + accent_type: int = Field(title="アクセント型") + mora_count: Optional[int] = Field(title="モーラ数", default=None) + accent_associative_rule: str = Field(title="アクセント結合規則") + + class Config: + validate_assignment = True + + @validator("surface") + def convert_to_zenkaku(cls, surface): + return surface.translate( + str.maketrans( + "".join(chr(0x21 + i) for i in range(94)), + "".join(chr(0xFF01 + i) for i in range(94)), + ) + ) + + @validator("pronunciation", pre=True) + def check_is_katakana(cls, pronunciation): + if not fullmatch(r"[ァ-ヴー]+", pronunciation): + raise ValueError("発音は有効なカタカナでなくてはいけません。") + sutegana = ["ァ", "ィ", "ゥ", "ェ", "ォ", "ャ", "ュ", "ョ", "ヮ", "ッ"] + for i in range(len(pronunciation)): + if pronunciation[i] in sutegana: + # 「キャット」のように、捨て仮名が連続する可能性が考えられるので、 + # 「ッ」に関しては「ッ」そのものが連続している場合と、「ッ」の後にほかの捨て仮名が連続する場合のみ無効とする + if i < len(pronunciation) - 1 and ( + pronunciation[i + 1] in sutegana[:-1] + or ( + pronunciation[i] == sutegana[-1] + and pronunciation[i + 1] == sutegana[-1] + ) + ): + raise ValueError("無効な発音です。(捨て仮名の連続)") + if pronunciation[i] == "ヮ": + if i != 0 and pronunciation[i - 1] not in ["ク", "グ"]: + raise ValueError( + "無効な発音です。(「くゎ」「ぐゎ」以外の「ゎ」の使用)" + ) + return pronunciation + + @validator("mora_count", pre=True, always=True) + def check_mora_count_and_accent_type(cls, mora_count, values): + if "pronunciation" not in values or "accent_type" not in values: + # 適切な場所でエラーを出すようにする + return mora_count + + if mora_count is None: + rule_others = ( + "[イ][ェ]|[ヴ][ャュョ]|[トド][ゥ]|[テデ][ィャュョ]|[デ][ェ]|[クグ][ヮ]" + ) + rule_line_i = "[キシチニヒミリギジビピ][ェャュョ]" + rule_line_u = "[ツフヴ][ァ]|[ウスツフヴズ][ィ]|[ウツフヴ][ェォ]" + rule_one_mora = "[ァ-ヴー]" + mora_count = len( + findall( + f"(?:{rule_others}|{rule_line_i}|{rule_line_u}|{rule_one_mora})", + values["pronunciation"], + ) + ) + + if not 0 <= values["accent_type"] <= mora_count: + raise ValueError( + "誤ったアクセント型です({})。 expect: 0 <= accent_type <= {}".format( + values["accent_type"], mora_count + ) + ) + return mora_count + + +class PartOfSpeechDetail(BaseModel): + """ + 品詞ごとの情報 + """ + + part_of_speech: str = Field(title="品詞") + part_of_speech_detail_1: str = Field(title="品詞細分類1") + part_of_speech_detail_2: str = Field(title="品詞細分類2") + part_of_speech_detail_3: str = Field(title="品詞細分類3") + # context_idは辞書の左・右文脈IDのこと + # https://github.com/VOICEVOX/open_jtalk/blob/427cfd761b78efb6094bea3c5bb8c968f0d711ab/src/mecab-naist-jdic/_left-id.def # noqa + context_id: int = Field(title="文脈ID") + cost_candidates: List[int] = Field(title="コストのパーセンタイル") + accent_associative_rules: List[str] = Field(title="アクセント結合規則の一覧") + + +class WordTypes(str, Enum): + """ + fastapiでword_type引数を検証する時に使用するクラス + """ + + PROPER_NOUN = "PROPER_NOUN" + COMMON_NOUN = "COMMON_NOUN" + VERB = "VERB" + ADJECTIVE = "ADJECTIVE" + SUFFIX = "SUFFIX" From e76dca6fbb0e255d8c3b1b60ca742fdab437c03e Mon Sep 17 00:00:00 2001 From: litagin02 Date: Sun, 25 Feb 2024 13:15:00 +0900 Subject: [PATCH 41/51] Update dict api --- server_editor.py | 96 ++++++++++++++++++++---------------------------- text/japanese.py | 9 ++--- 2 files changed, 44 insertions(+), 61 deletions(-) diff --git a/server_editor.py b/server_editor.py index f9420cf..afadaec 100644 --- a/server_editor.py +++ b/server_editor.py @@ -37,15 +37,16 @@ from common.constants import ( DEFAULT_STYLE, DEFAULT_STYLE_WEIGHT, LATEST_VERSION, - USER_DICT_CSV_PATH, - USER_DICT_PATH, Languages, ) from common.log import logger from common.tts_model import ModelHolder from text.japanese import g2kata_tone, kata_tone2phone_tone +from text.user_dict import apply_word, update_dict, read_dict, rewrite_word, delete_word +# ---フロントエンド部分に関する処理--- + # エディターのビルドファイルを配置するディレクトリ STATIC_DIR = Path("static") # エディターの最新のビルドファイルのダウンロード日時を記録するファイル @@ -135,6 +136,10 @@ def save_last_download(latest_release): file.write(latest_release["published_at"]) +# ---フロントエンド部分に関する処理ここまで--- +# 以降はAPIの設定 + + class AudioResponse(Response): media_type = "audio/wav" @@ -337,73 +342,52 @@ def multi_synthesis(request: MultiSynthesisRequest): return Response(content=wavContent.getvalue(), media_type="audio/wav") -class AddUserDictWordRequest(BaseModel): +class UserDictWordRequest(BaseModel): surface: str pronunciation: str - accentType: str # アクセント核位置/モーラ数、例：1/3, アクセント核位置は1から始まる + accent_type: int # アクセント核位置（存在しない場合は0、1文字目は1） priority: int = 5 -# コストの値は以下の値を参考にしている -# https://github.com/VOICEVOX/voicevox_engine/blob/master/voicevox_engine/user_dict/part_of_speech_data.py -COST_CANDIDATES = [-988, 3488, 4768, 6048, 7328, 8609, 8734, 8859, 8984, 9110, 14176] +@router.get("/user_dict") +def get_user_dict(): + return read_dict() @router.post("/user_dict_word") -def add_user_dict_word(request: AddUserDictWordRequest): - if request.surface == "" or request.pronunciation == "": - return JSONResponse( - status_code=status.HTTP_400_BAD_REQUEST, - content={"message": "単語か読みが空です。"}, - ) - user_dict_file = Path(USER_DICT_CSV_PATH) - user_dict_file.parent.mkdir(parents=True, exist_ok=True) +def add_user_dict_word(request: UserDictWordRequest): + uuid = apply_word( + surface=request.surface, + pronunciation=request.pronunciation, + accent_type=request.accent_type, + priority=request.priority, + ) + update_dict() - # 新規追加または更新する単語のCSV行 - new_csv_row = f"{request.surface},,,{COST_CANDIDATES[request.priority]},名詞,固有名詞,一般,*,*,*,{request.surface},{request.pronunciation},{request.pronunciation},{request.accentType},*\n" - found = False - updated = False + return JSONResponse( + status_code=status.HTTP_201_CREATED, + content={"uuid": uuid}, + ) - # ユーザー辞書ファイルが存在する場合、既存の単語をチェックし、必要に応じて更新する - if user_dict_file.exists(): - with user_dict_file.open(encoding="utf-8") as f: - lines = f.readlines() - with user_dict_file.open("w", encoding="utf-8") as f: - for line in lines: - if line.split(",")[0] == request.surface: - found = True - if line.strip() != new_csv_row.strip(): - f.write(new_csv_row) - updated = True - else: - f.write(line) - else: - f.write(line) - # 単語が新しい場合、新規に追加 - if not found: - with user_dict_file.open("a", encoding="utf-8") as f: - f.write(new_csv_row) +@router.put("/user_dict_word/{uuid}") +def update_user_dict_word(uuid: str, request: UserDictWordRequest): + rewrite_word( + word_uuid=uuid, + surface=request.surface, + pronunciation=request.pronunciation, + accent_type=request.accent_type, + priority=request.priority, + ) + update_dict() + return JSONResponse(status_code=status.HTTP_200_OK, content={"uuid": uuid}) - pyopenjtalk.unset_user_dict() - pyopenjtalk.mecab_dict_index(USER_DICT_CSV_PATH, USER_DICT_PATH) - pyopenjtalk.update_global_jtalk_with_user_dict(USER_DICT_PATH) - if updated: - return JSONResponse( - status_code=status.HTTP_200_OK, - content={"message": "単語が既に存在し、更新されました。"}, - ) - elif found: - return JSONResponse( - status_code=status.HTTP_200_OK, - content={"message": "既に登録されている単語です。更新はありません。"}, - ) - else: - return JSONResponse( - status_code=status.HTTP_201_CREATED, - content={"message": "単語が新規追加されました。"}, - ) +@router.delete("/user_dict_word/{uuid}") +def delete_user_dict_word(uuid: str): + delete_word(uuid) + update_dict() + return JSONResponse(status_code=status.HTTP_200_OK, content={"uuid": uuid}) app.include_router(router, prefix="/api") diff --git a/text/japanese.py b/text/japanese.py index 39c8c6a..5a36eb6 100644 --- a/text/japanese.py +++ b/text/japanese.py @@ -8,7 +8,6 @@ import pyopenjtalk from num2words import num2words from transformers import AutoTokenizer -from common.constants import USER_DICT_PATH, USER_DICT_CSV_PATH from common.log import logger from text import punctuation from text.japanese_mora_list import ( @@ -16,10 +15,10 @@ from text.japanese_mora_list import ( mora_phonemes_to_mora_kata, ) -if Path(USER_DICT_CSV_PATH).exists(): - pyopenjtalk.mecab_dict_index(USER_DICT_CSV_PATH, USER_DICT_PATH) -if Path(USER_DICT_PATH).exists(): - pyopenjtalk.update_global_jtalk_with_user_dict(USER_DICT_PATH) +from text.user_dict import update_dict + +# 最初にpyopenjtalkの辞書を更新 +update_dict() # 子音の集合 COSONANTS = set( From 8f27c79219f42e2e1527ed1e938090c9ff1fe6be Mon Sep 17 00:00:00 2001 From: litagin02 Date: Sun, 25 Feb 2024 13:15:40 +0900 Subject: [PATCH 42/51] Update --- Dockerfile.deploy | 3 ++- Editor.bat | 11 +++++++++++ common/constants.py | 3 +-- docs/paperspace.md | 4 ++-- scripts/Install-Style-Bert-VITS2-CPU.bat | 6 +++--- scripts/Install-Style-Bert-VITS2.bat | 6 +++--- scripts/Update-to-Dict-Editor.bat | 20 +++++++++++--------- scripts/Update-to-JP-Extra.bat | 9 +++++++++ transcribe.py | 11 ++++------- 9 files changed, 46 insertions(+), 27 deletions(-) create mode 100644 Editor.bat diff --git a/Dockerfile.deploy b/Dockerfile.deploy index f6cab39..dd351d1 100644 --- a/Dockerfile.deploy +++ b/Dockerfile.deploy @@ -19,4 +19,5 @@ COPY --chown=user . $HOME/app RUN pip install --no-cache-dir -r $HOME/app/requirements.txt -CMD ["python", "server_editor.py"] +# 必要に応じて制限を変更してください +CMD ["python", "server_editor.py", "--line_length", "50", "--line_count", "3"] diff --git a/Editor.bat b/Editor.bat new file mode 100644 index 0000000..7b0836c --- /dev/null +++ b/Editor.bat @@ -0,0 +1,11 @@ +chcp 65001 > NUL +@echo off + +pushd %~dp0 +echo Running server_editor.py --inbroser +venv\Scripts\python server_editor.py --inbrowser + +if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) + +popd +pause \ No newline at end of file diff --git a/common/constants.py b/common/constants.py index 5fe2e0a..751d5e1 100644 --- a/common/constants.py +++ b/common/constants.py @@ -6,8 +6,7 @@ GRADIO_THEME: str = "NoCrypt/miku" LATEST_VERSION: str = "2.3" -USER_DICT_CSV_PATH: str = "dict/user_dict.csv" -USER_DICT_PATH: str = "dict/user.dic" +USER_DICT_DIR = "dict_data" DEFAULT_STYLE: str = "Neutral" DEFAULT_STYLE_WEIGHT: float = 5.0 diff --git a/docs/paperspace.md b/docs/paperspace.md index 924d3b3..b77ef57 100644 --- a/docs/paperspace.md +++ b/docs/paperspace.md @@ -58,9 +58,9 @@ python slice.py --model_name Foo -i /notebooks/inputs python transcribe.py --model_name Foo ``` -それが終わったら、以下のコマンドで一括前処理を行う（パラメータは各自お好み、バッチサイズ6でVRAM 16GBギリくらい）。 +それが終わったら、以下のコマンドで一括前処理を行う（パラメータは各自お好み、バッチサイズ5か6でVRAM 16GBギリくらい）。 ```bash -python preprocess_all.py --model_name Foo -b 6 -e 300 --use_jp_extra +python preprocess_all.py --model_name Foo -b 5 -e 300 --use_jp_extra ``` ### 3. 学習 diff --git a/scripts/Install-Style-Bert-VITS2-CPU.bat b/scripts/Install-Style-Bert-VITS2-CPU.bat index aaf3536..aca6f50 100644 --- a/scripts/Install-Style-Bert-VITS2-CPU.bat +++ b/scripts/Install-Style-Bert-VITS2-CPU.bat @@ -55,11 +55,11 @@ pushd Style-Bert-VITS2 python initialize.py echo ---------------------------------------- -echo Model download is complete. Start the WebUI of the voice synthesis. +echo Model download is complete. Start Style-Bert-VITS2 Editor. echo ---------------------------------------- -@REM 音声合成WebUIの起動 -python app.py +@REM エディターの起動 +python server_editor.py --inbrowser pause diff --git a/scripts/Install-Style-Bert-VITS2.bat b/scripts/Install-Style-Bert-VITS2.bat index 6e99230..59fe97e 100644 --- a/scripts/Install-Style-Bert-VITS2.bat +++ b/scripts/Install-Style-Bert-VITS2.bat @@ -58,11 +58,11 @@ pushd Style-Bert-VITS2 python initialize.py echo ---------------------------------------- -echo Model download is complete. Start the WebUI of the voice synthesis. +echo Model download is complete. Start Style-Bert-VITS2 Editor. echo ---------------------------------------- -@REM 音声合成WebUIの起動 -python app.py +@REM エディターの起動 +python server_editor.py --inbrowser pause diff --git a/scripts/Update-to-Dict-Editor.bat b/scripts/Update-to-Dict-Editor.bat index 254b060..e10060d 100644 --- a/scripts/Update-to-Dict-Editor.bat +++ b/scripts/Update-to-Dict-Editor.bat @@ -12,7 +12,7 @@ if not exist %CURL_CMD% ( @REM Style-Bert-VITS2.zip をGitHubのmasterの最新のものをダウンロード %CURL_CMD% -Lo Style-Bert-VITS2.zip^ - https://github.com/litagin02/Style-Bert-VITS2/archive/refs/heads/master.zip + https://github.com/litagin02/Style-Bert-VITS2/archive/refs/heads/dev.zip if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) @REM Style-Bert-VITS2.zip を解凍（フォルダ名前がBert-VITS2-masterになる） @@ -24,8 +24,8 @@ del Style-Bert-VITS2.zip if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) @REM Bert-VITS2-masterの中身をStyle-Bert-VITS2に上書き移動 -xcopy /QSY .\Style-Bert-VITS2-master\ .\Style-Bert-VITS2\ -rmdir /s /q Style-Bert-VITS2-master +xcopy /QSY .\Style-Bert-VITS2-dev\ .\Style-Bert-VITS2\ +rmdir /s /q Style-Bert-VITS2-dev if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) @REM 仮想環境のpip requirements.txtを更新 @@ -34,17 +34,19 @@ echo call .\Style-Bert-VITS2\scripts\activate.bat call .\Style-Bert-VITS2\venv\Scripts\activate.bat if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) +pause + @REM pyopenjtalk-prebuiltやpyopenjtalkが入っていたら削除 -echo pip uninstall -y pyopenjtalk-prebuilt pyopenjtalk -pip uninstall -y pyopenjtalk-prebuilt pyopenjtalk +echo python -m pip uninstall -y pyopenjtalk-prebuilt pyopenjtalk +python -m pip uninstall -y pyopenjtalk-prebuilt pyopenjtalk if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) @REM pyopenjtalk-dictをインストール -echo pip install -U pyopenjtalk-dict -pip install -U pyopenjtalk-dict +echo python -m pip install -U pyopenjtalk-dict +python -m pip install -U pyopenjtalk-dict @REM その他のrequirements.txtも一応更新 -pip install -U -r Style-Bert-VITS2\requirements.txt +python -m pip install -U -r Style-Bert-VITS2\requirements.txt if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) pushd Style-Bert-VITS2 @@ -52,7 +54,7 @@ pushd Style-Bert-VITS2 echo Update completed. Running Style-Bert-VITS2 Editor... @REM Style-Bert-VITS2 Editorを起動 -python server_editor.py +python server_editor.py --inbrowser pause diff --git a/scripts/Update-to-JP-Extra.bat b/scripts/Update-to-JP-Extra.bat index 95738c6..228cf10 100644 --- a/scripts/Update-to-JP-Extra.bat +++ b/scripts/Update-to-JP-Extra.bat @@ -34,6 +34,15 @@ echo call .\Style-Bert-VITS2\scripts\activate.bat call .\Style-Bert-VITS2\venv\Scripts\activate.bat if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) +echo pip uninstall -y pyopenjtalk-prebuilt pyopenjtalk +pip uninstall -y pyopenjtalk-prebuilt pyopenjtalk +if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) + +echo pip install -U pyopenjtalk-dict +pip install -U pyopenjtalk-dict +if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) + +echo pip install -U -r Style-Bert-VITS2\requirements.txt pip install -U -r Style-Bert-VITS2\requirements.txt if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) diff --git a/transcribe.py b/transcribe.py index 0ec9e99..4a28fd1 100644 --- a/transcribe.py +++ b/transcribe.py @@ -78,12 +78,9 @@ if __name__ == "__main__": language_id = Languages.ZH.value else: raise ValueError(f"{language} is not supported.") - with open(output_file, "w", encoding="utf-8") as f: - for wav_file in tqdm(wav_files, file=SAFE_STDOUT): - file_name = os.path.basename(wav_file) - text = transcribe( - wav_file, initial_prompt=initial_prompt, language=language - ) + for wav_file in tqdm(wav_files, file=SAFE_STDOUT): + file_name = os.path.basename(wav_file) + text = transcribe(wav_file, initial_prompt=initial_prompt, language=language) + with open(output_file, "a", encoding="utf-8") as f: f.write(f"{file_name}|{model_name}|{language_id}|{text}\n") - f.flush() sys.exit(0) From 8d7d7c32652356021b70655f3ccc44463c5ccb47 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Sun, 25 Feb 2024 17:42:50 +0900 Subject: [PATCH 43/51] Avoid using pyworld by default --- app.py | 6 +++++- common/tts_model.py | 24 ++++++++++++++++-------- requirements.txt | 8 +++----- scripts/Update-to-Dict-Editor.bat | 4 +--- 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/app.py b/app.py index d99229b..e17a0a5 100644 --- a/app.py +++ b/app.py @@ -202,7 +202,9 @@ examples = [ initial_md = f""" # Style-Bert-VITS2 ver {LATEST_VERSION} 音声合成 -注意: 初期からある[jvnvのモデル](https://huggingface.co/litagin/style_bert_vits2_jvnv)は、[JVNVコーパス（言語音声と非言語音声を持つ日本語感情音声コーパス）](https://sites.google.com/site/shinnosuketakamichi/research-topics/jvnv_corpus)で学習されたモデルです。ライセンスは[CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/deed.ja)です。 +- Ver 2.3で追加されたエディターのほうが実際に読み上げさせるには使いやすいかもしれません。`Editor.bat`か`python server_editor.py`で起動できます。 + +- 初期からある[jvnvのモデル](https://huggingface.co/litagin/style_bert_vits2_jvnv)は、[JVNVコーパス（言語音声と非言語音声を持つ日本語感情音声コーパス）](https://sites.google.com/site/shinnosuketakamichi/research-topics/jvnv_corpus)で学習されたモデルです。ライセンスは[CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/deed.ja)です。 """ how_to_md = """ @@ -317,6 +319,7 @@ if __name__ == "__main__": value=1, step=0.05, label="音程(1以外では音質劣化)", + visible=False, # pyworldが必要 ) intonation_scale = gr.Slider( minimum=0, @@ -324,6 +327,7 @@ if __name__ == "__main__": value=1, step=0.1, label="抑揚(1以外では音質劣化)", + visible=False, # pyworldが必要 ) line_split = gr.Checkbox( diff --git a/common/tts_model.py b/common/tts_model.py index 4f3932a..de4e830 100644 --- a/common/tts_model.py +++ b/common/tts_model.py @@ -5,7 +5,7 @@ from typing import Optional, Union import gradio as gr import numpy as np -import pyworld + import torch from gradio.processing_utils import convert_to_16_bit_wav @@ -33,6 +33,13 @@ def adjust_voice(fs, wave, pitch_scale, intonation_scale): # 初期値の場合は、音質劣化を避けるためにそのまま返す return fs, wave + try: + import pyworld + except ImportError: + raise ImportError( + "pyworld is not installed. Please install it by `pip install pyworld`" + ) + # pyworldでf0を加工して合成 # pyworldよりもよいのがあるかもしれないが…… @@ -197,16 +204,17 @@ class Model: audios.append(np.zeros(int(44100 * split_interval))) audio = np.concatenate(audios) logger.info("Audio data generated successfully") - fs, audio = adjust_voice( - fs=self.hps.data.sampling_rate, - wave=audio, - pitch_scale=pitch_scale, - intonation_scale=intonation_scale, - ) + if not (pitch_scale == 1.0 and intonation_scale == 1.0): + _, audio = adjust_voice( + fs=self.hps.data.sampling_rate, + wave=audio, + pitch_scale=pitch_scale, + intonation_scale=intonation_scale, + ) with warnings.catch_warnings(): warnings.simplefilter("ignore") audio = convert_to_16_bit_wav(audio) - return (fs, audio) + return (self.hps.data.sampling_rate, audio) class ModelHolder: diff --git a/requirements.txt b/requirements.txt index b346afc..6bf329f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,14 @@ cmudict cn2an -faster-whisper>=0.10.0,<1.0.0 +faster-whisper==0.10.1 g2p_en GPUtil gradio -jaconv jieba langid librosa==0.9.2 loguru matplotlib -mecab-python3 num2words numba numpy @@ -20,12 +18,12 @@ pyloudnorm # pyopenjtalk-prebuilt # Should be manually uninstalled pyopenjtalk-dict pypinyin -pyworld +# pyworld # Not supported on Windows without Cython... PyYAML requests safetensors scipy tensorboard -torch>=2.1,<2.2 # For users without GPU or colab +torch>=2.1,<2.2 transformers umap-learn diff --git a/scripts/Update-to-Dict-Editor.bat b/scripts/Update-to-Dict-Editor.bat index e10060d..5fde8fc 100644 --- a/scripts/Update-to-Dict-Editor.bat +++ b/scripts/Update-to-Dict-Editor.bat @@ -28,14 +28,12 @@ xcopy /QSY .\Style-Bert-VITS2-dev\ .\Style-Bert-VITS2\ rmdir /s /q Style-Bert-VITS2-dev if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) -@REM 仮想環境のpip requirements.txtを更新 +@REM 仮想環境を有効化 echo call .\Style-Bert-VITS2\scripts\activate.bat call .\Style-Bert-VITS2\venv\Scripts\activate.bat if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) -pause - @REM pyopenjtalk-prebuiltやpyopenjtalkが入っていたら削除 echo python -m pip uninstall -y pyopenjtalk-prebuilt pyopenjtalk python -m pip uninstall -y pyopenjtalk-prebuilt pyopenjtalk From 9d6a1db0e91a0110d4ca03770a619978488c215f Mon Sep 17 00:00:00 2001 From: litagin02 Date: Sun, 25 Feb 2024 18:33:47 +0900 Subject: [PATCH 44/51] Fix: handling esd.list processing error --- docs/paperspace.md | 4 +++- webui_train.py | 20 ++++++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/docs/paperspace.md b/docs/paperspace.md index b77ef57..2ae9895 100644 --- a/docs/paperspace.md +++ b/docs/paperspace.md @@ -12,11 +12,13 @@ 以下では次のような方針でやっています。 - `/storage/`は永続ストレージなので、事前学習モデルとかを含めてリポジトリをクローンするとよい。 -- `/notebooks/`は一時ストレージなので、データセットやその結果を保存する。 +- `/notebooks/`はノートブックごとに変わるストレージなので（同一ノートブック違うランタイムだと共有されるらしい）、データセットやその結果を保存する。ただ容量が多い場合はあふれる可能性があるので`/tmp/`に保存するとよいかもしれない。 - hugging faceアカウントを作り、（プライベートな）リポジトリを作って、学習元データを置いたり、学習結果を随時アップロードする。 ### 1. 環境を作る +以下はデフォルトの`Start from Scratch`で作成した環境の場合。[Dockerfile.train](../Dockerfile.train)を使ったカスタムイメージをするとPythonの環境構築の手間がちょっと省けるので、それを使いたい人は`Advanced Options / Container / Name`に[`litagin/mygradient:latest`](https://hub.docker.com/r/litagin/mygradient/tags)を指定すると使えます（pipの箇所が不要になる等）。 + まずは永続ストレージにgit clone ```bash mkdir -p /storage/sbv2 diff --git a/webui_train.py b/webui_train.py index eccfeaf..cd7af20 100644 --- a/webui_train.py +++ b/webui_train.py @@ -9,6 +9,7 @@ import time import webbrowser from datetime import datetime from multiprocessing import cpu_count +from pathlib import Path import gradio as gr import yaml @@ -162,13 +163,20 @@ def preprocess_text(model_name, use_jp_extra, val_per_lang): except FileNotFoundError: logger.error(f"Step 3: {lbl_path} not found.") return False, f"Step 3, Error: 書き起こしファイル {lbl_path} が見つかりません。" - with open(lbl_path, "w", encoding="utf-8") as f: - for line in lines: - path, spk, language, text = line.strip().split("|") - path = os.path.join(dataset_path, "wavs", os.path.basename(path)).replace( - "\\", "/" + new_lines = [] + for line in lines: + if len(line.strip().split("|")) != 4: + logger.error(f"Step 3: {lbl_path} has invalid format at line:\n{line}") + return ( + False, + f"Step 3, Error: 書き起こしファイル次の行の形式が不正です:\n{line}", ) - f.writelines(f"{path}|{spk}|{language}|{text}\n") + path, spk, language, text = line.strip().split("|") + # pathをファイル名だけ取り出して正しいパスに変更 + path = Path(dataset_path) / "wavs" / Path(path).name + new_lines.append(f"{path}|{spk}|{language}|{text}\n") + with open(lbl_path, "w", encoding="utf-8") as f: + f.writelines(new_lines) cmd = [ "preprocess_text.py", "--config-path", From d3a5cb8569b84687b3e8f3e27382d4c9bf7f1125 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Sun, 25 Feb 2024 18:56:45 +0900 Subject: [PATCH 45/51] Raise error instead of warning for visibility --- preprocess_text.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/preprocess_text.py b/preprocess_text.py index 7ba2310..2c26941 100644 --- a/preprocess_text.py +++ b/preprocess_text.py @@ -151,7 +151,10 @@ def preprocess( with open(config_path, "w", encoding="utf-8") as f: json.dump(json_config, f, indent=2, ensure_ascii=False) if error_count > 0: - logger.warning( + logger.error( + f"An error occurred in {error_count} lines. Please check {error_log_path} for details. You can proceed with lines that do not have errors." + ) + raise Exception( f"An error occurred in {error_count} lines. Please check {error_log_path} for details. You can proceed with lines that do not have errors." ) else: From f749950be25ddc659be57b34bd931067a6e0b529 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Sun, 25 Feb 2024 22:28:02 +0900 Subject: [PATCH 46/51] Update --- .dockerignore | 3 ++- docs/CHANGELOG.md | 22 ++++++++++++++++++++++ server_editor.py | 16 +++++++++++++++- text/user_dict/README.md | 9 ++++++--- 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/.dockerignore b/.dockerignore index 95ef322..7ff74da 100644 --- a/.dockerignore +++ b/.dockerignore @@ -30,7 +30,8 @@ LICENSE /bert/deberta-v3-large/ Data/ -dict/ +dict_data/user_dic.json +dict_data/user_dic.dic docs/ inputs/ mos_results/ diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index d9a3015..77cdca5 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## v2.3 (2024-02-25) + +### 大きな変更 + +#### ユーザー辞書機能 +あらかじめ辞書に固有名詞を追加することができ、それが学習時・音声合成時の読み取得部分に適応されます。辞書の追加・編集は次のエディタ経由で行ってください。 + +辞書部分の[実装](/text/user_dict/) は、中のREADMEにある通り、[VOICEVOX Editor](https://github.com/VOICEVOX/voicevox) のものを使っており、この部分のコードライセンスはLGPL-3. + +#### 音声合成専用エディタ + +音声合成専用エディタを追加。今までのWebUIでできた機能のほか、次のような機能が使えます（つまり既存の日本語音声合成ソフトウェアのエディタを真似ました）： +- セリフ単位でキャラや設定を変更しながら原稿を作り、それを一括で生成したり、原稿を保存等したり読み込んだり +- GUIよる分かりやすいアクセント調整 +- ユーザー辞書への単語追加や編集 + +`Editor.bat`をダブルクリックか`python server_editor.py --inbrowser`で起動します。エディター部分は[こちらの別リポジトリ](https://github.com/litagin02/Style-Bert-VITS2-Editor)になります。フロントエンド初心者なのでプルリクや改善案等をお待ちしています。 + +### 改善 +- 学習時にデコーダー部分を凍結するオプションの追加。品質がもしかしたら上がるかもしれません。 +- + ## v2.2 (2024-02-09) ### 変更・機能追加 diff --git a/server_editor.py b/server_editor.py index afadaec..f859315 100644 --- a/server_editor.py +++ b/server_editor.py @@ -16,6 +16,7 @@ import zipfile from datetime import datetime from io import BytesIO from pathlib import Path +import yaml import numpy as np import pyopenjtalk @@ -151,6 +152,12 @@ origins = [ "http://127.0.0.1:8000", ] +# Get path settings +with open(Path("configs/paths.yml"), "r", encoding="utf-8") as f: + path_config: dict[str, str] = yaml.safe_load(f.read()) + # dataset_root = path_config["dataset_root"] + assets_root = path_config["assets_root"] + parser = argparse.ArgumentParser() parser.add_argument("--model_dir", type=str, default="model_assets/") parser.add_argument("--device", type=str, default="cuda") @@ -158,6 +165,9 @@ parser.add_argument("--port", type=int, default=8000) parser.add_argument("--inbrowser", action="store_true") parser.add_argument("--line_length", type=int, default=None) parser.add_argument("--line_count", type=int, default=None) +parser.add_argument( + "--dir", "-d", type=str, help="Model directory", default=assets_root +) args = parser.parse_args() device = args.device @@ -296,7 +306,11 @@ def multi_synthesis(request: MultiSynthesisRequest): ) audios = [] for i, req in enumerate(lines): - # Loade model + if args.line_length is not None and len(req.text) > args.line_length: + raise HTTPException( + status_code=400, + detail=f"1行の文字数は{args.line_length}文字以下にしてください。", + ) try: model = model_holder.load_model( model_name=req.model, model_path_str=req.modelFile diff --git a/text/user_dict/README.md b/text/user_dict/README.md index c7190ea..d09d80a 100644 --- a/text/user_dict/README.md +++ b/text/user_dict/README.md @@ -1,15 +1,18 @@ -このモジュールは、[voicevox_engine](https://github.com/VOICEVOX/voicevox_engine)を使わせていただいています。 +このフォルダのコードは、[voicevox_engine](https://github.com/VOICEVOX/voicevox_engine)のものを使わせていただいています。 引用元: -https://github.com/VOICEVOX/voicevox_engine/tree/709527be089c0410c08e989df95a4a1d78439423/voicevox_engine/user_dict +https://github.com/VOICEVOX/voicevox_engine/tree/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/user_dict + +https://github.com/VOICEVOX/voicevox_engine/blob/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/model.py#L207 改変部分は以下のとおりです。 - ファイル名の書き換え、それに伴うimport文の書き換え -- VOICEVOX固有の他のモジュールへの依存のコメントアウト +- VOICEVOX固有の部分のコメントアウト - mutexを使用している部分のコメントアウト - 参照しているpyopenjtalkの違いによるメソッド名書き換え - UserDictWordのmora_countのデフォルト値をNoneに指定 +- Pydanticのモデルは必要な箇所のみを抽出 ライセンス: LGPL From 4e372cf72459af8589e282e38b780ab058798cb4 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Sun, 25 Feb 2024 23:40:43 +0900 Subject: [PATCH 47/51] Improve licene --- README.md | 12 +++++++++++ text/user_dict/README.md | 29 ++++++++++++--------------- text/user_dict/part_of_speech_data.py | 6 ++++++ 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index edd5ba5..7be91a2 100644 --- a/README.md +++ b/README.md @@ -170,6 +170,18 @@ In addition to the original reference (written below), I used the following repo [The pretrained model](https://huggingface.co/litagin/Style-Bert-VITS2-1.0-base) and [JP-Extra version](https://huggingface.co/litagin/Style-Bert-VITS2-2.0-base-JP-Extra) is essentially taken from [the original base model of Bert-VITS2 v2.1](https://huggingface.co/Garydesu/bert-vits2_base_model-2.1) and [JP-Extra pretrained model of Bert-VITS2](https://huggingface.co/Stardust-minus/Bert-VITS2-Japanese-Extra), so all the credits go to the original author ([Fish Audio](https://github.com/fishaudio)): +In addition, [text/user_dict/] module is based on the following repositories: +- [[voicevox_engine](https://github.com/VOICEVOX/voicevox_engine)] +and the license of this module is LGPL v3. + +## LICENSE + +This repository is licensed under the GNU Affero General Public License v3.0, the same as the original Bert-VITS2 repository. For more details, see [LICENSE](LICENSE). + +In addition, [text/user_dict/] module is licensed under the GNU Lesser General Public License v3.0, inherited from the original VOICEVOX engine repository. For more details, see [LGPL_LICENSE](text/user_dict/LGPL_LICENSE). + + + Below is the original README.md. --- diff --git a/text/user_dict/README.md b/text/user_dict/README.md index d09d80a..def5f71 100644 --- a/text/user_dict/README.md +++ b/text/user_dict/README.md @@ -1,19 +1,16 @@ -このフォルダのコードは、[voicevox_engine](https://github.com/VOICEVOX/voicevox_engine)のものを使わせていただいています。 +このフォルダに含まれるユーザー辞書関連のコードは、[VOICEVOX engine](https://github.com/VOICEVOX/voicevox_engine)プロジェクトのコードを改変したものを使用しています。VOICEVOXプロジェクトのチームに深く感謝し、その貢献を尊重します。 -引用元: +**元のコード**: +- [voicevox_engine/user_dict](https://github.com/VOICEVOX/voicevox_engine/tree/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/user_dict) +- [voicevox_engine/model.py](https://github.com/VOICEVOX/voicevox_engine/blob/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/model.py#L207) -https://github.com/VOICEVOX/voicevox_engine/tree/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/user_dict +**改変の詳細**: +- ファイル名の書き換えおよびそれに伴うimport文の書き換え。 +- VOICEVOX固有の部分をコメントアウト。 +- mutexを使用している部分をコメントアウト。 +- 参照しているpyopenjtalkの違いによるメソッド名の書き換え。 +- UserDictWordのmora_countのデフォルト値をNoneに指定。 +- Pydanticのモデルで必要な箇所のみを抽出。 -https://github.com/VOICEVOX/voicevox_engine/blob/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/model.py#L207 - -改変部分は以下のとおりです。 -- ファイル名の書き換え、それに伴うimport文の書き換え -- VOICEVOX固有の部分のコメントアウト -- mutexを使用している部分のコメントアウト -- 参照しているpyopenjtalkの違いによるメソッド名書き換え -- UserDictWordのmora_countのデフォルト値をNoneに指定 -- Pydanticのモデルは必要な箇所のみを抽出 - -ライセンス: LGPL - -[LGPL_LICENSE](LGPL_LICENSE)を参照してください。 +**ライセンス**: +元のVOICEVOX engineのリポジトリのコードは、LGPL v3 と、ソースコードの公開が不要な別ライセンスのデュアルライセンスの下で使用されています。当プロジェクトにおけるこのモジュールもLGPLライセンスの下にあります。詳細については、プロジェクトのルートディレクトリにある[LGPL_LICENSE](LGPL_LICENSE)ファイルをご参照ください。また、元のVOICEVOX engineプロジェクトのライセンスについては、[こちら](https://github.com/VOICEVOX/voicevox_engine/blob/master/LICENSE)をご覧ください。 diff --git a/text/user_dict/part_of_speech_data.py b/text/user_dict/part_of_speech_data.py index 3e80115..db7f839 100644 --- a/text/user_dict/part_of_speech_data.py +++ b/text/user_dict/part_of_speech_data.py @@ -1,3 +1,9 @@ +# このファイルは、VOICEVOXプロジェクトのVOICEVOX engineからお借りしています。 +# 引用元: +# https://github.com/VOICEVOX/voicevox_engine/blob/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/model.py#L207 +# ライセンス: LGPL-3.0 +# 詳しくは、このファイルの場所にあるREADME.mdを参照してください + from typing import Dict from .word_model import ( From e7d42185044fb420d659fbb083f347039ce74692 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Sun, 25 Feb 2024 23:46:43 +0900 Subject: [PATCH 48/51] Improve license docs --- text/user_dict/LGPL_LICENSE => LGPL_LICENSE | 0 README.md | 6 +++--- text/user_dict/README.md | 2 +- text/user_dict/__init__.py | 5 +++++ text/user_dict/part_of_speech_data.py | 4 ++-- text/user_dict/word_model.py | 5 +++++ 6 files changed, 16 insertions(+), 6 deletions(-) rename text/user_dict/LGPL_LICENSE => LGPL_LICENSE (100%) diff --git a/text/user_dict/LGPL_LICENSE b/LGPL_LICENSE similarity index 100% rename from text/user_dict/LGPL_LICENSE rename to LGPL_LICENSE diff --git a/README.md b/README.md index 7be91a2..b857e1d 100644 --- a/README.md +++ b/README.md @@ -170,15 +170,15 @@ In addition to the original reference (written below), I used the following repo [The pretrained model](https://huggingface.co/litagin/Style-Bert-VITS2-1.0-base) and [JP-Extra version](https://huggingface.co/litagin/Style-Bert-VITS2-2.0-base-JP-Extra) is essentially taken from [the original base model of Bert-VITS2 v2.1](https://huggingface.co/Garydesu/bert-vits2_base_model-2.1) and [JP-Extra pretrained model of Bert-VITS2](https://huggingface.co/Stardust-minus/Bert-VITS2-Japanese-Extra), so all the credits go to the original author ([Fish Audio](https://github.com/fishaudio)): -In addition, [text/user_dict/] module is based on the following repositories: -- [[voicevox_engine](https://github.com/VOICEVOX/voicevox_engine)] +In addition, [text/user_dict/](text/user_dict) module is based on the following repositories: +- [voicevox_engine](https://github.com/VOICEVOX/voicevox_engine)] and the license of this module is LGPL v3. ## LICENSE This repository is licensed under the GNU Affero General Public License v3.0, the same as the original Bert-VITS2 repository. For more details, see [LICENSE](LICENSE). -In addition, [text/user_dict/] module is licensed under the GNU Lesser General Public License v3.0, inherited from the original VOICEVOX engine repository. For more details, see [LGPL_LICENSE](text/user_dict/LGPL_LICENSE). +In addition, [text/user_dict/](text/user_dict) module is licensed under the GNU Lesser General Public License v3.0, inherited from the original VOICEVOX engine repository. For more details, see [LGPL_LICENSE](LGPL_LICENSE). diff --git a/text/user_dict/README.md b/text/user_dict/README.md index def5f71..b086a4d 100644 --- a/text/user_dict/README.md +++ b/text/user_dict/README.md @@ -13,4 +13,4 @@ - Pydanticのモデルで必要な箇所のみを抽出。 **ライセンス**: -元のVOICEVOX engineのリポジトリのコードは、LGPL v3 と、ソースコードの公開が不要な別ライセンスのデュアルライセンスの下で使用されています。当プロジェクトにおけるこのモジュールもLGPLライセンスの下にあります。詳細については、プロジェクトのルートディレクトリにある[LGPL_LICENSE](LGPL_LICENSE)ファイルをご参照ください。また、元のVOICEVOX engineプロジェクトのライセンスについては、[こちら](https://github.com/VOICEVOX/voicevox_engine/blob/master/LICENSE)をご覧ください。 +元のVOICEVOX engineのリポジトリのコードは、LGPL v3 と、ソースコードの公開が不要な別ライセンスのデュアルライセンスの下で使用されています。当プロジェクトにおけるこのモジュールもLGPLライセンスの下にあります。詳細については、プロジェクトのルートディレクトリにある[LGPL_LICENSE](/LGPL_LICENSE)ファイルをご参照ください。また、元のVOICEVOX engineプロジェクトのライセンスについては、[こちら](https://github.com/VOICEVOX/voicevox_engine/blob/master/LICENSE)をご覧ください。 diff --git a/text/user_dict/__init__.py b/text/user_dict/__init__.py index d258359..c12b3d1 100644 --- a/text/user_dict/__init__.py +++ b/text/user_dict/__init__.py @@ -1,3 +1,8 @@ +# このファイルは、VOICEVOXプロジェクトのVOICEVOX engineからお借りしています。 +# 引用元: +# https://github.com/VOICEVOX/voicevox_engine/blob/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/user_dict/user_dict.py +# ライセンス: LGPL-3.0 +# 詳しくは、このファイルと同じフォルダにあるREADME.mdを参照してください。 import json import sys import threading diff --git a/text/user_dict/part_of_speech_data.py b/text/user_dict/part_of_speech_data.py index db7f839..7e22699 100644 --- a/text/user_dict/part_of_speech_data.py +++ b/text/user_dict/part_of_speech_data.py @@ -1,8 +1,8 @@ # このファイルは、VOICEVOXプロジェクトのVOICEVOX engineからお借りしています。 # 引用元: -# https://github.com/VOICEVOX/voicevox_engine/blob/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/model.py#L207 +# https://github.com/VOICEVOX/voicevox_engine/blob/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/user_dict/part_of_speech_data.py # ライセンス: LGPL-3.0 -# 詳しくは、このファイルの場所にあるREADME.mdを参照してください +# 詳しくは、このファイルと同じフォルダにあるREADME.mdを参照してください。 from typing import Dict diff --git a/text/user_dict/word_model.py b/text/user_dict/word_model.py index fda5576..f05d8dc 100644 --- a/text/user_dict/word_model.py +++ b/text/user_dict/word_model.py @@ -1,3 +1,8 @@ +# このファイルは、VOICEVOXプロジェクトのVOICEVOX engineからお借りしています。 +# 引用元: +# https://github.com/VOICEVOX/voicevox_engine/blob/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/model.py#L207 +# ライセンス: LGPL-3.0 +# 詳しくは、このファイルと同じフォルダにあるREADME.mdを参照してください。 from enum import Enum from re import findall, fullmatch from typing import List, Optional From 36462c8203d82c9f2ab1c1c7734d7812b1818c84 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Mon, 26 Feb 2024 01:05:05 +0900 Subject: [PATCH 49/51] Format and docs --- common/log.py | 1 + docs/CHANGELOG.md | 30 +++++++++++++++++++++++++++--- text/tone_sandhi.py | 5 ++++- text/user_dict/README.md | 5 ++++- tools/translate.py | 1 + update_status.py | 8 ++++++-- webui.py | 1 + 7 files changed, 44 insertions(+), 7 deletions(-) diff --git a/common/log.py b/common/log.py index 51dca5f..679bb2c 100644 --- a/common/log.py +++ b/common/log.py @@ -1,6 +1,7 @@ """ logger封装 """ + from loguru import logger from .stdout_wrapper import SAFE_STDOUT diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 77cdca5..bbf1159 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -5,9 +5,15 @@ ### 大きな変更 #### ユーザー辞書機能 -あらかじめ辞書に固有名詞を追加することができ、それが学習時・音声合成時の読み取得部分に適応されます。辞書の追加・編集は次のエディタ経由で行ってください。 +あらかじめ辞書に固有名詞を追加することができ、それが学習時・音声合成時の読み取得部分に適応されます。辞書の追加・編集は次のエディタ経由で行ってください。または、手持ちのOpenJTalkのcsv形式の辞書がある場合は、`dict_data/default.csv`ファイルを直接上書きや追加しても可能です。 -辞書部分の[実装](/text/user_dict/) は、中のREADMEにある通り、[VOICEVOX Editor](https://github.com/VOICEVOX/voicevox) のものを使っており、この部分のコードライセンスはLGPL-3. +使えそうな辞書（ライセンス等は各自ご確認ください）（他に良いのがあったら教えて下さい）： + +- [WariHima/Kanayomi-dict](https://github.com/WariHima/KanaYomi-dict) +- [takana-v/tsumu_dic](https://github.com/takana-v/tsumu_dic) + + +辞書機能部分の[実装](/text/user_dict/) は、中のREADMEにある通り、[VOICEVOX Editor](https://github.com/VOICEVOX/voicevox) のものを使っており、この部分のコードライセンスはLGPL-3.0です。 #### 音声合成専用エディタ @@ -18,9 +24,27 @@ `Editor.bat`をダブルクリックか`python server_editor.py --inbrowser`で起動します。エディター部分は[こちらの別リポジトリ](https://github.com/litagin02/Style-Bert-VITS2-Editor)になります。フロントエンド初心者なのでプルリクや改善案等をお待ちしています。 +### バグ修正 + +- 「うっうあ」等の特定の状況で読みが正しく取得できないバグの修正 +- 前処理時に、書き起こしファイルのある行の形式が不正だと、書き起こしファイルのそれ以降の内容が消えてしまうバグの修正 +- faster-whisperが1.0.0にメジャーバージョンアップされ（今のところ）大幅に劣化したので、バージョンを0.10.1へ固定 + ### 改善 + +- テキスト前処理時に、読みの取得の失敗等があった場合に、処理を中断せず、エラーがおきた箇所を`text_error.log`ファイルへ保存するように変更。 +- 音声合成時に、読めない文字があったときはエラーを起こさず、その部分を無視して読み上げるように変更（学習段階ではエラーを出します） +- コマンドラインで前処理や学習が簡単にできるよう、前処理を行う`preprocess_all.py`を追加（詳しくは[CLI.md](/docs/CLI.md)を参照） +- 学習の際に、自動的に自分のhugging faceリポジトリへ結果をアップロードするオプションを追加。コマンドライン引数で`--repo_id username/my_model`のように指定してください（詳しくは[CLI.md](/docs/CLI.md)を参照）。🤗の無制限ストレージが使えるのでクラウドでの学習に便利です。 - 学習時にデコーダー部分を凍結するオプションの追加。品質がもしかしたら上がるかもしれません。 -- +- `initialize.py`に引数`--dataset_root`と`--assets_root`を追加し、`configs/paths.yml`を変更できるようにした + +### その他 + +- [paperspaceでの学習の手引きを追加](/docs/paperspace.md)、paperspaceでのimageに使える[Dockerfile](/Dockerfile.train)を追加 +- [CLIでの学習の仕方を追加](/docs/CLI.md) +- [Hugging Face spacesで遊べる音声合成エディタ](https://huggingface.co/spaces/litagin/Style-Bert-VITS2-Editor-Demo)をデプロイするための[Dockerfile](Dockerfile.deploy)を追加 + ## v2.2 (2024-02-09) diff --git a/text/tone_sandhi.py b/text/tone_sandhi.py index 3723086..38f3137 100644 --- a/text/tone_sandhi.py +++ b/text/tone_sandhi.py @@ -497,7 +497,10 @@ class ToneSandhi: # 个做量词 elif ( ge_idx >= 1 - and (word[ge_idx - 1].isnumeric() or word[ge_idx - 1] in "几有两半多各整每做是") + and ( + word[ge_idx - 1].isnumeric() + or word[ge_idx - 1] in "几有两半多各整每做是" + ) ) or word == "个": finals[ge_idx] = finals[ge_idx][:-1] + "5" else: diff --git a/text/user_dict/README.md b/text/user_dict/README.md index b086a4d..6f5618e 100644 --- a/text/user_dict/README.md +++ b/text/user_dict/README.md @@ -1,10 +1,12 @@ このフォルダに含まれるユーザー辞書関連のコードは、[VOICEVOX engine](https://github.com/VOICEVOX/voicevox_engine)プロジェクトのコードを改変したものを使用しています。VOICEVOXプロジェクトのチームに深く感謝し、その貢献を尊重します。 **元のコード**: -- [voicevox_engine/user_dict](https://github.com/VOICEVOX/voicevox_engine/tree/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/user_dict) + +- [voicevox_engine/user_dict/](https://github.com/VOICEVOX/voicevox_engine/tree/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/user_dict) - [voicevox_engine/model.py](https://github.com/VOICEVOX/voicevox_engine/blob/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/model.py#L207) **改変の詳細**: + - ファイル名の書き換えおよびそれに伴うimport文の書き換え。 - VOICEVOX固有の部分をコメントアウト。 - mutexを使用している部分をコメントアウト。 @@ -13,4 +15,5 @@ - Pydanticのモデルで必要な箇所のみを抽出。 **ライセンス**: + 元のVOICEVOX engineのリポジトリのコードは、LGPL v3 と、ソースコードの公開が不要な別ライセンスのデュアルライセンスの下で使用されています。当プロジェクトにおけるこのモジュールもLGPLライセンスの下にあります。詳細については、プロジェクトのルートディレクトリにある[LGPL_LICENSE](/LGPL_LICENSE)ファイルをご参照ください。また、元のVOICEVOX engineプロジェクトのライセンスについては、[こちら](https://github.com/VOICEVOX/voicevox_engine/blob/master/LICENSE)をご覧ください。 diff --git a/tools/translate.py b/tools/translate.py index 9368b5f..be0f7ea 100644 --- a/tools/translate.py +++ b/tools/translate.py @@ -1,6 +1,7 @@ """ 翻译api """ + from config import config import random diff --git a/update_status.py b/update_status.py index 65b4f93..7d768c6 100644 --- a/update_status.py +++ b/update_status.py @@ -38,7 +38,9 @@ def update_c_files(): c_files.append(os.path.join(root, file)) cnt += 1 print(c_files) - return f"更新模型列表完成, 共找到{cnt}个配置文件", gr.Dropdown.update(choices=c_files) + return f"更新模型列表完成, 共找到{cnt}个配置文件", gr.Dropdown.update( + choices=c_files + ) def update_model_folders(): @@ -50,7 +52,9 @@ def update_model_folders(): subdirs.append(os.path.join(root, dir_name)) cnt += 1 print(subdirs) - return f"更新模型文件夹列表完成, 共找到{cnt}个文件夹", gr.Dropdown.update(choices=subdirs) + return f"更新模型文件夹列表完成, 共找到{cnt}个文件夹", gr.Dropdown.update( + choices=subdirs + ) def update_wav_lab_pairs(): diff --git a/webui.py b/webui.py index e7c1abd..90318a1 100644 --- a/webui.py +++ b/webui.py @@ -1,6 +1,7 @@ """ Original `webui.py` for Bert-VITS2, not working with Style-Bert-VITS2 yet. """ + # flake8: noqa: E402 import os import logging From a6ae2bd15a9f7e69cbf375d44602fe0b6855d651 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Mon, 26 Feb 2024 01:49:06 +0900 Subject: [PATCH 50/51] Add text_normalize api --- server_editor.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/server_editor.py b/server_editor.py index f859315..7f5a6ef 100644 --- a/server_editor.py +++ b/server_editor.py @@ -42,7 +42,7 @@ from common.constants import ( ) from common.log import logger from common.tts_model import ModelHolder -from text.japanese import g2kata_tone, kata_tone2phone_tone +from text.japanese import g2kata_tone, kata_tone2phone_tone, text_normalize from text.user_dict import apply_word, update_dict, read_dict, rewrite_word, delete_word @@ -205,12 +205,12 @@ class MoraTone(BaseModel): tone: int -class G2PRequest(BaseModel): +class TextRequest(BaseModel): text: str @router.post("/g2p") -async def read_item(item: G2PRequest): +async def read_item(item: TextRequest): try: kata_tone_list = g2kata_tone(item.text, ignore_unknown=True) except Exception as e: @@ -221,6 +221,11 @@ async def read_item(item: G2PRequest): return [MoraTone(mora=kata, tone=tone) for kata, tone in kata_tone_list] +@router.post("/normalize") +async def normalize_text(item: TextRequest): + return text_normalize(item.text) + + @router.get("/models_info") def models_info(): return model_holder.models_info() From 8caa895a6ff4c71566f93b27895814c57db1c8cd Mon Sep 17 00:00:00 2001 From: litagin02 Date: Mon, 26 Feb 2024 15:10:00 +0900 Subject: [PATCH 51/51] Maybe v2.3 --- README.md | 29 +++++++++----- dict_data/default.csv | 4 +- docs/CHANGELOG.md | 28 ++++++++++++-- scripts/Update-Style-Bert-VITS2.bat | 5 ++- scripts/Update-to-Dict-Editor.bat | 19 +++++---- scripts/Update-to-JP-Extra.bat | 60 ----------------------------- server_editor.py | 4 +- 7 files changed, 65 insertions(+), 84 deletions(-) delete mode 100644 scripts/Update-to-JP-Extra.bat diff --git a/README.md b/README.md index b857e1d..db2043d 100644 --- a/README.md +++ b/README.md @@ -7,14 +7,15 @@ https://github.com/litagin02/Style-Bert-VITS2/assets/139731664/e853f9a2-db4a-420 - **解説チュートリアル動画** [YouTube](https://youtu.be/aTUSzgDl1iY)　[ニコニコ動画](https://www.nicovideo.jp/watch/sm43391524) - [English README](docs/README_en.md) - [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](http://colab.research.google.com/github/litagin02/Style-Bert-VITS2/blob/master/colab.ipynb) -- [🤗 オンラインデモはこちらから](https://huggingface.co/spaces/litagin/Style-Bert-VITS2-JVNV) +- [🤗 オンラインデモはこちらから](https://huggingface.co/spaces/litagin/Style-Bert-VITS2-Editor-Demo) - [Zennの解説記事](https://zenn.dev/litagin/articles/034819a5256ff4) -- [**リリースページ**](https://github.com/litagin02/Style-Bert-VITS2/releases/)、[更新履歴](docs/CHANGELOG.md) +- [**リリースページ**](https://github.com/litagin02/Style-Bert-VITS2/releases/)、[更新履歴](/docs/CHANGELOG.md) + - 2024-02-26: ver 2.3 (辞書機能とエディター機能) - 2024-02-09: ver 2.2 - 2024-02-07: ver 2.1 - - 2024-02-03: ver 2.0 + - 2024-02-03: ver 2.0 (JP-Extra) - 2024-01-09: ver 1.3 - 2023-12-31: ver 1.2 - 2023-12-29: ver 1.1 @@ -33,7 +34,7 @@ This repository is based on [Bert-VITS2](https://github.com/fishaudio/Bert-VITS2 ## 使い方 - +CLIでの使い方は[こちら](/docs/CLI.md)を参照してください。 ### 動作環境 @@ -45,13 +46,14 @@ This repository is based on [Bert-VITS2](https://github.com/fishaudio/Bert-VITS2 Windowsを前提としています。 -1. [このzipファイル](https://github.com/litagin02/Style-Bert-VITS2/releases/download/2.2/Style-Bert-VITS2.zip)を**パスに日本語や空白が含まれない場所に**ダウンロードして展開します。 +1. [このzipファイル](https://github.com/litagin02/Style-Bert-VITS2/releases/download/2.3/Style-Bert-VITS2.zip)を**パスに日本語や空白が含まれない場所に**ダウンロードして展開します。 - グラボがある方は、`Install-Style-Bert-VITS2.bat`をダブルクリックします。 - グラボがない方は、`Install-Style-Bert-VITS2-CPU.bat`をダブルクリックします。CPU版では学習はできませんが、音声合成とマージは可能です。 2. 待つと自動で必要な環境がインストールされます。 -3. その後、自動的に音声合成するためのWebUIが起動したらインストール成功です。デフォルトのモデルがダウンロードされるているので、そのまま遊ぶことができます。 +3. その後、自動的に音声合成するためのエディターが起動したらインストール成功です。デフォルトのモデルがダウンロードされるているので、そのまま遊ぶことができます。 -またアップデートをしたい場合は、`Update-Style-Bert-VITS2.bat`をダブルクリックしてください。ただし**1.x**から**2.x**へアップデートする場合は、[このbatファイル](https://github.com/litagin02/Style-Bert-VITS2/releases/download/2.2/Update-to-JP-Extra.bat)を`Style-Bert-VITS2`フォルダがあるフォルダ（`Update-Style-Bert-VITS2.bat`等があるフォルダ）へ保存してからダブルクリックしてください。 +またアップデートをしたい場合は、`Update-Style-Bert-VITS2.bat`をダブルクリックしてください。ただし以下の場合は、専用のアップデートbatファイルを`Style-Bert-VITS2`フォルダがあるフォルダ（`Update-Style-Bert-VITS2.bat`等があるフォルダ）へ保存してからダブルクリックしてください。 +- **2.3以前**から**2.3以上**（辞書・エディター付き）へアップデート: [Update-to-Dict-Editor.bat](https://github.com/litagin02/Style-Bert-VITS2/releases/download/2.3/Update-to-Dict-Editor.bat) #### GitやPython使える人 @@ -69,7 +71,12 @@ python initialize.py # 必要なモデルとデフォルトTTSモデルをダ ### 音声合成 -`App.bat`をダブルクリックか、`python app.py`するとWebUIが起動します（`python app.py --cpu`でCPUモードで起動、学習中チェックに便利です）。インストール時にデフォルトのモデルがダウンロードされているので、学習していなくてもそれを使うことができます。 +音声合成エディターは`Editor.bat`をダブルクリックか、`python server_editor.py --inbrowser`すると起動します（`--device cpu`でCPUモードで起動）。画面内で各セリフごとに設定を変えて原稿を作ったり、保存や読み込みや辞書の編集等ができます。 +インストール時にデフォルトのモデルがダウンロードされているので、学習していなくてもそれを使うことができます。 + +エディター部分は[別リポジトリ](https://github.com/litagin02/Style-Bert-VITS2-Editor)に分かれています。 + +バージョン2.2以前での音声合成WebUIは、`App.bat`をダブルクリックか、`python app.py`するとWebUIが起動します。 音声合成に必要なモデルファイルたちの構造は以下の通りです（手動で配置する必要はありません）。 ``` @@ -91,7 +98,7 @@ model_assets ### 学習 - CLIでの学習の詳細は[こちら](docs/CLI.md)を参照してください。 -- paperspace上での学習の詳細は[こちら](docs/paperspace.md)を参照してください。 +- paperspace上での学習の詳細は[こちら](docs/paperspace.md)、colabでの学習は[こちら](http://colab.research.google.com/github/litagin02/Style-Bert-VITS2/blob/master/colab.ipynb)を参照してください。 学習には2-14秒程度の音声ファイルが複数と、それらの書き起こしデータが必要です。 @@ -126,6 +133,10 @@ API仕様は起動後に`/docs`にて確認ください。 - 入力文字数はデフォルトで100文字が上限となっています。これは`config.yml`の`server.limit`で変更できます。 - デフォルトではCORS設定を全てのドメインで許可しています。できる限り、`config.yml`の`server.origins`の値を変更し、信頼できるドメインに制限ください(キーを消せばCORS設定を無効にできます)。 +また音声合成エディターのAPIサーバーは`python server_editor.py`で起動します。があまりまだ整備をしていません。[エディターのリポジトリ](https://github.com/litagin02/Style-Bert-VITS2-Editor)から必要な最低限のAPIしか現在は実装していません。 + +音声合成エディターのウェブデプロイについては[このDockerfile](Dockerfile.deploy)を参考にしてください。 + ### マージ 2つのモデルを、「声質」「声の高さ」「感情表現」「テンポ」の4点で混ぜ合わせて、新しいモデルを作ることが出来ます。 diff --git a/dict_data/default.csv b/dict_data/default.csv index 8869f97..84085cd 100644 --- a/dict_data/default.csv +++ b/dict_data/default.csv @@ -1,3 +1,5 @@ Ｂｅｒｔ,,,8609,名詞,固有名詞,一般,*,*,*,Ｂｅｒｔ,バアト,バアト,0/3,* ＶＩＴＳ,,,8609,名詞,固有名詞,一般,*,*,*,ＶＩＴＳ,ビッツ,ビッツ,0/3,* -ＶＩＴＳ２,,,8609,名詞,固有名詞,一般,*,*,*,ＶＩＴＳ２,ビッツツウ,ビッツツウ,4/5,* +ＶＩＴＳ二,,,8609,名詞,固有名詞,一般,*,*,*,ＶＩＴＳ二,ビッツツー,ビッツツー,4/5,* +ＢｅｒｔＶＩＴＳ,,,8609,名詞,固有名詞,一般,*,*,*,ＢｅｒｔＶＩＴＳ,バアトビッツ,バアトビッツ,4/6,* +担々麺,,,8609,名詞,固有名詞,一般,*,*,*,担々麺,タンタンメン,タンタンメン,3/6,* diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index bbf1159..15e0ef9 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -4,8 +4,10 @@ ### 大きな変更 +大きい変更をいくつかしたため、**アップデートはまた専用の手順**が必要です。下記の指示にしたがってください。 + #### ユーザー辞書機能 -あらかじめ辞書に固有名詞を追加することができ、それが学習時・音声合成時の読み取得部分に適応されます。辞書の追加・編集は次のエディタ経由で行ってください。または、手持ちのOpenJTalkのcsv形式の辞書がある場合は、`dict_data/default.csv`ファイルを直接上書きや追加しても可能です。 +あらかじめ辞書に固有名詞を追加することができ、それが**学習時**・**音声合成時**の読み取得部分に適応されます。辞書の追加・編集は次のエディタ経由で行ってください。または、手持ちのOpenJTalkのcsv形式の辞書がある場合は、`dict_data/default.csv`ファイルを直接上書きや追加しても可能です。 使えそうな辞書（ライセンス等は各自ご確認ください）（他に良いのがあったら教えて下さい）： @@ -26,7 +28,7 @@ ### バグ修正 -- 「うっうあ」等の特定の状況で読みが正しく取得できないバグの修正 +- 特定の状況で読みが正しく取得できず `list index out of range` となるバグの修正 - 前処理時に、書き起こしファイルのある行の形式が不正だと、書き起こしファイルのそれ以降の内容が消えてしまうバグの修正 - faster-whisperが1.0.0にメジャーバージョンアップされ（今のところ）大幅に劣化したので、バージョンを0.10.1へ固定 @@ -37,14 +39,32 @@ - コマンドラインで前処理や学習が簡単にできるよう、前処理を行う`preprocess_all.py`を追加（詳しくは[CLI.md](/docs/CLI.md)を参照） - 学習の際に、自動的に自分のhugging faceリポジトリへ結果をアップロードするオプションを追加。コマンドライン引数で`--repo_id username/my_model`のように指定してください（詳しくは[CLI.md](/docs/CLI.md)を参照）。🤗の無制限ストレージが使えるのでクラウドでの学習に便利です。 - 学習時にデコーダー部分を凍結するオプションの追加。品質がもしかしたら上がるかもしれません。 -- `initialize.py`に引数`--dataset_root`と`--assets_root`を追加し、`configs/paths.yml`を変更できるようにした +- `initialize.py`に引数`--dataset_root`と`--assets_root`を追加し、`configs/paths.yml`をその時点で変更できるようにした ### その他 - [paperspaceでの学習の手引きを追加](/docs/paperspace.md)、paperspaceでのimageに使える[Dockerfile](/Dockerfile.train)を追加 -- [CLIでの学習の仕方を追加](/docs/CLI.md) +- [CLIでの各種処理の実行の仕方を追加](/docs/CLI.md) - [Hugging Face spacesで遊べる音声合成エディタ](https://huggingface.co/spaces/litagin/Style-Bert-VITS2-Editor-Demo)をデプロイするための[Dockerfile](Dockerfile.deploy)を追加 +### アップデート手順 + +- [Update-to-Dict-Editor.bat](https://github.com/litagin02/Style-Bert-VITS2/releases/download/2.3/Update-to-Dict-Editor.bat)をダウンロードし、`Style-Bert-VITS2`フォルダがある場所（インストールbatファイルとかがあったところ）においてダブルクリックしてください。 + +- 手動での場合は、以下の手順で実行してください： +```bash +git pull +venv\Scripts\activate +pip uninstall pyopenjtalk-prebuilt +pip install -r requirements.txt +# python initialize.py # これを1.x系からのアップデートの場合は実行してください +python server_editor.py --inbrowser +``` + +### 新規インストール手順 +[このzip](https://github.com/litagin02/Style-Bert-VITS2/releases/download/2.3/Style-Bert-VITS2.zip)をダウンロードし、解凍してください。 +を展開し、`Install-Style-Bert-VITS2.bat`をダブルクリックしてください。 + ## v2.2 (2024-02-09) diff --git a/scripts/Update-Style-Bert-VITS2.bat b/scripts/Update-Style-Bert-VITS2.bat index 782f2ef..b4f8b8d 100644 --- a/scripts/Update-Style-Bert-VITS2.bat +++ b/scripts/Update-Style-Bert-VITS2.bat @@ -37,7 +37,10 @@ if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) pip install -U -r Style-Bert-VITS2\requirements.txt if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) -echo Style-Bert-VITS2のアップデートが完了しました。 +echo Update completed. Running Style-Bert-VITS2 Editor... + +@REM Style-Bert-VITS2 Editorを起動 +python server_editor.py --inbrowser pause diff --git a/scripts/Update-to-Dict-Editor.bat b/scripts/Update-to-Dict-Editor.bat index 5fde8fc..1eff1bd 100644 --- a/scripts/Update-to-Dict-Editor.bat +++ b/scripts/Update-to-Dict-Editor.bat @@ -12,7 +12,7 @@ if not exist %CURL_CMD% ( @REM Style-Bert-VITS2.zip をGitHubのmasterの最新のものをダウンロード %CURL_CMD% -Lo Style-Bert-VITS2.zip^ - https://github.com/litagin02/Style-Bert-VITS2/archive/refs/heads/dev.zip + https://github.com/litagin02/Style-Bert-VITS2/archive/refs/heads/master.zip if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) @REM Style-Bert-VITS2.zip を解凍（フォルダ名前がBert-VITS2-masterになる） @@ -24,8 +24,8 @@ del Style-Bert-VITS2.zip if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) @REM Bert-VITS2-masterの中身をStyle-Bert-VITS2に上書き移動 -xcopy /QSY .\Style-Bert-VITS2-dev\ .\Style-Bert-VITS2\ -rmdir /s /q Style-Bert-VITS2-dev +xcopy /QSY .\Style-Bert-VITS2-master\ .\Style-Bert-VITS2\ +rmdir /s /q Style-Bert-VITS2-master if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) @REM 仮想環境を有効化 @@ -49,16 +49,19 @@ if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) pushd Style-Bert-VITS2 +@REM JP-Extra版以前からの場合のために一応initialize.pyを実行 + +echo python initialize.py +python initialize.py +if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) + +echo ---------------------------------------- echo Update completed. Running Style-Bert-VITS2 Editor... +echo ---------------------------------------- @REM Style-Bert-VITS2 Editorを起動 python server_editor.py --inbrowser -pause - -popd - - pause popd diff --git a/scripts/Update-to-JP-Extra.bat b/scripts/Update-to-JP-Extra.bat deleted file mode 100644 index 228cf10..0000000 --- a/scripts/Update-to-JP-Extra.bat +++ /dev/null @@ -1,60 +0,0 @@ -chcp 65001 > NUL -@echo off - -pushd %~dp0 -set PS_CMD=PowerShell -Version 5.1 -ExecutionPolicy Bypass - -set CURL_CMD=C:\Windows\System32\curl.exe -if not exist %CURL_CMD% ( - echo [ERROR] %CURL_CMD% が見つかりません。 - pause & popd & exit /b 1 -) - -@REM Style-Bert-VITS2.zip をGitHubのmasterの最新のものをダウンロード -%CURL_CMD% -Lo Style-Bert-VITS2.zip^ - https://github.com/litagin02/Style-Bert-VITS2/archive/refs/heads/master.zip -if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) - -@REM Style-Bert-VITS2.zip を解凍（フォルダ名前がBert-VITS2-masterになる） -%PS_CMD% Expand-Archive -Path Style-Bert-VITS2.zip -DestinationPath . -Force -if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) - -@REM 元のzipを削除 -del Style-Bert-VITS2.zip -if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) - -@REM Bert-VITS2-masterの中身をStyle-Bert-VITS2に上書き移動 -xcopy /QSY .\Style-Bert-VITS2-master\ .\Style-Bert-VITS2\ -rmdir /s /q Style-Bert-VITS2-master -if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) - -@REM 仮想環境のpip requirements.txtを更新 - -echo call .\Style-Bert-VITS2\scripts\activate.bat -call .\Style-Bert-VITS2\venv\Scripts\activate.bat -if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) - -echo pip uninstall -y pyopenjtalk-prebuilt pyopenjtalk -pip uninstall -y pyopenjtalk-prebuilt pyopenjtalk -if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) - -echo pip install -U pyopenjtalk-dict -pip install -U pyopenjtalk-dict -if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) - -echo pip install -U -r Style-Bert-VITS2\requirements.txt -pip install -U -r Style-Bert-VITS2\requirements.txt -if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% ) - -pushd Style-Bert-VITS2 - -@REM 初期化（必要なモデルのダウンロード） -python initialize.py - -echo Style-Bert-VITS2の2.xへのアップデートが完了しました。 - -pause - -popd - -popd \ No newline at end of file diff --git a/server_editor.py b/server_editor.py index 7f5a6ef..93c7c58 100644 --- a/server_editor.py +++ b/server_editor.py @@ -212,7 +212,9 @@ class TextRequest(BaseModel): @router.post("/g2p") async def read_item(item: TextRequest): try: - kata_tone_list = g2kata_tone(item.text, ignore_unknown=True) + # 最初に正規化しないと整合性がとれない + text = text_normalize(item.text) + kata_tone_list = g2kata_tone(text, ignore_unknown=True) except Exception as e: raise HTTPException( status_code=400,