From c81cc7b06fd74a59bc442f02482e9249ca4633b4 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Tue, 6 Feb 2024 10:21:06 +0900 Subject: [PATCH 01/17] Add experimental speedup option for training (disable log and eval) --- train_ms.py | 21 ++++++++++++++++----- train_ms_jp_extra.py | 21 ++++++++++++++++----- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/train_ms.py b/train_ms.py index ca18667..c48216c 100644 --- a/train_ms.py +++ b/train_ms.py @@ -79,6 +79,11 @@ def run(): action="store_true", help="Do not show the progress bar while training.", ) + parser.add_argument( + "--speedup", + action="store_true", + help="Speed up training by disabling logging and evaluation.", + ) args = parser.parse_args() # Set log file @@ -115,8 +120,9 @@ def run(): n_gpus = dist.get_world_size() hps = utils.get_hparams_from_file(args.config) - # This is needed because we have to pass `model_dir` to `train_and_evaluate()` + # This is needed because we have to pass values to `train_and_evaluate()` hps.model_dir = model_dir + hps.speedup = args.speedup # 比较路径是否相同 if os.path.realpath(args.config) != os.path.realpath( @@ -161,7 +167,9 @@ def run(): torch.cuda.set_device(local_rank) global global_step - if rank == 0: + writer = None + writer_eval = None + if rank == 0 and not args.speedup: # logger = utils.get_logger(hps.model_dir) # logger.info(hps) utils.check_git_hash(model_dir) @@ -189,7 +197,9 @@ def run(): persistent_workers=True, prefetch_factor=4, ) # DataLoader config could be adjusted. - if rank == 0: + eval_dataset = None + eval_loader = None + if rank == 0 and not args.speedup: eval_dataset = TextAudioSpeakerLoader(hps.data.validation_files, hps.data) eval_loader = DataLoader( eval_dataset, @@ -647,7 +657,7 @@ def train_and_evaluate( scaler.update() if rank == 0: - if global_step % hps.train.log_interval == 0: + if global_step % hps.train.log_interval == 0 and not hps.speedup: lr = optim_g.param_groups[0]["lr"] losses = [loss_disc, loss_gen, loss_fm, loss_mel, loss_dur, loss_kl] # logger.info( @@ -708,7 +718,8 @@ def train_and_evaluate( and global_step != 0 and initial_step != global_step ): - evaluate(hps, net_g, eval_loader, writer_eval) + if not hps.speedup: + evaluate(hps, net_g, eval_loader, writer_eval) utils.save_checkpoint( net_g, optim_g, diff --git a/train_ms_jp_extra.py b/train_ms_jp_extra.py index a689fc9..cdb09f3 100644 --- a/train_ms_jp_extra.py +++ b/train_ms_jp_extra.py @@ -82,6 +82,11 @@ def run(): action="store_true", help="Do not show the progress bar while training.", ) + parser.add_argument( + "--speedup", + action="store_true", + help="Speed up training by disabling logging and evaluation.", + ) args = parser.parse_args() # Set log file @@ -118,8 +123,9 @@ def run(): n_gpus = dist.get_world_size() hps = utils.get_hparams_from_file(args.config) - # This is needed because we have to pass `model_dir` to `train_and_evaluate()` + # This is needed because we have to pass values to `train_and_evaluate() hps.model_dir = model_dir + hps.speedup = args.speedup # 比较路径是否相同 if os.path.realpath(args.config) != os.path.realpath( @@ -164,7 +170,9 @@ def run(): torch.cuda.set_device(local_rank) global global_step - if rank == 0: + writer = None + writer_eval = None + if rank == 0 and not args.speedup: # logger = utils.get_logger(hps.model_dir) # logger.info(hps) utils.check_git_hash(model_dir) @@ -192,7 +200,9 @@ def run(): persistent_workers=True, prefetch_factor=6, ) # DataLoader config could be adjusted. - if rank == 0: + eval_dataset = None + eval_loader = None + if rank == 0 and not args.speedup: eval_dataset = TextAudioSpeakerLoader(hps.data.validation_files, hps.data) eval_loader = DataLoader( eval_dataset, @@ -748,7 +758,7 @@ def train_and_evaluate( scaler.update() if rank == 0: - if global_step % hps.train.log_interval == 0: + if global_step % hps.train.log_interval == 0 and not hps.speedup: lr = optim_g.param_groups[0]["lr"] losses = [loss_disc, loss_gen, loss_fm, loss_mel, loss_dur, loss_kl] # logger.info( @@ -842,7 +852,8 @@ def train_and_evaluate( and global_step != 0 and initial_step != global_step ): - evaluate(hps, net_g, eval_loader, writer_eval) + if not hps.speedup: + evaluate(hps, net_g, eval_loader, writer_eval) utils.save_checkpoint( net_g, optim_g, From cae019d401ef4ca2577ff987caa3c468d6cd031c Mon Sep 17 00:00:00 2001 From: litagin02 Date: Tue, 6 Feb 2024 20:42:43 +0900 Subject: [PATCH 02/17] Set default bf16 false (divergence bug?) --- webui_dataset.py | 18 ++++++++++++++---- webui_train.py | 25 ++++++++++++++++--------- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/webui_dataset.py b/webui_dataset.py index ca50d1b..5b25afe 100644 --- a/webui_dataset.py +++ b/webui_dataset.py @@ -52,7 +52,7 @@ def do_transcribe( return "Error: モデル名を入力してください。" if initial_prompt == "": initial_prompt = "こんにちは。元気、ですかー？私は……ふふっ、ちゃんと元気だよ！" - logger.debug(f"initial_prompt: {initial_prompt}") + # logger.debug(f"initial_prompt: {initial_prompt}") if input_dir == "": input_dir = os.path.join(dataset_root, model_name, "raw") output_file = os.path.join(dataset_root, model_name, "esd.list") @@ -117,7 +117,9 @@ Style-Bert-VITS2の学習用データセットを作成するためのツール with gr.Blocks(theme="NoCrypt/miku") as app: gr.Markdown(initial_md) - model_name = gr.Textbox(label="モデル名を入力してください（話者名としても使われます）。") + model_name = gr.Textbox( + label="モデル名を入力してください（話者名としても使われます）。" + ) with gr.Accordion("音声のスライス"): with gr.Row(): with gr.Column(): @@ -127,10 +129,18 @@ with gr.Blocks(theme="NoCrypt/miku") as app: info="下記フォルダにwavファイルを入れておいてください", ) min_sec = gr.Slider( - minimum=0, maximum=10, value=2, step=0.5, label="この秒数未満は切り捨てる" + minimum=0, + maximum=10, + value=2, + step=0.5, + label="この秒数未満は切り捨てる", ) max_sec = gr.Slider( - minimum=0, maximum=15, value=12, step=0.5, label="この秒数以上は切り捨てる" + minimum=0, + maximum=15, + value=12, + step=0.5, + label="この秒数以上は切り捨てる", ) min_silence_dur_ms = gr.Slider( minimum=0, diff --git a/webui_train.py b/webui_train.py index 28f1233..b278cde 100644 --- a/webui_train.py +++ b/webui_train.py @@ -288,7 +288,7 @@ def preprocess_all( ) -def train(model_name, skip_style=False, use_jp_extra=True): +def train(model_name, skip_style=False, use_jp_extra=True, speedup=False): dataset_path, _, _, _, config_path = get_path(model_name) # 学習再開の場合は念のためconfig.ymlの名前等を更新 with open("config.yml", "r", encoding="utf-8") as f: @@ -302,6 +302,8 @@ def train(model_name, skip_style=False, use_jp_extra=True): cmd = [train_py, "--config", config_path, "--model", dataset_path] if skip_style: cmd.append("--skip_default_style") + if speedup: + cmd.append("--speedup") success, message = run_script_with_log(cmd, ignore_warning=True) if not success: logger.error(f"Train failed.") @@ -401,11 +403,6 @@ if __name__ == "__main__": maximum=10000, step=100, ) - bf16_run = gr.Checkbox( - label="bfloat16を使う", - info="bfloat16を使うかどうか。新しめのグラボだと学習が早くなるかも、古いグラボだと動かないかも。", - value=True, - ) normalize = gr.Checkbox( label="音声の音量を正規化する(音量の大小が揃っていない場合など)", value=False, @@ -415,9 +412,14 @@ if __name__ == "__main__": value=False, ) with gr.Accordion("詳細設定", open=False): + bf16_run = gr.Checkbox( + label="bfloat16を使う", + info="bfloat16を使うかどうか。オンにすると学習が発散する可能性がありますが、メモリ使用量が減る可能性があります。", + value=False, + ) num_processes = gr.Slider( label="プロセス数", - info="前処理時の並列処理プロセス数、大きすぎるとフリーズするかも", + info="前処理時の並列処理プロセス数、前処理でフリーズしたら下げてください", value=cpu_count() // 2, minimum=1, maximum=cpu_count(), @@ -477,7 +479,7 @@ if __name__ == "__main__": ) bf16_run_manual = gr.Checkbox( label="bfloat16を使う", - value=True, + value=False, ) freeze_EN_bert_manual = gr.Checkbox( label="英語bert部分を凍結", @@ -555,6 +557,11 @@ if __name__ == "__main__": label="JP-Extra版を使う", value=True, ) + speedup = gr.Checkbox( + label="ログ等をスキップして学習を高速化する", + value=False, + visible=False, # Experimental + ) train_btn = gr.Button(value="学習を開始する", variant="primary") info_train = gr.Textbox(label="状況") @@ -624,7 +631,7 @@ if __name__ == "__main__": # Train train_btn.click( second_elem_of(train), - inputs=[model_name, skip_style, use_jp_extra_train], + inputs=[model_name, skip_style, use_jp_extra_train, speedup], outputs=[info_train], ) use_jp_extra.change( From a0d4444db7979c799a95a040ce2ea23232793bd4 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Wed, 7 Feb 2024 00:12:38 +0900 Subject: [PATCH 03/17] Try to reduce memory consumption (experimental) --- train_ms_jp_extra.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/train_ms_jp_extra.py b/train_ms_jp_extra.py index cdb09f3..ddb18b2 100644 --- a/train_ms_jp_extra.py +++ b/train_ms_jp_extra.py @@ -192,13 +192,13 @@ def run(): train_dataset, # num_workers=min(config.train_ms_config.num_workers, os.cpu_count() - 1), # Slow and often freezes, so use only half of the cores. - num_workers=min(config.train_ms_config.num_workers, os.cpu_count() // 2), + # num_workers=min(config.train_ms_config.num_workers, os.cpu_count() // 2), shuffle=False, pin_memory=True, collate_fn=collate_fn, batch_sampler=train_sampler, persistent_workers=True, - prefetch_factor=6, + # prefetch_factor=6, ) # DataLoader config could be adjusted. eval_dataset = None eval_loader = None @@ -307,16 +307,28 @@ def run(): ) else: optim_wd = None - net_g = DDP(net_g, device_ids=[local_rank], bucket_cap_mb=512) - net_d = DDP(net_d, device_ids=[local_rank], bucket_cap_mb=512) + net_g = DDP( + net_g, + device_ids=[local_rank], + # bucket_cap_mb=512 + ) + net_d = DDP( + net_d, + device_ids=[local_rank], + # bucket_cap_mb=512 + ) if net_dur_disc is not None: net_dur_disc = DDP( net_dur_disc, device_ids=[local_rank], - bucket_cap_mb=512, + # bucket_cap_mb=512, ) if net_wd is not None: - net_wd = DDP(net_wd, device_ids=[local_rank], bucket_cap_mb=512) + net_wd = DDP( + net_wd, + device_ids=[local_rank], + # bucket_cap_mb=512 + ) if utils.is_resuming(model_dir): if net_dur_disc is not None: From cb0002941cc5717787b61b88122fb7961d7b58ab Mon Sep 17 00:00:00 2001 From: litagin02 Date: Wed, 7 Feb 2024 10:20:15 +0900 Subject: [PATCH 04/17] Feat: save merge recipe --- webui_merge.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/webui_merge.py b/webui_merge.py index 2f35f73..7820651 100644 --- a/webui_merge.py +++ b/webui_merge.py @@ -87,6 +87,17 @@ def merge_style(model_name_a, model_name_b, weight, output_name, style_triple_li ) as f: json.dump(new_config, f, indent=2, ensure_ascii=False) + # info.jsonを読み込んで、style_triple_listを追記 + info_path = os.path.join(assets_root, output_name, "info.json") + if os.path.exists(info_path): + with open(info_path, encoding="utf-8") as f: + info = json.load(f) + else: + info = {} + info["style_triple_list"] = style_triple_list + with open(info_path, "w", encoding="utf-8") as f: + json.dump(info, f, indent=2, ensure_ascii=False) + return output_style_path, list(new_style2id.keys()) @@ -133,6 +144,19 @@ def merge_models( ) os.makedirs(os.path.dirname(merged_model_path), exist_ok=True) save_file(merged_model_weight, merged_model_path) + + info = { + "model_a": model_path_a, + "model_b": model_path_b, + "voice_weight": voice_weight, + "voice_pitch_weight": voice_pitch_weight, + "speech_style_weight": speech_style_weight, + "tempo_weight": tempo_weight, + } + with open( + os.path.join(assets_root, output_name, "recipe.json"), "w", encoding="utf-8" + ) as f: + json.dump(info, f, indent=2, ensure_ascii=False) return merged_model_path @@ -235,6 +259,8 @@ initial_md = """ 以上でマージは完了で、`model_assets/マージ後のモデル名`にマージ後のモデルが保存され、音声合成のときに使えます。 +また`model_asses/マージ後のモデル名/info.json`には、マージ前のモデル名やマージの重みが記録されます（推論にはいらないので配合メモ用です）。 + 一番下にマージしたモデルによる簡易的な音声合成機能もつけています。 ## 注意 From bf883789bd929f91fe4642ee035107b0f7678a3f Mon Sep 17 00:00:00 2001 From: litagin02 Date: Wed, 7 Feb 2024 10:20:58 +0900 Subject: [PATCH 05/17] Fix filename --- webui_merge.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/webui_merge.py b/webui_merge.py index 7820651..96fc366 100644 --- a/webui_merge.py +++ b/webui_merge.py @@ -87,8 +87,8 @@ def merge_style(model_name_a, model_name_b, weight, output_name, style_triple_li ) as f: json.dump(new_config, f, indent=2, ensure_ascii=False) - # info.jsonを読み込んで、style_triple_listを追記 - info_path = os.path.join(assets_root, output_name, "info.json") + # recipe.jsonを読み込んで、style_triple_listを追記 + info_path = os.path.join(assets_root, output_name, "recipe.json") if os.path.exists(info_path): with open(info_path, encoding="utf-8") as f: info = json.load(f) @@ -259,7 +259,7 @@ initial_md = """ 以上でマージは完了で、`model_assets/マージ後のモデル名`にマージ後のモデルが保存され、音声合成のときに使えます。 -また`model_asses/マージ後のモデル名/info.json`には、マージ前のモデル名やマージの重みが記録されます（推論にはいらないので配合メモ用です）。 +また`model_asses/マージ後のモデル名/recipe.json`には、マージの配合レシピが記録されます（推論にはいらないので配合メモ用です）。 一番下にマージしたモデルによる簡易的な音声合成機能もつけています。 From 0d7de644cd3276c39970f5cf54f55e9c1fd27051 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Wed, 7 Feb 2024 10:29:49 +0900 Subject: [PATCH 06/17] Update comment --- train_ms.py | 8 ++++---- train_ms_jp_extra.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/train_ms.py b/train_ms.py index c48216c..3e61bf2 100644 --- a/train_ms.py +++ b/train_ms.py @@ -187,15 +187,15 @@ def run(): collate_fn = TextAudioSpeakerCollate() train_loader = DataLoader( train_dataset, - # num_workers=min(config.train_ms_config.num_workers, os.cpu_count() - 1), - # Slow and often freezes, so use only half of the cores. - num_workers=min(config.train_ms_config.num_workers, os.cpu_count() // 2), + # メモリ消費量を減らそうとnum_workersを0にしてみる + # num_workers=min(config.train_ms_config.num_workers, os.cpu_count() // 2), shuffle=False, pin_memory=True, collate_fn=collate_fn, batch_sampler=train_sampler, persistent_workers=True, - prefetch_factor=4, + # これもメモリ消費量を減らそうとしてコメントアウト（事前ロード量？） + # prefetch_factor=4, ) # DataLoader config could be adjusted. eval_dataset = None eval_loader = None diff --git a/train_ms_jp_extra.py b/train_ms_jp_extra.py index ddb18b2..ce44ebe 100644 --- a/train_ms_jp_extra.py +++ b/train_ms_jp_extra.py @@ -190,14 +190,14 @@ def run(): collate_fn = TextAudioSpeakerCollate(use_jp_extra=True) train_loader = DataLoader( train_dataset, - # num_workers=min(config.train_ms_config.num_workers, os.cpu_count() - 1), - # Slow and often freezes, so use only half of the cores. + # メモリ消費量を減らそうとnum_workersを0にしてみる # num_workers=min(config.train_ms_config.num_workers, os.cpu_count() // 2), shuffle=False, pin_memory=True, collate_fn=collate_fn, batch_sampler=train_sampler, persistent_workers=True, + # これもメモリ消費量を減らそうとしてコメントアウト（事前ロード量？） # prefetch_factor=6, ) # DataLoader config could be adjusted. eval_dataset = None From 27518345dd52eccac88fd64dbeffca880027d5f7 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Wed, 7 Feb 2024 10:36:36 +0900 Subject: [PATCH 07/17] Clean --- .gitignore | 2 + safetensors.ipynb | 115 ---------------------------------------------- 2 files changed, 2 insertions(+), 115 deletions(-) delete mode 100644 safetensors.ipynb diff --git a/.gitignore b/.gitignore index 85c0cf2..a1763c9 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,5 @@ venv/ *.csv *.bak /mos_results/ + +safetensors.ipynb diff --git a/safetensors.ipynb b/safetensors.ipynb deleted file mode 100644 index 6722654..0000000 --- a/safetensors.ipynb +++ /dev/null @@ -1,115 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "\n", - "model_dir = \"pretrained-Japanese-Extra\"\n", - "step = 0\n", - "\n", - "g_model = torch.load(f\"{model_dir}/G_{step}.pth\", map_location=\"cpu\")\n", - "d_model = torch.load(f\"{model_dir}/D_{step}.pth\", map_location=\"cpu\")\n", - "wd_model = torch.load(f\"{model_dir}/WD_{step}.pth\", map_location=\"cpu\")" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "skip enc_p.in_feature_net.0.weight\n", - "skip enc_p.in_feature_net.2.weight\n", - "skip enc_p.in_feature_net.2.bias\n", - "skip enc_p.in_feature_net.3.norm.weight\n", - "skip enc_p.in_feature_net.3.norm.bias\n", - "skip enc_p.in_feature_net.3.mlp.c_fc1.weight\n", - "skip enc_p.in_feature_net.3.mlp.c_fc2.weight\n", - "skip enc_p.in_feature_net.3.mlp.c_proj.weight\n", - "skip enc_p.in_feature_net.4.weight\n", - "skip enc_p.emo_vq._codebook.initted\n", - "skip enc_p.emo_vq._codebook.cluster_size\n", - "skip enc_p.emo_vq._codebook.embed_avg\n", - "skip enc_p.emo_vq._codebook.embed\n", - "skip enc_p.out_feature_net.weight\n", - "skip enc_p.out_feature_net.bias\n" - ] - } - ], - "source": [ - "g_dict = {}\n", - "skip_list = [\"enc_p.in_feature_net.\", \"enc_p.emo_vq.\", \"enc_p.out_feature_net.\"]\n", - "for key in g_model[\"model\"].keys():\n", - " if any([key.startswith(s) for s in skip_list]):\n", - " print(f\"skip {key}\")\n", - " continue\n", - " else:\n", - " g_dict[key] = g_model[\"model\"][key]" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "d_dict = {}\n", - "for key in d_model[\"model\"].keys():\n", - " d_dict[key] = d_model[\"model\"][key]" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "wd_dict = {}\n", - "for key in wd_model[\"model\"].keys():\n", - " wd_dict[key] = wd_model[\"model\"][key]" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "from safetensors.torch import save_file\n", - "\n", - "\n", - "save_file(g_dict, f\"G_0.safetensors\")\n", - "save_file (d_dict, f\"D_0.safetensors\")\n", - "save_file (wd_dict, f\"WD_0.safetensors\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.11" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From 849a64811a795489ded43da4ac0aeff7d1c8a9ed Mon Sep 17 00:00:00 2001 From: litagin02 Date: Wed, 7 Feb 2024 10:44:17 +0900 Subject: [PATCH 08/17] Refactor: move code to main block --- initialize.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/initialize.py b/initialize.py index 6dfd03d..c163ef9 100644 --- a/initialize.py +++ b/initialize.py @@ -90,17 +90,18 @@ def download_jvnv_models(): ) -parser = argparse.ArgumentParser() -parser.add_argument("--skip_jvnv", action="store_true") -args = parser.parse_args() +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--skip_jvnv", action="store_true") + args = parser.parse_args() -download_bert_models() + download_bert_models() -download_slm_model() + download_slm_model() -download_pretrained_models() + download_pretrained_models() -download_jp_extra_pretrained_models() + download_jp_extra_pretrained_models() -if not args.skip_jvnv: - download_jvnv_models() + if not args.skip_jvnv: + download_jvnv_models() From b2cde01d77e1e1207d3e8818a559b2bda0b5f7d7 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Wed, 7 Feb 2024 13:34:45 +0900 Subject: [PATCH 09/17] Fix incompatible setting --- train_ms.py | 2 +- train_ms_jp_extra.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/train_ms.py b/train_ms.py index 3e61bf2..1a7eac6 100644 --- a/train_ms.py +++ b/train_ms.py @@ -193,8 +193,8 @@ def run(): pin_memory=True, collate_fn=collate_fn, batch_sampler=train_sampler, - persistent_workers=True, # これもメモリ消費量を減らそうとしてコメントアウト（事前ロード量？） + # persistent_workers=True, # prefetch_factor=4, ) # DataLoader config could be adjusted. eval_dataset = None diff --git a/train_ms_jp_extra.py b/train_ms_jp_extra.py index ce44ebe..9f7ab23 100644 --- a/train_ms_jp_extra.py +++ b/train_ms_jp_extra.py @@ -196,8 +196,8 @@ def run(): pin_memory=True, collate_fn=collate_fn, batch_sampler=train_sampler, - persistent_workers=True, - # これもメモリ消費量を減らそうとしてコメントアウト（事前ロード量？） + # これもメモリ消費量を減らそうとしてコメントアウト + # persistent_workers=True, # prefetch_factor=6, ) # DataLoader config could be adjusted. eval_dataset = None From d249ba54a0a5c99171a37cb5753d79aae0d67b75 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Wed, 7 Feb 2024 13:41:31 +0900 Subject: [PATCH 10/17] Improve line split message --- app.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index 31b6d86..284098f 100644 --- a/app.py +++ b/app.py @@ -304,7 +304,8 @@ if __name__ == "__main__": text_input = gr.TextArea(label="テキスト", value=initial_text) line_split = gr.Checkbox( - label="改行で分けて生成", value=DEFAULT_LINE_SPLIT + label="改行で分けて生成（分けたほうが感情が乗ります）", + value=DEFAULT_LINE_SPLIT, ) split_interval = gr.Slider( minimum=0.0, From 78689f6c2a0960f1a59ae3d37b6713109da0e8db Mon Sep 17 00:00:00 2001 From: litagin02 Date: Wed, 7 Feb 2024 13:42:20 +0900 Subject: [PATCH 11/17] Try to find best setting --- train_ms.py | 3 ++- train_ms_jp_extra.py | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/train_ms.py b/train_ms.py index 1a7eac6..a65a89f 100644 --- a/train_ms.py +++ b/train_ms.py @@ -187,8 +187,9 @@ def run(): collate_fn = TextAudioSpeakerCollate() train_loader = DataLoader( train_dataset, - # メモリ消費量を減らそうとnum_workersを0にしてみる + # メモリ消費量を減らそうとnum_workersを1にしてみる # num_workers=min(config.train_ms_config.num_workers, os.cpu_count() // 2), + num_workers=1, shuffle=False, pin_memory=True, collate_fn=collate_fn, diff --git a/train_ms_jp_extra.py b/train_ms_jp_extra.py index 9f7ab23..e20868f 100644 --- a/train_ms_jp_extra.py +++ b/train_ms_jp_extra.py @@ -190,14 +190,15 @@ def run(): collate_fn = TextAudioSpeakerCollate(use_jp_extra=True) train_loader = DataLoader( train_dataset, - # メモリ消費量を減らそうとnum_workersを0にしてみる + # メモリ消費量を減らそうとnum_workersを1にしてみる # num_workers=min(config.train_ms_config.num_workers, os.cpu_count() // 2), + num_workers=1, shuffle=False, pin_memory=True, collate_fn=collate_fn, batch_sampler=train_sampler, + persistent_workers=True, # これもメモリ消費量を減らそうとしてコメントアウト - # persistent_workers=True, # prefetch_factor=6, ) # DataLoader config could be adjusted. eval_dataset = None From d51c8bf6392ea9e3468368af7ae20b6ca1b5d694 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Wed, 7 Feb 2024 15:33:17 +0900 Subject: [PATCH 12/17] Fix: conflict of current_model in model_holder for multiple requests --- app.py | 6 +++++- common/tts_model.py | 12 ++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index 284098f..5665cf2 100644 --- a/app.py +++ b/app.py @@ -36,6 +36,8 @@ languages = [l.value for l in Languages] def tts_fn( + model_name, + model_path, text, language, reference_audio_path, @@ -54,7 +56,7 @@ def tts_fn( use_tone, speaker, ): - assert model_holder.current_model is not None + model_holder.load_model_gr(model_name, model_path) wrong_tone_message = "" kata_tone: Optional[list[tuple[str, int]]] = None @@ -418,6 +420,8 @@ if __name__ == "__main__": tts_button.click( tts_fn, inputs=[ + model_name, + model_path, text_input, language, ref_audio_path, diff --git a/common/tts_model.py b/common/tts_model.py index 7f44256..c148598 100644 --- a/common/tts_model.py +++ b/common/tts_model.py @@ -209,6 +209,18 @@ class ModelHolder: raise ValueError(f"Model `{model_name}` is not found") if model_path not in self.model_files_dict[model_name]: raise ValueError(f"Model file `{model_path}` is not found") + if ( + self.current_model is not None + and self.current_model.model_path == model_path + ): + # Already loaded + speakers = list(self.current_model.spk2id.keys()) + styles = list(self.current_model.style2id.keys()) + return ( + gr.Dropdown(choices=styles, value=styles[0]), + gr.Button(interactive=True, value="音声合成"), + gr.Dropdown(choices=speakers, value=speakers[0]), + ) self.current_model = Model( model_path=model_path, config_path=os.path.join(self.root_dir, model_name, "config.json"), From 6762c654acf33d0a2a1f52230d3dfefe6f7abb4a Mon Sep 17 00:00:00 2001 From: litagin02 Date: Wed, 7 Feb 2024 15:58:43 +0900 Subject: [PATCH 13/17] Improve: when cuda is unavailable, use cpu device --- config.py | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/config.py b/config.py index 123e4c9..056e7f1 100644 --- a/config.py +++ b/config.py @@ -1,15 +1,19 @@ """ @Desc: 全局配置文件读取 """ -import argparse + import os import shutil from typing import Dict, List +import torch import yaml from common.log import logger +# If not cuda available, set possible devices to cpu +cuda_available = torch.cuda.is_available() + class Resample_config: """重采样配置""" @@ -44,13 +48,23 @@ class Preprocess_text_config: max_val_total: int = 10000, clean: bool = True, ): - self.transcription_path: str = transcription_path # 原始文本文件路径，文本格式应为{wav_path}|{speaker_name}|{language}|{text}。 - self.cleaned_path: str = cleaned_path # 数据清洗后文本路径，可以不填。不填则将在原始文本目录生成 - self.train_path: str = train_path # 训练集路径，可以不填。不填则将在原始文本目录生成 - self.val_path: str = val_path # 验证集路径，可以不填。不填则将在原始文本目录生成 + self.transcription_path: str = ( + transcription_path # 原始文本文件路径，文本格式应为{wav_path}|{speaker_name}|{language}|{text}。 + ) + self.cleaned_path: str = ( + cleaned_path # 数据清洗后文本路径，可以不填。不填则将在原始文本目录生成 + ) + self.train_path: str = ( + train_path # 训练集路径，可以不填。不填则将在原始文本目录生成 + ) + self.val_path: str = ( + val_path # 验证集路径，可以不填。不填则将在原始文本目录生成 + ) self.config_path: str = config_path # 配置文件路径 self.val_per_lang: int = val_per_lang # 每个speaker的验证集条数 - self.max_val_total: int = max_val_total # 验证集最大条数，多于的会被截断并放到训练集中 + self.max_val_total: int = ( + max_val_total # 验证集最大条数，多于的会被截断并放到训练集中 + ) self.clean: bool = clean # 是否进行数据清洗 @classmethod @@ -83,6 +97,8 @@ class Bert_gen_config: ): self.config_path = config_path self.num_processes = num_processes + if not cuda_available: + device = "cpu" self.device = device self.use_multi_device = use_multi_device @@ -104,6 +120,8 @@ class Style_gen_config: ): self.config_path = config_path self.num_processes = num_processes + if not cuda_available: + device = "cpu" self.device = device @classmethod @@ -143,7 +161,7 @@ class Train_ms_config: class Webui_config: - """webui 配置""" + """webui 配置 (for webui.py, not supported now)""" def __init__( self, @@ -155,6 +173,8 @@ class Webui_config: share: bool = False, debug: bool = False, ): + if not cuda_available: + device = "cpu" self.device: str = device self.model: str = model # 端口号 self.config_path: str = config_path # 是否公开部署，对外网开放 @@ -182,6 +202,8 @@ class Server_config: origins: List[str] = None, ): self.port: int = port + if not cuda_available: + device = "cpu" self.device: str = device self.language: str = language self.limit: int = limit From 50833cda5fc195eabed27e403b84a1a1a16f2153 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Wed, 7 Feb 2024 16:11:13 +0900 Subject: [PATCH 14/17] Update (maybe ver 2.1) --- app.py | 5 +++-- clustering.ipynb | 29 ++++++++++++----------------- colab.ipynb | 2 +- common/constants.py | 1 + configs/config.json | 2 +- configs/configs_jp_extra.json | 2 +- docs/CHANGELOG.md | 11 +++++++++++ train_ms.py | 4 ++-- webui_train.py | 5 +++-- 9 files changed, 35 insertions(+), 26 deletions(-) diff --git a/app.py b/app.py index 5665cf2..92c2e13 100644 --- a/app.py +++ b/app.py @@ -19,6 +19,7 @@ from common.constants import ( DEFAULT_SPLIT_INTERVAL, DEFAULT_STYLE, DEFAULT_STYLE_WEIGHT, + LATEST_VERSION, Languages, ) from common.log import logger @@ -192,8 +193,8 @@ examples = [ ], ] -initial_md = """ -# Style-Bert-VITS2 ver 2.0 音声合成 +initial_md = f""" +# Style-Bert-VITS2 ver {LATEST_VERSION} 音声合成 注意: 初期からある[jvnvのモデル](https://huggingface.co/litagin/style_bert_vits2_jvnv)は、[JVNVコーパス（言語音声と非言語音声を持つ日本語感情音声コーパス）](https://sites.google.com/site/shinnosuketakamichi/research-topics/jvnv_corpus)で学習されたモデルです。ライセンスは[CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/deed.ja)です。 """ diff --git a/clustering.ipynb b/clustering.ipynb index 955f44a..12e728e 100644 --- a/clustering.ipynb +++ b/clustering.ipynb @@ -10,6 +10,8 @@ "もっといろいろ足したりいろんなスタイルベクトルを作って遊べると思います。\n", "ある程度慣れている人向けです。\n", "\n", + "JVNVコーパスのように**スタイルが既にファイル名等で分かれている場合は、最後の方のセルを使えばそれを利用してスタイルを作ることができます。**\n", + "\n", "例では[JVNVコーパス](https://sites.google.com/site/shinnosuketakamichi/research-topics/jvnv_corpus)のjvnv-M1を使います。\n", "\n", "## そもそもスタイルベクトルとは\n", @@ -204,6 +206,14 @@ "np.save(\"model_assets/jvnv-M1/style_vectors.npy\", save_vectors)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 正解ラベルがファイル名から分かる場合の作り方\n", + "JVNVコーパス等でファイル名によってスタイルラベルが分かる場合、以下のようにしてスタイルベクトルを作ることができます（デフォルトのJVNVモデルのスタイルはこれで作成しています）。" + ] + }, { "cell_type": "code", "execution_count": 8, @@ -211,24 +221,9 @@ "outputs": [], "source": [ "# JVNVコーパスの場合は正解はファイル名の最初の方から分かる。それを使って正解ラベルを作成\n", - "y_true = [name[3:6] for name in names]\n", + "label_dict = {\"ang\": 0, \"dis\": 1, \"fea\": 2, \"hap\": 3, \"sad\": 4, \"sur\": 5}\n", "\n", - "for i, y in enumerate(y_true):\n", - " if y == \"ang\":\n", - " y_true[i] = 0\n", - " elif y == \"dis\":\n", - " y_true[i] = 1\n", - " elif y == \"fea\":\n", - " y_true[i] = 2\n", - " elif y == \"hap\":\n", - " y_true[i] = 3\n", - " elif y == \"sad\":\n", - " y_true[i] = 4\n", - " elif y == \"sur\":\n", - " y_true[i] = 5\n", - " else:\n", - " print(y)\n", - " raise ValueError" + "y_true = [label_dict[name[3:6]] for name in names]" ] }, { diff --git a/colab.ipynb b/colab.ipynb index 30cf3cd..fb1a401 100644 --- a/colab.ipynb +++ b/colab.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Style-Bert-VITS2 (ver 2.0) のGoogle Colabでの学習\n", + "# Style-Bert-VITS2 (ver 2.1) のGoogle Colabでの学習\n", "\n", "Google Colab上でStyle-Bert-VITS2の学習を行うことができます。\n", "\n", diff --git a/common/constants.py b/common/constants.py index f000095..3571f51 100644 --- a/common/constants.py +++ b/common/constants.py @@ -1,5 +1,6 @@ import enum +LATEST_VERSION: str = "2.1" DEFAULT_STYLE: str = "Neutral" DEFAULT_STYLE_WEIGHT: float = 5.0 diff --git a/configs/config.json b/configs/config.json index 1b00c64..c125074 100644 --- a/configs/config.json +++ b/configs/config.json @@ -67,5 +67,5 @@ "use_spectral_norm": false, "gin_channels": 256 }, - "version": "2.0.1" + "version": "2.1" } diff --git a/configs/configs_jp_extra.json b/configs/configs_jp_extra.json index f60b39e..efd4634 100644 --- a/configs/configs_jp_extra.json +++ b/configs/configs_jp_extra.json @@ -74,5 +74,5 @@ "initial_channel": 64 } }, - "version": "2.0.1-JP-Extra" + "version": "2.1-JP-Extra" } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index b010a33..00a890c 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## v2.1 (2024-02-07) + +### 変更 +- 学習の際、デフォルトではbfloat16オプションを使わないよう変更（学習が発散したり質が下がることがある模様） +- 学習の際のメモリ使用量を削減しようと頑張った + +### バグ修正や改善 +- 音声合成（やそのAPI）において、同時に別の話者が選択され音声合成がリクエストされた場合に発生するエラーを修正 +- モデルマージ時に、そのレシピを`recipe.json`ファイルへ保存するように変更 +- 「改行で分けて生成」がより感情が乗る旨の明記等、軽微な説明文の改善 + ## v2.0.1 (2024-02-05) 軽微なバグ修正や改善 diff --git a/train_ms.py b/train_ms.py index a65a89f..57eb5f6 100644 --- a/train_ms.py +++ b/train_ms.py @@ -194,8 +194,8 @@ def run(): pin_memory=True, collate_fn=collate_fn, batch_sampler=train_sampler, - # これもメモリ消費量を減らそうとしてコメントアウト（事前ロード量？） - # persistent_workers=True, + persistent_workers=True, + # これもメモリ消費量を減らそうとしてコメントアウト # prefetch_factor=4, ) # DataLoader config could be adjusted. eval_dataset = None diff --git a/webui_train.py b/webui_train.py index b278cde..77301ba 100644 --- a/webui_train.py +++ b/webui_train.py @@ -8,6 +8,7 @@ from multiprocessing import cpu_count import gradio as gr import yaml +from common.constants import LATEST_VERSION from common.log import logger from common.subprocess_utils import run_script_with_log, second_elem_of @@ -315,8 +316,8 @@ def train(model_name, skip_style=False, use_jp_extra=True, speedup=False): return True, "Success: 学習が完了しました" -initial_md = """ -# Style-Bert-VITS2 ver 2.0 学習用WebUI +initial_md = f""" +# Style-Bert-VITS2 ver {LATEST_VERSION} 学習用WebUI ## 使い方 From 63506a2ca53f615c956ab3e38fd26f593c82f325 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Wed, 7 Feb 2024 16:26:21 +0900 Subject: [PATCH 15/17] Update README --- README.md | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 1fd0e26..3e7d012 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,9 @@ https://github.com/litagin02/Style-Bert-VITS2/assets/139731664/e853f9a2-db4a-420 - [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](http://colab.research.google.com/github/litagin02/Style-Bert-VITS2/blob/master/colab.ipynb) - [🤗 オンラインデモはこちらから](https://huggingface.co/spaces/litagin/Style-Bert-VITS2-JVNV) -- [**リリースページ**](https://github.com/litagin02/Style-Bert-VITS2/releases/tag/2.0)、[更新履歴](docs/CHANGELOG.md) +- [**リリースページ**](https://github.com/litagin02/Style-Bert-VITS2/releases/)、[更新履歴](docs/CHANGELOG.md) - - 2024-02-05: ver 2.0.1 + - 2024-02-07: ver 2.1 - 2024-02-03: ver 2.0 - 2024-01-09: ver 1.3 - 2023-12-31: ver 1.2 @@ -25,7 +25,7 @@ This repository is based on [Bert-VITS2](https://github.com/fishaudio/Bert-VITS2 - GitやPythonがない人でも（Windowsユーザーなら）簡単にインストールでき、学習もできます (多くを[EasyBertVits2](https://github.com/Zuntan03/EasyBertVits2/)からお借りしました)。またGoogle Colabでの学習もサポートしています: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](http://colab.research.google.com/github/litagin02/Style-Bert-VITS2/blob/master/colab.ipynb) - 音声合成のみに使う場合は、グラボがなくてもCPUで動作します。 - 他との連携に使えるAPIサーバーも同梱しています ([@darai0512](https://github.com/darai0512) 様によるPRです、ありがとうございます)。 -- 元々が「楽しそうな文章は楽しそうに、悲しそうな文章は悲しそうに」読むのがBert-VITS2の強みですので、このフォークで付加されたスタイル指定を無理に使わずとも感情豊かな音声を生成することができます。 +- 元々「楽しそうな文章は楽しそうに、悲しそうな文章は悲しそうに」読むのがBert-VITS2の強みですので、スタイル指定がデフォルトでも感情豊かな音声を生成することができます。 ## 使い方 @@ -141,7 +141,7 @@ python speech_mos.py -m - [EasyBertVits2](https://github.com/Zuntan03/EasyBertVits2)のように、PythonやGitを知らない人でも簡単に使える。 - 感情埋め込みのモデルを変更（256次元の[wespeaker-voxceleb-resnet34-LM](https://huggingface.co/pyannote/wespeaker-voxceleb-resnet34-LM)へ、感情埋め込みというよりは話者識別のための埋め込み） -- 埋め込みもベクトル量子化を取り払い、単なる全結合層に。 +- 感情埋め込みもベクトル量子化を取り払い、単なる全結合層に。 - スタイルベクトルファイル`style_vectors.npy`を作ることで、そのスタイルを使って効果の強さも連続的に指定しつつ音声を生成することができる。 - 各種WebUIを作成 - bf16での学習のサポート @@ -149,15 +149,12 @@ python speech_mos.py -m - その他軽微なbugfixやリファクタリング ## TODO -- [ ] デフォルトのJVNVモデルにJP-Extra版のものを追加 +- [x] デフォルトのJVNVモデルにJP-Extra版のものを追加 - [x] LinuxやWSL等、Windowsの通常環境以外でのサポート ← おそらく問題ないとの報告あり - [x] 複数話者学習での音声合成対応（学習は現在でも可能） -- [ ] 本家のver 2.1, 2.2, 2.3モデルの推論対応？（ver 2.1以外は明らかにめんどいのでたぶんやらない） - [x] `server_fastapi.py`の対応、とくにAPIで使えるようになると嬉しい人が増えるのかもしれない - [x] モデルのマージで声音と感情表現を混ぜる機能の実装 - [ ] 英語等多言語対応？ -- [ ] ONNX対応 - ## References In addition to the original reference (written below), I used the following repositories: From 75e0e03cdea0bceae26b4be78d73054ee8674c4f Mon Sep 17 00:00:00 2001 From: litagin02 Date: Wed, 7 Feb 2024 16:30:00 +0900 Subject: [PATCH 16/17] Avoid librosa latest version bug --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fbf8489..aa84daf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ gradio jaconv jieba langid -librosa +librosa==0.9.2 loguru matplotlib mecab-python3 From 865b75f5415067da36b101b8ab1ca5fbc27bb6f4 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Wed, 7 Feb 2024 16:31:50 +0900 Subject: [PATCH 17/17] Update bat echo version --- scripts/Update-to-JP-Extra.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/Update-to-JP-Extra.bat b/scripts/Update-to-JP-Extra.bat index 7033e63..95738c6 100644 --- a/scripts/Update-to-JP-Extra.bat +++ b/scripts/Update-to-JP-Extra.bat @@ -42,7 +42,7 @@ pushd Style-Bert-VITS2 @REM 初期化（必要なモデルのダウンロード） python initialize.py -echo Style-Bert-VITS2の2.0.1へのアップデートが完了しました。 +echo Style-Bert-VITS2の2.xへのアップデートが完了しました。 pause