diff --git a/README.md b/README.md index f38d5ba..ce41eb3 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ You can install via `pip install style-bert-vits2` (inference only), see [librar - [Zennの解説記事](https://zenn.dev/litagin/articles/034819a5256ff4) - [**リリースページ**](https://github.com/litagin02/Style-Bert-VITS2/releases/)、[更新履歴](/docs/CHANGELOG.md) + - 2025-08-24: Ver 2.7.0: 外部ライブラリ [Aivis Project](https://aivis-project.com/) 等との連携のため、ONNX変換のGUI追加、また音声認識モデルとして `litagin/anime-whisper` の追加等 - 2024-09-09: Ver 2.6.1: Google colabでうまく学習できない等のバグ修正のみ - 2024-06-16: Ver 2.6.0 (モデルの差分マージ・加重マージ・ヌルモデルマージの追加、使い道については[この記事](https://zenn.dev/litagin/articles/1297b1dc7bdc79)参照) - 2024-06-14: Ver 2.5.1 (利用規約をお願いへ変更したのみ) @@ -155,6 +156,10 @@ API仕様は起動後に`/docs`にて確認ください。 2つのモデルを、「声質」「声の高さ」「感情表現」「テンポ」の4点で混ぜ合わせて、新しいモデルを作ったり、また「あるモデルに、別の2つのモデルの差分を足す」等の操作ができます。 `App.bat`をダブルクリックか`python app.py`して開くWebUIの「マージ」タブから、2つのモデルを選択してマージすることができます。または`Merge.bat`をダブルクリックでもその単独タブが開きます。 +### ONNX変換 + +タブの「ONNX変換」から、学習済みsafetensorsファイルをONNX形式に変換することができます。これは外部ライブラリ等でONNX形式ファイルが必要な場合に使えます。例えば [Aivis Project](https://aivis-project.com/) では [AIVM Generator](https://aivm-generator.aivis-project.com/) を使って、safetensorsファイルとONNXファイルからAivis Speech用のモデルを作成できます。 + ### 自然性評価 学習結果のうちどのステップ数がいいかの「一つの」指標として、[SpeechMOS](https://github.com/tarepan/SpeechMOS) を使うスクリプトを用意しています: diff --git a/app.py b/app.py index 0d4ecbc..4da8e7a 100644 --- a/app.py +++ b/app.py @@ -5,6 +5,7 @@ import gradio as gr import torch from config import get_path_config +from gradio_tabs.convert_onnx import create_onnx_app from gradio_tabs.dataset import create_dataset_app from gradio_tabs.inference import create_inference_app from gradio_tabs.merge import create_merge_app @@ -42,7 +43,10 @@ if device == "cuda" and not torch.cuda.is_available(): path_config = get_path_config() model_holder = TTSModelHolder( - Path(path_config.assets_root), device, torch_device_to_onnx_providers(device) + Path(path_config.assets_root), + device, + torch_device_to_onnx_providers(device), + ignore_onnx=True, ) with gr.Blocks(theme=GRADIO_THEME) as app: @@ -58,6 +62,8 @@ with gr.Blocks(theme=GRADIO_THEME) as app: create_style_vectors_app() with gr.Tab("マージ"): create_merge_app(model_holder=model_holder) + with gr.Tab("ONNX変換"): + create_onnx_app(model_holder=model_holder) app.launch( server_name=args.host, diff --git a/bert_gen.py b/bert_gen.py index c4995cb..49be3e6 100644 --- a/bert_gen.py +++ b/bert_gen.py @@ -90,6 +90,7 @@ if __name__ == "__main__": executor.map(process_line, zip(lines, add_blank)), total=len(lines), file=SAFE_STDOUT, + dynamic_ncols=True, ) ) diff --git a/colab.ipynb b/colab.ipynb index dcf7bb5..bff3f39 100644 --- a/colab.ipynb +++ b/colab.ipynb @@ -425,6 +425,16 @@ "# 学習結果を試す・マージ・スタイル分けはこちらから\n", "!python app.py --share" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# ONNX変換は、変換したいsafetensorsファイルを指定してこのセルを実行してください。\n", + "!python convert_onnx.py --model \"Data/your_model/your_model_e100_s10000.safetensors\"" + ] } ], "metadata": { diff --git a/data_utils.py b/data_utils.py index 96eab38..54d4e27 100644 --- a/data_utils.py +++ b/data_utils.py @@ -68,7 +68,7 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset): skipped = 0 logger.info("Init dataset...") for _id, spk, language, text, phones, tone, word2ph in tqdm( - self.audiopaths_sid_text, file=sys.stdout + self.audiopaths_sid_text, file=sys.stdout, dynamic_ncols=True ): audiopath = f"{_id}" # if self.min_text_len <= len(phones) and len(phones) <= self.max_text_len: diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index e19f74d..91b7faa 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## v2.7.0 (2025-08-24) + +- [AivisProject](https://aivis-project.com/) 等との連携のため、ONNX変換のGUI追加 (Gradioタブの一つとして) +- また音声認識モデルとして `litagin/anime-whisper` の追加 +- その他軽微な修正等 + ## v2.6.1 (2024-09-09) - Google colabで、torchのバージョン由来でエラーが発生する不具合の修正（たぶん） diff --git a/gradio_tabs/convert_onnx.py b/gradio_tabs/convert_onnx.py new file mode 100644 index 0000000..33f7ba4 --- /dev/null +++ b/gradio_tabs/convert_onnx.py @@ -0,0 +1,101 @@ +from pathlib import Path + +import gradio as gr + +from style_bert_vits2.constants import GRADIO_THEME +from style_bert_vits2.logging import logger +from style_bert_vits2.tts_model import NullModelParam, TTSModelHolder +from style_bert_vits2.utils.subprocess import run_script_with_log + + +def call_convert_onnx( + model: str, +): + if model == "": + return "Error: モデル名を入力してください。" + logger.info("Start converting model to onnx...") + cmd = [ + "convert_onnx.py", + "--model", + model, + ] + success, message = run_script_with_log(cmd, ignore_warning=True) + if not success: + return f"Error: {message}" + return "ONNX変換が完了しました。" + + +initial_md = """ +safetensors形式のモデルをONNX形式に変換します。 +このONNXモデルは、[AIVM Generator](https://aivm-generator.aivis-project.com/) 等でさらにAIVM形式・AIVMX形式に変換して[AivisSpeech](https://aivis-project.com/)で利用できます。 + +変換には5分以上ほどの時間がかかります。進捗状況はターミナルのログを参照してください。 +""" + + +def create_onnx_app(model_holder: TTSModelHolder) -> gr.Blocks: + def get_model_files(model_name: str): + return [str(f) for f in model_holder.model_files_dict[model_name]] + + model_names = model_holder.model_names + if len(model_names) == 0: + logger.error( + f"モデルが見つかりませんでした。{model_holder.root_dir}にモデルを置いてください。" + ) + with gr.Blocks() as app: + gr.Markdown( + f"Error: モデルが見つかりませんでした。{model_holder.root_dir}にモデルを置いてください。" + ) + return app + initial_id = 0 + initial_pth_files = get_model_files(model_names[initial_id]) + + with gr.Blocks(theme=GRADIO_THEME) as app: + gr.Markdown(initial_md) + with gr.Row(): + with gr.Column(): + model_name = gr.Dropdown( + label="モデル一覧", + choices=model_names, + value=model_names[initial_id], + ) + model_path = gr.Dropdown( + label="モデルファイル", + choices=initial_pth_files, + value=initial_pth_files[0], + ) + refresh_button = gr.Button("更新") + convert_button = gr.Button("ONNX形式に変換", variant="primary") + info = gr.Textbox(label="情報") + + model_name.change( + model_holder.update_model_files_for_gradio, + inputs=[model_name], + outputs=[model_path], + ) + + def refresh_fn() -> tuple[gr.Dropdown, gr.Dropdown]: + names, files, _ = model_holder.update_model_names_for_gradio() + return names, files + + refresh_button.click( + refresh_fn, + outputs=[model_name, model_path], + ) + convert_button.click( + call_convert_onnx, + inputs=[model_path], + outputs=[info], + ) + + return app + + +if __name__ == "__main__": + from config import get_path_config + + path_config = get_path_config() + assets_root = path_config.assets_root + model_holder = TTSModelHolder(assets_root, "cpu", "", ignore_onnx=True) + app = create_onnx_app(model_holder) + app.launch(inbrowser=True) diff --git a/gradio_tabs/dataset.py b/gradio_tabs/dataset.py index 2c437d2..abedf62 100644 --- a/gradio_tabs/dataset.py +++ b/gradio_tabs/dataset.py @@ -51,6 +51,11 @@ def do_transcribe( ): if model_name == "": return "Error: モデル名を入力してください。" + if hf_repo_id == "litagin/anime-whisper": + logger.info( + "Since litagin/anime-whisper does not support initial prompt, it will be ignored." + ) + initial_prompt = "" cmd = [ "transcribe.py", @@ -159,34 +164,30 @@ def create_dataset_app() -> gr.Blocks: result1 = gr.Textbox(label="結果") with gr.Row(): with gr.Column(): + use_hf_whisper = gr.Checkbox( + label="HuggingFaceのWhisperを使う（速度が速いがVRAMを多く使う）", + value=False, + ) whisper_model = gr.Dropdown( [ - "tiny", - "base", - "small", - "medium", "large", "large-v2", "large-v3", ], label="Whisperモデル", value="large-v3", - ) - use_hf_whisper = gr.Checkbox( - label="HuggingFaceのWhisperを使う（速度が速いがVRAMを多く使う）", - value=False, + visible=True, ) hf_repo_id = gr.Dropdown( [ "openai/whisper-large-v3-turbo", "openai/whisper-large-v3", "openai/whisper-large-v2", - "kotoba-tech/kotoba-whisper-v1.1", "kotoba-tech/kotoba-whisper-v2.1", "litagin/anime-whisper", ], label="HuggingFaceのWhisper repo_id", - value="openai/whisper-large-v3", + value="openai/whisper-large-v3-turbo", visible=False, ) compute_type = gr.Dropdown( @@ -258,12 +259,13 @@ def create_dataset_app() -> gr.Blocks: ) use_hf_whisper.change( lambda x: ( + gr.update(visible=not x), gr.update(visible=x), gr.update(visible=x), gr.update(visible=not x), ), inputs=[use_hf_whisper], - outputs=[hf_repo_id, batch_size, compute_type], + outputs=[whisper_model, hf_repo_id, batch_size, compute_type], ) return app diff --git a/gradio_tabs/style_vectors.py b/gradio_tabs/style_vectors.py index c9014cd..46e8a74 100644 --- a/gradio_tabs/style_vectors.py +++ b/gradio_tabs/style_vectors.py @@ -322,6 +322,7 @@ def save_style_vectors_by_dirs(model_name: str, audio_dir_str: str): total=len(audio_files), file=SAFE_STDOUT, desc="Generating style vectors", + dynamic_ncols=True, ) ) diff --git a/preprocess_text.py b/preprocess_text.py index 97474a9..6d0363b 100644 --- a/preprocess_text.py +++ b/preprocess_text.py @@ -98,7 +98,7 @@ def preprocess( transcription_path.open("r", encoding="utf-8") as trans_file, cleaned_path.open("w", encoding="utf-8") as out_file, ): - for line in tqdm(trans_file, file=SAFE_STDOUT, total=total_lines): + for line in tqdm(trans_file, file=SAFE_STDOUT, total=total_lines, dynamic_ncols=True): try: processed_line = process_line( line, diff --git a/requirements-infer.txt b/requirements-infer.txt index 967d407..ad8e4df 100644 --- a/requirements-infer.txt +++ b/requirements-infer.txt @@ -27,7 +27,7 @@ pypinyin pyworld-prebuilt # stable_ts # tensorboard -torch -torchaudio +torch<2.4 +torchaudio<2.4 transformers umap-learn diff --git a/requirements.txt b/requirements.txt index 9a5501d..180e061 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,7 +27,7 @@ pypinyin pyworld-prebuilt stable_ts tensorboard -torch -torchaudio +torch<2.4 +torchaudio<2.4 transformers umap-learn diff --git a/resample.py b/resample.py index 285105c..6e255b1 100644 --- a/resample.py +++ b/resample.py @@ -140,7 +140,7 @@ if __name__ == "__main__": for file in original_files ] for future in tqdm( - as_completed(futures), total=len(original_files), file=SAFE_STDOUT + as_completed(futures), total=len(original_files), file=SAFE_STDOUT, dynamic_ncols=True ): pass diff --git a/server_editor.py b/server_editor.py index 7124aa6..732b578 100644 --- a/server_editor.py +++ b/server_editor.py @@ -202,7 +202,9 @@ if args.preload_onnx_bert: ) onnx_bert_models.load_tokenizer(Languages.JP) -model_holder = TTSModelHolder(model_dir, device, torch_device_to_onnx_providers(device)) +model_holder = TTSModelHolder( + model_dir, device, torch_device_to_onnx_providers(device), ignore_onnx=True +) if len(model_holder.model_names) == 0: logger.error(f"Models not found in {model_dir}.") sys.exit(1) diff --git a/slice.py b/slice.py index bfcb916..b231ce8 100644 --- a/slice.py +++ b/slice.py @@ -230,7 +230,7 @@ if __name__ == "__main__": for t in threads: t.start() - pbar = tqdm(total=len(audio_files), file=SAFE_STDOUT) + pbar = tqdm(total=len(audio_files), file=SAFE_STDOUT, dynamic_ncols=True) for file in audio_files: q.put(file) diff --git a/speech_mos.py b/speech_mos.py index c7a6a25..b01acb4 100644 --- a/speech_mos.py +++ b/speech_mos.py @@ -70,7 +70,7 @@ safetensors_files = list(safetensors_files) logger.info(f"There are {len(safetensors_files)} models.") -for model_file in tqdm(safetensors_files): +for model_file in tqdm(safetensors_files, dynamic_ncols=True): # `test_e10_s1000.safetensors`` -> 1000を取り出す match = re.search(r"_s(\d+)\.safetensors$", model_file.name) if match: diff --git a/style_bert_vits2/models/utils/__init__.py b/style_bert_vits2/models/utils/__init__.py index d04d0ff..75b03b4 100644 --- a/style_bert_vits2/models/utils/__init__.py +++ b/style_bert_vits2/models/utils/__init__.py @@ -11,10 +11,8 @@ import torch from numpy.typing import NDArray from style_bert_vits2.logging import logger -from style_bert_vits2.models.utils import ( - checkpoints, # type: ignore # noqa: F401 - safetensors, # type: ignore # noqa: F401 -) +from style_bert_vits2.models.utils import checkpoints # type: ignore # noqa: F401 +from style_bert_vits2.models.utils import safetensors # type: ignore # noqa: F401 if TYPE_CHECKING: diff --git a/style_bert_vits2/nlp/japanese/normalizer.py b/style_bert_vits2/nlp/japanese/normalizer.py index 7ecbfb6..5580341 100644 --- a/style_bert_vits2/nlp/japanese/normalizer.py +++ b/style_bert_vits2/nlp/japanese/normalizer.py @@ -121,7 +121,7 @@ def normalize_text(text: str) -> str: # 結合文字の濁点・半濁点を削除 # 通常の「ば」等はそのままのこされる、「あ゛」は上で「あ゙」になりここで「あ」になる res = res.replace("\u3099", "") # 結合文字の濁点を削除、る゙ → る - res = res.replace("\u309A", "") # 結合文字の半濁点を削除、な゚ → な + res = res.replace("\u309a", "") # 結合文字の半濁点を削除、な゚ → な return res diff --git a/style_bert_vits2/nlp/japanese/pyopenjtalk_worker/worker_server.py b/style_bert_vits2/nlp/japanese/pyopenjtalk_worker/worker_server.py index 37a12f8..0a46b57 100644 --- a/style_bert_vits2/nlp/japanese/pyopenjtalk_worker/worker_server.py +++ b/style_bert_vits2/nlp/japanese/pyopenjtalk_worker/worker_server.py @@ -71,7 +71,7 @@ class WorkerServer: def start_server(self, port: int, no_client_timeout: int = 30) -> None: logger.info("start pyopenjtalk worker server") with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server_socket: - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) server_socket.bind((socket.gethostname(), port)) server_socket.listen() sockets = [server_socket] diff --git a/style_bert_vits2/utils/subprocess.py b/style_bert_vits2/utils/subprocess.py index 8f159a2..25a1b8d 100644 --- a/style_bert_vits2/utils/subprocess.py +++ b/style_bert_vits2/utils/subprocess.py @@ -41,7 +41,7 @@ def run_script_with_log( def second_elem_of( - original_function: Callable[..., tuple[Any, Any]] + original_function: Callable[..., tuple[Any, Any]], ) -> Callable[..., Any]: """ 与えられた関数をラップし、その戻り値の 2 番目の要素のみを返す関数を生成する。 diff --git a/style_gen.py b/style_gen.py index 1ced5f0..55f1fc0 100644 --- a/style_gen.py +++ b/style_gen.py @@ -80,6 +80,7 @@ if __name__ == "__main__": executor.map(process_line, training_lines), total=len(training_lines), file=SAFE_STDOUT, + dynamic_ncols=True, ) ) ok_training_lines = [line for line, error in training_results if error is None] @@ -102,6 +103,7 @@ if __name__ == "__main__": executor.map(process_line, val_lines), total=len(val_lines), file=SAFE_STDOUT, + dynamic_ncols=True, ) ) ok_val_lines = [line for line, error in val_results if error is None] diff --git a/train_ms.py b/train_ms.py index 7244bb8..7df71d1 100644 --- a/train_ms.py +++ b/train_ms.py @@ -485,6 +485,7 @@ def run(): initial=global_step, smoothing=0.05, file=SAFE_STDOUT, + dynamic_ncols=True, ) initial_step = global_step diff --git a/train_ms_jp_extra.py b/train_ms_jp_extra.py index 8c33858..1d4893f 100644 --- a/train_ms_jp_extra.py +++ b/train_ms_jp_extra.py @@ -562,6 +562,7 @@ def run(): initial=global_step, smoothing=0.05, file=SAFE_STDOUT, + dynamic_ncols=True, ) initial_step = global_step diff --git a/transcribe.py b/transcribe.py index 47cd61a..1739f6c 100644 --- a/transcribe.py +++ b/transcribe.py @@ -73,8 +73,8 @@ def transcribe_files_with_hf_whisper( max_new_tokens=128, chunk_length_s=30, batch_size=batch_size, - torch_dtype=torch.float16, - device="cuda", + torch_dtype=torch.float16 if device == "cuda" else torch.float32, + device=device, trust_remote_code=True, # generate_kwargs=generate_kwargs, ) @@ -154,6 +154,10 @@ if __name__ == "__main__": wav_files = [f for f in input_dir.rglob("*.wav") if f.is_file()] wav_files = sorted(wav_files, key=lambda x: str(x)) + logger.info(f"Found {len(wav_files)} WAV files") + if len(wav_files) == 0: + logger.warning(f"No WAV files found in {input_dir}") + sys.exit(1) if output_file.exists(): logger.warning(f"{output_file} exists, backing up to {output_file}.bak") @@ -183,7 +187,7 @@ if __name__ == "__main__": except ValueError as e: logger.warning(f"Failed to load model, so use `auto` compute_type: {e}") model = WhisperModel(args.model, device=device) - for wav_file in tqdm(wav_files, file=SAFE_STDOUT): + for wav_file in tqdm(wav_files, file=SAFE_STDOUT, dynamic_ncols=True): text = transcribe_with_faster_whisper( model=model, audio_file=wav_file, @@ -198,7 +202,7 @@ if __name__ == "__main__": else: model_id = args.hf_repo_id logger.info(f"Loading HF Whisper model ({model_id})") - pbar = tqdm(total=len(wav_files), file=SAFE_STDOUT) + pbar = tqdm(total=len(wav_files), file=SAFE_STDOUT, dynamic_ncols=True) results = transcribe_files_with_hf_whisper( audio_files=wav_files, model_id=model_id,