From a6bc65271cd432addca20981c7ed598fb4105238 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Sun, 17 Nov 2024 12:41:05 +0900 Subject: [PATCH 1/4] hatch fmt --- convert_bert_onnx.py | 8 ++++---- convert_onnx.py | 10 +++++----- gradio_tabs/merge.py | 2 +- preprocess_text.py | 2 +- style_bert_vits2/models/infer_onnx.py | 3 ++- style_bert_vits2/models/utils/__init__.py | 6 ++++-- style_bert_vits2/nlp/__init__.py | 3 ++- style_bert_vits2/nlp/bert_models.py | 1 - style_bert_vits2/nlp/chinese/bert_feature.py | 3 ++- style_bert_vits2/nlp/english/bert_feature.py | 3 ++- style_bert_vits2/nlp/japanese/bert_feature.py | 3 ++- style_bert_vits2/nlp/onnx_bert_models.py | 3 ++- style_bert_vits2/tts_model.py | 3 ++- style_bert_vits2/utils/__init__.py | 3 ++- tests/test_main.py | 3 ++- 15 files changed, 33 insertions(+), 23 deletions(-) diff --git a/convert_bert_onnx.py b/convert_bert_onnx.py index f30935c..411e03c 100644 --- a/convert_bert_onnx.py +++ b/convert_bert_onnx.py @@ -31,8 +31,8 @@ if __name__ == "__main__": # モデルの入出力先ファイルパスを取得 language = Languages(args.language) pretrained_model_name_or_path = DEFAULT_BERT_MODEL_PATHS[language] - onnx_temp_model_path = Path(pretrained_model_name_or_path) / f"model_temp.onnx" - onnx_optimized_model_path = Path(pretrained_model_name_or_path) / f"model.onnx" + onnx_temp_model_path = Path(pretrained_model_name_or_path) / "model_temp.onnx" + onnx_optimized_model_path = Path(pretrained_model_name_or_path) / "model.onnx" tokenizer_json_path = Path(pretrained_model_name_or_path) / "tokenizer.json" print(Rule(characters="=", style=Style(color="blue"))) print(f"[bold cyan]Language:[/bold cyan] {language.name}") @@ -68,7 +68,7 @@ if __name__ == "__main__": # モデルを ONNX に変換 print(Rule(characters="=", style=Style(color="blue"))) - print(f"[bold cyan]Exporting ONNX model...[/bold cyan]") + print("[bold cyan]Exporting ONNX model...[/bold cyan]") print(Rule(characters="=", style=Style(color="blue"))) export_start_time = time.time() torch.onnx.export( @@ -98,7 +98,7 @@ if __name__ == "__main__": # ONNX モデルを最適化 print(Rule(characters="=", style=Style(color="blue"))) - print(f"[bold cyan]Optimizing ONNX model...[/bold cyan]") + print("[bold cyan]Optimizing ONNX model...[/bold cyan]") print(Rule(characters="=", style=Style(color="blue"))) optimize_start_time = time.time() onnx_model = onnx.load(onnx_temp_model_path) diff --git a/convert_onnx.py b/convert_onnx.py index b61f9b1..1433cb9 100644 --- a/convert_onnx.py +++ b/convert_onnx.py @@ -1,5 +1,5 @@ -# Usage: .venv/bin/python convert_onnx.py --model model_assets/koharune-ami/koharune-ami.safetensors -# Usage: .venv/bin/python convert_onnx.py --model model_assets/ (All models in the directory will be converted) +# Usage: python convert_onnx.py --model model_assets/koharune-ami/koharune-ami.safetensors +# Usage: python convert_onnx.py --model model_assets/ (All models in the directory will be converted) # ref: https://github.com/tuna2134/sbv2-api/blob/main/convert/convert_model.py import time @@ -162,7 +162,7 @@ if __name__ == "__main__": # モデルを ONNX に変換 print(Rule(characters="=", style=Style(color="blue"))) print( - f"[bold cyan]Exporting ONNX model... (Architecture: JP-Extra)[/bold cyan]" + "[bold cyan]Exporting ONNX model... (Architecture: JP-Extra)[/bold cyan]" ) print(Rule(characters="=", style=Style(color="blue"))) export_start_time = time.time() @@ -253,7 +253,7 @@ if __name__ == "__main__": # モデルを ONNX に変換 print(Rule(characters="=", style=Style(color="blue"))) print( - f"[bold cyan]Exporting ONNX model... (Architecture: Non-JP-Extra)[/bold cyan]" + "[bold cyan]Exporting ONNX model... (Architecture: Non-JP-Extra)[/bold cyan]" ) print(Rule(characters="=", style=Style(color="blue"))) export_start_time = time.time() @@ -310,7 +310,7 @@ if __name__ == "__main__": # ONNX モデルを最適化 print(Rule(characters="=", style=Style(color="blue"))) - print(f"[bold cyan]Optimizing ONNX model...[/bold cyan]") + print("[bold cyan]Optimizing ONNX model...[/bold cyan]") print(Rule(characters="=", style=Style(color="blue"))) optimize_start_time = time.time() onnx_model = onnx.load(onnx_temp_model_path) diff --git a/gradio_tabs/merge.py b/gradio_tabs/merge.py index 49117a3..75cc661 100644 --- a/gradio_tabs/merge.py +++ b/gradio_tabs/merge.py @@ -897,7 +897,7 @@ initial_md = """ - 話者数が違うモデル同士はおそらくマージできません。 """ -style_merge_md = f""" +style_merge_md = """ ## 3. スタイルベクトルのマージ 1. マージ後のモデルにいくつスタイルを追加したいかを「作りたいスタイル数」で指定 diff --git a/preprocess_text.py b/preprocess_text.py index 4dd7e33..97474a9 100644 --- a/preprocess_text.py +++ b/preprocess_text.py @@ -2,7 +2,7 @@ import argparse import json from collections import defaultdict from pathlib import Path -from random import sample, shuffle +from random import sample from typing import Optional from tqdm import tqdm diff --git a/style_bert_vits2/models/infer_onnx.py b/style_bert_vits2/models/infer_onnx.py index 422b3de..9ea8996 100644 --- a/style_bert_vits2/models/infer_onnx.py +++ b/style_bert_vits2/models/infer_onnx.py @@ -1,4 +1,5 @@ -from typing import Any, Optional, Sequence, Union +from collections.abc import Sequence +from typing import Any, Optional, Union import numpy as np import onnxruntime diff --git a/style_bert_vits2/models/utils/__init__.py b/style_bert_vits2/models/utils/__init__.py index b91c74a..189f05e 100644 --- a/style_bert_vits2/models/utils/__init__.py +++ b/style_bert_vits2/models/utils/__init__.py @@ -11,8 +11,10 @@ import torch from numpy.typing import NDArray from style_bert_vits2.logging import logger -from style_bert_vits2.models.utils import checkpoints # type: ignore -from style_bert_vits2.models.utils import safetensors # type: ignore +from style_bert_vits2.models.utils import ( + checkpoints, # type: ignore + safetensors, # type: ignore +) if TYPE_CHECKING: diff --git a/style_bert_vits2/nlp/__init__.py b/style_bert_vits2/nlp/__init__.py index b0cd478..36c51b3 100644 --- a/style_bert_vits2/nlp/__init__.py +++ b/style_bert_vits2/nlp/__init__.py @@ -1,6 +1,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional, Sequence, Union +from collections.abc import Sequence +from typing import TYPE_CHECKING, Any, Optional, Union from numpy.typing import NDArray diff --git a/style_bert_vits2/nlp/bert_models.py b/style_bert_vits2/nlp/bert_models.py index 4640ce3..926b372 100644 --- a/style_bert_vits2/nlp/bert_models.py +++ b/style_bert_vits2/nlp/bert_models.py @@ -251,7 +251,6 @@ def unload_tokenizer(language: Languages) -> None: language (Languages): アンロードする BERT トークナイザーの言語 """ - import torch if language in __loaded_tokenizers: del __loaded_tokenizers[language] diff --git a/style_bert_vits2/nlp/chinese/bert_feature.py b/style_bert_vits2/nlp/chinese/bert_feature.py index 9ff997d..2add3a2 100644 --- a/style_bert_vits2/nlp/chinese/bert_feature.py +++ b/style_bert_vits2/nlp/chinese/bert_feature.py @@ -1,6 +1,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional, Sequence, Union +from collections.abc import Sequence +from typing import TYPE_CHECKING, Any, Optional, Union import numpy as np from numpy.typing import NDArray diff --git a/style_bert_vits2/nlp/english/bert_feature.py b/style_bert_vits2/nlp/english/bert_feature.py index 5c3e24f..b81dcea 100644 --- a/style_bert_vits2/nlp/english/bert_feature.py +++ b/style_bert_vits2/nlp/english/bert_feature.py @@ -1,6 +1,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional, Sequence, Union +from collections.abc import Sequence +from typing import TYPE_CHECKING, Any, Optional, Union import numpy as np from numpy.typing import NDArray diff --git a/style_bert_vits2/nlp/japanese/bert_feature.py b/style_bert_vits2/nlp/japanese/bert_feature.py index 079ca85..87bf62d 100644 --- a/style_bert_vits2/nlp/japanese/bert_feature.py +++ b/style_bert_vits2/nlp/japanese/bert_feature.py @@ -1,6 +1,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional, Sequence, Union +from collections.abc import Sequence +from typing import TYPE_CHECKING, Any, Optional, Union import numpy as np from numpy.typing import NDArray diff --git a/style_bert_vits2/nlp/onnx_bert_models.py b/style_bert_vits2/nlp/onnx_bert_models.py index d4b095c..d9a413c 100644 --- a/style_bert_vits2/nlp/onnx_bert_models.py +++ b/style_bert_vits2/nlp/onnx_bert_models.py @@ -11,8 +11,9 @@ Style-Bert-VITS2 の ONNX 推論に必要な各言語ごとの ONNX 版 BERT モ import gc import time +from collections.abc import Sequence from pathlib import Path -from typing import Any, Optional, Sequence, Union +from typing import Any, Optional, Union import onnxruntime from huggingface_hub import hf_hub_download diff --git a/style_bert_vits2/tts_model.py b/style_bert_vits2/tts_model.py index 1351db7..5da0d4c 100644 --- a/style_bert_vits2/tts_model.py +++ b/style_bert_vits2/tts_model.py @@ -2,8 +2,9 @@ from __future__ import annotations import gc import time +from collections.abc import Sequence from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional, Sequence, Union +from typing import TYPE_CHECKING, Any, Optional, Union import numpy as np import onnxruntime diff --git a/style_bert_vits2/utils/__init__.py b/style_bert_vits2/utils/__init__.py index 9d43784..92839e9 100644 --- a/style_bert_vits2/utils/__init__.py +++ b/style_bert_vits2/utils/__init__.py @@ -1,4 +1,5 @@ -from typing import Any, Sequence, Union +from collections.abc import Sequence +from typing import Any, Union def torch_device_to_onnx_providers( diff --git a/tests/test_main.py b/tests/test_main.py index d50b502..143e15e 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,4 +1,5 @@ -from typing import Any, Literal, Sequence +from collections.abc import Sequence +from typing import Any, Literal import pytest from scipy.io import wavfile From 8d476175a759cd419486d046dc18b655bacd4067 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Sun, 17 Nov 2024 12:44:27 +0900 Subject: [PATCH 2/4] hathc run style:fmt --- style_bert_vits2/models/utils/__init__.py | 6 ++---- style_bert_vits2/nlp/bert_models.py | 1 - 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/style_bert_vits2/models/utils/__init__.py b/style_bert_vits2/models/utils/__init__.py index 189f05e..b91c74a 100644 --- a/style_bert_vits2/models/utils/__init__.py +++ b/style_bert_vits2/models/utils/__init__.py @@ -11,10 +11,8 @@ import torch from numpy.typing import NDArray from style_bert_vits2.logging import logger -from style_bert_vits2.models.utils import ( - checkpoints, # type: ignore - safetensors, # type: ignore -) +from style_bert_vits2.models.utils import checkpoints # type: ignore +from style_bert_vits2.models.utils import safetensors # type: ignore if TYPE_CHECKING: diff --git a/style_bert_vits2/nlp/bert_models.py b/style_bert_vits2/nlp/bert_models.py index 926b372..639025b 100644 --- a/style_bert_vits2/nlp/bert_models.py +++ b/style_bert_vits2/nlp/bert_models.py @@ -251,7 +251,6 @@ def unload_tokenizer(language: Languages) -> None: language (Languages): アンロードする BERT トークナイザーの言語 """ - if language in __loaded_tokenizers: del __loaded_tokenizers[language] gc.collect() From 9740256c8751ab54022092a8eed07ebc76e40111 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Sun, 17 Nov 2024 16:47:37 +0900 Subject: [PATCH 3/4] Add anime-whisper --- gradio_tabs/dataset.py | 13 ++++++++++--- transcribe.py | 9 +++------ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/gradio_tabs/dataset.py b/gradio_tabs/dataset.py index 55c98bd..2c437d2 100644 --- a/gradio_tabs/dataset.py +++ b/gradio_tabs/dataset.py @@ -177,9 +177,16 @@ def create_dataset_app() -> gr.Blocks: value=False, ) hf_repo_id = gr.Dropdown( - ["openai/whisper", "kotoba-tech/kotoba-whisper-v1.1"], - label="HuggingFaceのWhisperモデル", - value="openai/whisper", + [ + "openai/whisper-large-v3-turbo", + "openai/whisper-large-v3", + "openai/whisper-large-v2", + "kotoba-tech/kotoba-whisper-v1.1", + "kotoba-tech/kotoba-whisper-v2.1", + "litagin/anime-whisper", + ], + label="HuggingFaceのWhisper repo_id", + value="openai/whisper-large-v3", visible=False, ) compute_type = gr.Dropdown( diff --git a/transcribe.py b/transcribe.py index 2752126..47cd61a 100644 --- a/transcribe.py +++ b/transcribe.py @@ -67,8 +67,7 @@ def transcribe_files_with_hf_whisper( "num_beams": num_beams, "no_repeat_ngram_size": no_repeat_ngram_size, } - logger.info(f"generate_kwargs: {generate_kwargs}") - + logger.info(f"generate_kwargs: {generate_kwargs}, loading pipeline...") pipe = pipeline( model=model_id, max_new_tokens=128, @@ -79,6 +78,7 @@ def transcribe_files_with_hf_whisper( trust_remote_code=True, # generate_kwargs=generate_kwargs, ) + logger.info("Loaded pipeline") if initial_prompt is not None: prompt_ids: torch.Tensor = pipe.tokenizer.get_prompt_ids( initial_prompt, return_tensors="pt" @@ -196,10 +196,7 @@ if __name__ == "__main__": with open(output_file, "a", encoding="utf-8") as f: f.write(f"{wav_rel_path}|{model_name}|{language_id}|{text}\n") else: - if args.hf_repo_id == "": - model_id = f"openai/whisper-{args.model}" - else: - model_id = args.hf_repo_id + model_id = args.hf_repo_id logger.info(f"Loading HF Whisper model ({model_id})") pbar = tqdm(total=len(wav_files), file=SAFE_STDOUT) results = transcribe_files_with_hf_whisper( From 338345831749d61e258b7db126f60a3d9ee1121a Mon Sep 17 00:00:00 2001 From: litagin02 Date: Wed, 20 Nov 2024 18:58:15 +0900 Subject: [PATCH 4/4] delete unrelated file --- vad_filter.py | 92 --------------------------------------------------- 1 file changed, 92 deletions(-) delete mode 100644 vad_filter.py diff --git a/vad_filter.py b/vad_filter.py deleted file mode 100644 index 1259051..0000000 --- a/vad_filter.py +++ /dev/null @@ -1,92 +0,0 @@ -import argparse -import os -import shutil -import sys -from pathlib import Path - -import pandas as pd -import torch -from tqdm import tqdm - -from style_bert_vits2.logging import logger - - -vad_model, utils = torch.hub.load( - repo_or_dir="litagin02/silero-vad", - model="silero_vad", - onnx=True, - trust_repo=True, -) - -(get_speech_timestamps, _, read_audio, *_) = utils - - -def get_speech_ratio(audio_file): - sampling_rate = 16000 - - wav = read_audio(audio_file, sampling_rate=sampling_rate) - speech_timestamps = get_speech_timestamps( - wav, vad_model, sampling_rate=sampling_rate - ) - - speech_dur_ms = 0 - - for ts in speech_timestamps: - start_ms = ts["start"] / 16 - end_ms = ts["end"] / 16 - speech_dur_ms += end_ms - start_ms - - total_dur_ms = len(wav) / sampling_rate * 1000 - return speech_dur_ms / total_dur_ms - - -def process(file: Path): - speech_ratio = get_speech_ratio(file) - return file, speech_ratio - - -def main(): - parser = argparse.ArgumentParser(description="Calculate speech ratio.") - parser.add_argument( - "-i", "--input", help="Directory containing audio files", required=True - ) - args = parser.parse_args() - - if os.path.exists(os.path.join(args.input, "low_speech_ratio")): - logger.info("Low speech ratio directory already exists, skipping...") - exit(0) - - data_dir = Path(args.input) - wav_files = list(data_dir.glob("*.wav")) - wav_files.sort() - - if len(wav_files) < 100: - logger.warning("Too few files, skipping...") - exit(0) - - logger.info(f"Start VAD filtering for {data_dir}...") - - results = [] - - for wav_file in tqdm(wav_files, file=sys.stdout): - speech_ratio = get_speech_ratio(wav_file) - results.append((wav_file, speech_ratio)) - - results_df = pd.DataFrame(results, columns=["file", "speech_ratio"]) - results_df.to_csv(os.path.join(data_dir, "speech_ratio.csv"), index=False) - - logger.info(f"Speech ratio stats:\n{results_df['speech_ratio'].describe()}") - threshold = 0.5 - - low_speech_ratio_dir = os.path.join(data_dir, "low_speech_ratio") - os.makedirs(low_speech_ratio_dir, exist_ok=True) - - low_speech_files = results_df[results_df["speech_ratio"] < threshold]["file"] - logger.info(f"Moving {len(low_speech_files)} files to {low_speech_ratio_dir}...") - for low_speech_file in low_speech_files: - shutil.move(low_speech_file, low_speech_ratio_dir) - logger.success("VAD filtering completed.") - - -if __name__ == "__main__": - main()