diff --git a/.dockerignore b/.dockerignore
index a90abd2..da10b92 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -3,23 +3,15 @@
*
+!/style_bert_vits2/
+
!/bert/deberta-v2-large-japanese-char-wwm/
!/common/
!/configs/
!/dict_data/default.csv
!/model_assets/
-!/monotonic_align/
-!/text/
-!/attentions.py
-!/commons.py
!/config.py
!/default_config.yml
-!/infer.py
-!/models.py
-!/models_jp_extra.py
-!/modules.py
!/requirements.txt
!/server_editor.py
-!/transforms.py
-!/utils.py
diff --git a/.gitignore b/.gitignore
index b556dff..fe8824b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,8 +1,13 @@
-.vscode/
-
__pycache__/
venv/
+.venv/
+dist/
+.coverage
.ipynb_checkpoints/
+.ruff_cache/
+
+/Data/
+/model_assets/
/*.yml
!/default_config.yml
diff --git a/.vscode/extensions.json b/.vscode/extensions.json
new file mode 100644
index 0000000..7478fbd
--- /dev/null
+++ b/.vscode/extensions.json
@@ -0,0 +1,6 @@
+{
+ "recommendations": [
+ "ms-python.python",
+ "ms-python.vscode-pylance"
+ ]
+}
\ No newline at end of file
diff --git a/.vscode/settings.json b/.vscode/settings.json
new file mode 100644
index 0000000..2a583eb
--- /dev/null
+++ b/.vscode/settings.json
@@ -0,0 +1,26 @@
+{
+ // Pylance の Type Checking を有効化
+ "python.languageServer": "Pylance",
+ "python.analysis.typeCheckingMode": "strict",
+ // Pylance の Type Checking のうち、いくつかのエラー報告を抑制する
+ "python.analysis.diagnosticSeverityOverrides": {
+ "reportConstantRedefinition": "none",
+ "reportGeneralTypeIssues": "warning",
+ "reportMissingParameterType": "warning",
+ "reportMissingTypeStubs": "none",
+ "reportPrivateImportUsage": "none",
+ "reportPrivateUsage": "warning",
+ "reportShadowedImports": "none",
+ "reportUnnecessaryComparison": "none",
+ "reportUnknownArgumentType": "none",
+ "reportUnknownMemberType": "none",
+ "reportUnknownParameterType": "warning",
+ "reportUnknownVariableType": "none",
+ "reportUnusedFunction": "none",
+ "reportUnusedVariable": "information",
+ },
+ "[python]": {
+ "editor.defaultFormatter": "ms-python.black-formatter",
+ "editor.formatOnType": true,
+ },
+}
\ No newline at end of file
diff --git a/Dataset.bat b/Dataset.bat
deleted file mode 100644
index 03d3850..0000000
--- a/Dataset.bat
+++ /dev/null
@@ -1,11 +0,0 @@
-chcp 65001 > NUL
-@echo off
-
-pushd %~dp0
-echo Running webui_dataset.py...
-venv\Scripts\python webui_dataset.py
-
-if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% )
-
-popd
-pause
\ No newline at end of file
diff --git a/Merge.bat b/Merge.bat
deleted file mode 100644
index 8e1dedb..0000000
--- a/Merge.bat
+++ /dev/null
@@ -1,13 +0,0 @@
-chcp 65001 > NUL
-
-@echo off
-
-pushd %~dp0
-
-echo Running webui_merge.py...
-venv\Scripts\python webui_merge.py
-
-if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% )
-
-popd
-pause
\ No newline at end of file
diff --git a/Style.bat b/Style.bat
deleted file mode 100644
index 409cf91..0000000
--- a/Style.bat
+++ /dev/null
@@ -1,12 +0,0 @@
-chcp 65001 > NUL
-
-@echo off
-
-pushd %~dp0
-echo Running webui_style_vectors.py...
-venv\Scripts\python webui_style_vectors.py
-
-if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% )
-
-popd
-pause
\ No newline at end of file
diff --git a/Train.bat b/Train.bat
deleted file mode 100644
index 5a93d02..0000000
--- a/Train.bat
+++ /dev/null
@@ -1,13 +0,0 @@
-chcp 65001 > NUL
-
-@echo off
-
-pushd %~dp0
-
-echo Running webui_train.py...
-venv\Scripts\python webui_train.py
-
-if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% )
-
-popd
-pause
\ No newline at end of file
diff --git a/app.py b/app.py
index 997bd04..281eb27 100644
--- a/app.py
+++ b/app.py
@@ -5,15 +5,22 @@ import gradio as gr
import torch
import yaml
-from common.constants import GRADIO_THEME, LATEST_VERSION
-from common.tts_model import ModelHolder
-from webui import (
- create_dataset_app,
- create_inference_app,
- create_merge_app,
- create_style_vectors_app,
- create_train_app,
-)
+from style_bert_vits2.constants import GRADIO_THEME, VERSION
+from style_bert_vits2.nlp.japanese import pyopenjtalk_worker
+from style_bert_vits2.nlp.japanese.user_dict import update_dict
+from style_bert_vits2.tts_model import TTSModelHolder
+from webui.dataset import create_dataset_app
+from webui.inference import create_inference_app
+from webui.merge import create_merge_app
+from webui.style_vectors import create_style_vectors_app
+from webui.train import create_train_app
+
+
+# このプロセスからはワーカーを起動して辞書を使いたいので、ここで初期化
+pyopenjtalk_worker.initialize_worker()
+
+# dict_data/ 以下の辞書データを pyopenjtalk に適用
+update_dict()
# Get path settings
with Path("configs/paths.yml").open("r", encoding="utf-8") as f:
@@ -23,6 +30,8 @@ with Path("configs/paths.yml").open("r", encoding="utf-8") as f:
parser = argparse.ArgumentParser()
parser.add_argument("--device", type=str, default="cuda")
+parser.add_argument("--host", type=str, default="127.0.0.1")
+parser.add_argument("--port", type=int, default=None)
parser.add_argument("--no_autolaunch", action="store_true")
parser.add_argument("--share", action="store_true")
@@ -31,10 +40,10 @@ device = args.device
if device == "cuda" and not torch.cuda.is_available():
device = "cpu"
-model_holder = ModelHolder(Path(assets_root), device)
+model_holder = TTSModelHolder(Path(assets_root), device)
with gr.Blocks(theme=GRADIO_THEME) as app:
- gr.Markdown(f"# Style-Bert-VITS2 WebUI (version {LATEST_VERSION})")
+ gr.Markdown(f"# Style-Bert-VITS2 WebUI (version {VERSION})")
with gr.Tabs():
with gr.Tab("音声合成"):
create_inference_app(model_holder=model_holder)
@@ -48,4 +57,9 @@ with gr.Blocks(theme=GRADIO_THEME) as app:
create_merge_app(model_holder=model_holder)
-app.launch(inbrowser=not args.no_autolaunch, share=args.share)
+app.launch(
+ server_name=args.host,
+ server_port=args.port,
+ inbrowser=not args.no_autolaunch,
+ share=args.share,
+)
diff --git a/bert_gen.py b/bert_gen.py
index 19dbbb6..068afd6 100644
--- a/bert_gen.py
+++ b/bert_gen.py
@@ -5,14 +5,21 @@ import torch
import torch.multiprocessing as mp
from tqdm import tqdm
-import commons
-from text.get_bert import get_bert
-import text.pyopenjtalk_worker as pyopenjtalk
-import utils
-from common.log import logger
-from common.stdout_wrapper import SAFE_STDOUT
from config import config
-from text import cleaned_text_to_sequence
+from style_bert_vits2.logging import logger
+from style_bert_vits2.models import commons
+from style_bert_vits2.models.hyper_parameters import HyperParameters
+from style_bert_vits2.nlp import cleaned_text_to_sequence, extract_bert_feature
+from style_bert_vits2.nlp.japanese import pyopenjtalk_worker
+from style_bert_vits2.nlp.japanese.user_dict import update_dict
+from style_bert_vits2.utils.stdout_wrapper import SAFE_STDOUT
+
+
+# このプロセスからはワーカーを起動して辞書を使いたいので、ここで初期化
+pyopenjtalk_worker.initialize_worker()
+
+# dict_data/ 以下の辞書データを pyopenjtalk に適用
+update_dict()
def process_line(x):
@@ -47,7 +54,7 @@ def process_line(x):
bert = torch.load(bert_path)
assert bert.shape[-1] == len(phone)
except Exception:
- bert = get_bert(text, word2ph, language_str, device)
+ bert = extract_bert_feature(text, word2ph, language_str, device)
assert bert.shape[-1] == len(phone)
torch.save(bert, bert_path)
@@ -64,12 +71,12 @@ if __name__ == "__main__":
)
args, _ = parser.parse_known_args()
config_path = args.config
- hps = utils.get_hparams_from_file(config_path)
+ hps = HyperParameters.load_from_json(config_path)
lines = []
- with open(hps.data.training_files, encoding="utf-8") as f:
+ with open(hps.data.training_files, "r", encoding="utf-8") as f:
lines.extend(f.readlines())
- with open(hps.data.validation_files, encoding="utf-8") as f:
+ with open(hps.data.validation_files, "r", encoding="utf-8") as f:
lines.extend(f.readlines())
add_blank = [hps.data.add_blank] * len(lines)
diff --git a/colab.ipynb b/colab.ipynb
index a48affc..0684088 100644
--- a/colab.ipynb
+++ b/colab.ipynb
@@ -4,7 +4,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "# Style-Bert-VITS2 (ver 2.3.1) のGoogle Colabでの学習\n",
+ "# Style-Bert-VITS2 (ver 2.4) のGoogle Colabでの学習\n",
"\n",
"Google Colab上でStyle-Bert-VITS2の学習を行うことができます。\n",
"\n",
@@ -35,8 +35,8 @@
"metadata": {},
"outputs": [],
"source": [
- "#@title このセルを実行して環境構築してください。\n",
- "#@markdown 最後に赤文字でエラーや警告が出ても何故かうまくいくみたいです。\n",
+ "# このセルを実行して環境構築してください。\n",
+ "# エラーダイアログ「WARNING: The following packages were previously imported in this runtime: [pydevd_plugins]」が出るが「キャンセル」を選択して続行してください。\n",
"\n",
"!git clone https://github.com/litagin02/Style-Bert-VITS2.git\n",
"%cd Style-Bert-VITS2/\n",
@@ -219,7 +219,7 @@
"batch_size = 4\n",
"\n",
"# 学習のエポック数（データセットを合計何周するか）。\n",
- "# 100ぐらいで十分かもしれませんが、もっと多くやると質が上がるのかもしれません。\n",
+ "# 100で多すぎるほどかもしれませんが、もっと多くやると質が上がるのかもしれません。\n",
"epochs = 100\n",
"\n",
"# 保存頻度。何ステップごとにモデルを保存するか。分からなければデフォルトのままで。\n",
@@ -255,7 +255,7 @@
},
"outputs": [],
"source": [
- "from webui_train import preprocess_all\n",
+ "from webui.train import preprocess_all\n",
"\n",
"preprocess_all(\n",
" model_name=model_name,\n",
@@ -307,7 +307,7 @@
"\n",
"\n",
"import yaml\n",
- "from webui_train import get_path\n",
+ "from webui.train import get_path\n",
"\n",
"dataset_path, _, _, _, config_path = get_path(model_name)\n",
"\n",
@@ -350,41 +350,9 @@
},
"outputs": [],
"source": [
- "#@title 学習結果を試すならここから\n",
+ "# 学習結果を試す・マージ・スタイル分けはこちらから\n",
"!python app.py --share --dir {assets_root}"
]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## 5. スタイル分け"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "!python webui_style_vectors.py --share"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## 6. マージ"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "!python webui_merge.py --share"
- ]
}
],
"metadata": {
diff --git a/common/constants.py b/common/constants.py
deleted file mode 100644
index f3ccb69..0000000
--- a/common/constants.py
+++ /dev/null
@@ -1,28 +0,0 @@
-import enum
-
-# Built-in theme: "default", "base", "monochrome", "soft", "glass"
-# See https://huggingface.co/spaces/gradio/theme-gallery for more themes
-GRADIO_THEME: str = "NoCrypt/miku"
-
-LATEST_VERSION: str = "2.4"
-
-USER_DICT_DIR = "dict_data"
-
-DEFAULT_STYLE: str = "Neutral"
-DEFAULT_STYLE_WEIGHT: float = 5.0
-
-
-class Languages(str, enum.Enum):
- JP = "JP"
- EN = "EN"
- ZH = "ZH"
-
-
-DEFAULT_SDP_RATIO: float = 0.2
-DEFAULT_NOISE: float = 0.6
-DEFAULT_NOISEW: float = 0.8
-DEFAULT_LENGTH: float = 1.0
-DEFAULT_LINE_SPLIT: bool = True
-DEFAULT_SPLIT_INTERVAL: float = 0.5
-DEFAULT_ASSIST_TEXT_WEIGHT: float = 0.7
-DEFAULT_ASSIST_TEXT_WEIGHT: float = 1.0
diff --git a/common/log.py b/common/log.py
deleted file mode 100644
index 71b5e63..0000000
--- a/common/log.py
+++ /dev/null
@@ -1,18 +0,0 @@
-"""
-logger封装
-"""
-
-from loguru import logger
-
-from .stdout_wrapper import SAFE_STDOUT
-
-# 移除所有默认的处理器
-logger.remove()
-
-# 自定义格式并添加到标准输出
-log_format = (
- "{time:MM-DD HH:mm:ss} |{level:^8}| {file}:{line} | {message}"
-)
-
-# logger.add(SAFE_STDOUT, format=log_format, backtrace=True, diagnose=True, level="TRACE")
-logger.add(SAFE_STDOUT, format=log_format, backtrace=True, diagnose=True)
diff --git a/common/subprocess_utils.py b/common/subprocess_utils.py
deleted file mode 100644
index 40426f7..0000000
--- a/common/subprocess_utils.py
+++ /dev/null
@@ -1,33 +0,0 @@
-import subprocess
-import sys
-
-from .log import logger
-from .stdout_wrapper import SAFE_STDOUT
-
-python = sys.executable
-
-
-def run_script_with_log(cmd: list[str], ignore_warning=False) -> tuple[bool, str]:
- logger.info(f"Running: {' '.join(cmd)}")
- result = subprocess.run(
- [python] + cmd,
- stdout=SAFE_STDOUT, # type: ignore
- stderr=subprocess.PIPE,
- text=True,
- encoding="utf-8",
- )
- if result.returncode != 0:
- logger.error(f"Error: {' '.join(cmd)}\n{result.stderr}")
- return False, result.stderr
- elif result.stderr and not ignore_warning:
- logger.warning(f"Warning: {' '.join(cmd)}\n{result.stderr}")
- return True, result.stderr
- logger.success(f"Success: {' '.join(cmd)}")
- return True, ""
-
-
-def second_elem_of(original_function):
- def inner_function(*args, **kwargs):
- return original_function(*args, **kwargs)[1]
-
- return inner_function
diff --git a/common/tts_model.py b/common/tts_model.py
deleted file mode 100644
index 9e2171d..0000000
--- a/common/tts_model.py
+++ /dev/null
@@ -1,327 +0,0 @@
-import os
-import warnings
-from pathlib import Path
-from typing import Optional, Union
-
-import gradio as gr
-import numpy as np
-
-import torch
-from gradio.processing_utils import convert_to_16_bit_wav
-
-import utils
-from infer import get_net_g, infer
-from models import SynthesizerTrn
-from models_jp_extra import SynthesizerTrn as SynthesizerTrnJPExtra
-
-from .constants import (
- DEFAULT_ASSIST_TEXT_WEIGHT,
- DEFAULT_LENGTH,
- DEFAULT_LINE_SPLIT,
- DEFAULT_NOISE,
- DEFAULT_NOISEW,
- DEFAULT_SDP_RATIO,
- DEFAULT_SPLIT_INTERVAL,
- DEFAULT_STYLE,
- DEFAULT_STYLE_WEIGHT,
-)
-from .log import logger
-
-
-def adjust_voice(fs, wave, pitch_scale, intonation_scale):
- if pitch_scale == 1.0 and intonation_scale == 1.0:
- # 初期値の場合は、音質劣化を避けるためにそのまま返す
- return fs, wave
-
- try:
- import pyworld
- except ImportError:
- raise ImportError(
- "pyworld is not installed. Please install it by `pip install pyworld`"
- )
-
- # pyworldでf0を加工して合成
- # pyworldよりもよいのがあるかもしれないが……
-
- wave = wave.astype(np.double)
- f0, t = pyworld.harvest(wave, fs)
- # 質が高そうだしとりあえずharvestにしておく
-
- sp = pyworld.cheaptrick(wave, f0, t, fs)
- ap = pyworld.d4c(wave, f0, t, fs)
-
- non_zero_f0 = [f for f in f0 if f != 0]
- f0_mean = sum(non_zero_f0) / len(non_zero_f0)
-
- for i, f in enumerate(f0):
- if f == 0:
- continue
- f0[i] = pitch_scale * f0_mean + intonation_scale * (f - f0_mean)
-
- wave = pyworld.synthesize(f0, sp, ap, fs)
- return fs, wave
-
-
-class Model:
- def __init__(
- self, model_path: Path, config_path: Path, style_vec_path: Path, device: str
- ):
- self.model_path: Path = model_path
- self.config_path: Path = config_path
- self.style_vec_path: Path = style_vec_path
- self.device: str = device
- self.hps: utils.HParams = utils.get_hparams_from_file(self.config_path)
- self.spk2id: dict[str, int] = self.hps.data.spk2id
- self.id2spk: dict[int, str] = {v: k for k, v in self.spk2id.items()}
-
- self.num_styles: int = self.hps.data.num_styles
- if hasattr(self.hps.data, "style2id"):
- self.style2id: dict[str, int] = self.hps.data.style2id
- else:
- self.style2id: dict[str, int] = {str(i): i for i in range(self.num_styles)}
- if len(self.style2id) != self.num_styles:
- raise ValueError(
- f"Number of styles ({self.num_styles}) does not match the number of style2id ({len(self.style2id)})"
- )
-
- self.style_vectors: np.ndarray = np.load(self.style_vec_path)
- if self.style_vectors.shape[0] != self.num_styles:
- raise ValueError(
- f"The number of styles ({self.num_styles}) does not match the number of style vectors ({self.style_vectors.shape[0]})"
- )
-
- self.net_g: Union[SynthesizerTrn, SynthesizerTrnJPExtra, None] = None
-
- def load_net_g(self):
- self.net_g = get_net_g(
- model_path=str(self.model_path),
- version=self.hps.version,
- device=self.device,
- hps=self.hps,
- )
-
- def get_style_vector(self, style_id: int, weight: float = 1.0) -> np.ndarray:
- mean = self.style_vectors[0]
- style_vec = self.style_vectors[style_id]
- style_vec = mean + (style_vec - mean) * weight
- return style_vec
-
- def get_style_vector_from_audio(
- self, audio_path: str, weight: float = 1.0
- ) -> np.ndarray:
- from style_gen import get_style_vector
-
- xvec = get_style_vector(audio_path)
- mean = self.style_vectors[0]
- xvec = mean + (xvec - mean) * weight
- return xvec
-
- def infer(
- self,
- text: str,
- language: str = "JP",
- sid: int = 0,
- reference_audio_path: Optional[str] = None,
- sdp_ratio: float = DEFAULT_SDP_RATIO,
- noise: float = DEFAULT_NOISE,
- noisew: float = DEFAULT_NOISEW,
- length: float = DEFAULT_LENGTH,
- line_split: bool = DEFAULT_LINE_SPLIT,
- split_interval: float = DEFAULT_SPLIT_INTERVAL,
- assist_text: Optional[str] = None,
- assist_text_weight: float = DEFAULT_ASSIST_TEXT_WEIGHT,
- use_assist_text: bool = False,
- style: str = DEFAULT_STYLE,
- style_weight: float = DEFAULT_STYLE_WEIGHT,
- given_tone: Optional[list[int]] = None,
- pitch_scale: float = 1.0,
- intonation_scale: float = 1.0,
- ) -> tuple[int, np.ndarray]:
- logger.info(f"Start generating audio data from text:\n{text}")
- if language != "JP" and self.hps.version.endswith("JP-Extra"):
- raise ValueError(
- "The model is trained with JP-Extra, but the language is not JP"
- )
- if reference_audio_path == "":
- reference_audio_path = None
- if assist_text == "" or not use_assist_text:
- assist_text = None
-
- if self.net_g is None:
- self.load_net_g()
- if reference_audio_path is None:
- style_id = self.style2id[style]
- style_vector = self.get_style_vector(style_id, style_weight)
- else:
- style_vector = self.get_style_vector_from_audio(
- reference_audio_path, style_weight
- )
- if not line_split:
- with torch.no_grad():
- audio = infer(
- text=text,
- sdp_ratio=sdp_ratio,
- noise_scale=noise,
- noise_scale_w=noisew,
- length_scale=length,
- sid=sid,
- language=language,
- hps=self.hps,
- net_g=self.net_g,
- device=self.device,
- assist_text=assist_text,
- assist_text_weight=assist_text_weight,
- style_vec=style_vector,
- given_tone=given_tone,
- )
- else:
- texts = text.split("\n")
- texts = [t for t in texts if t != ""]
- audios = []
- with torch.no_grad():
- for i, t in enumerate(texts):
- audios.append(
- infer(
- text=t,
- sdp_ratio=sdp_ratio,
- noise_scale=noise,
- noise_scale_w=noisew,
- length_scale=length,
- sid=sid,
- language=language,
- hps=self.hps,
- net_g=self.net_g,
- device=self.device,
- assist_text=assist_text,
- assist_text_weight=assist_text_weight,
- style_vec=style_vector,
- )
- )
- if i != len(texts) - 1:
- audios.append(np.zeros(int(44100 * split_interval)))
- audio = np.concatenate(audios)
- logger.info("Audio data generated successfully")
- if not (pitch_scale == 1.0 and intonation_scale == 1.0):
- _, audio = adjust_voice(
- fs=self.hps.data.sampling_rate,
- wave=audio,
- pitch_scale=pitch_scale,
- intonation_scale=intonation_scale,
- )
- with warnings.catch_warnings():
- warnings.simplefilter("ignore")
- audio = convert_to_16_bit_wav(audio)
- return (self.hps.data.sampling_rate, audio)
-
-
-class ModelHolder:
- def __init__(self, root_dir: Path, device: str):
- self.root_dir: Path = root_dir
- self.device: str = device
- self.model_files_dict: dict[str, list[Path]] = {}
- self.current_model: Optional[Model] = None
- self.model_names: list[str] = []
- self.models: list[Model] = []
- self.models_info: list[dict[str, Union[str, list[str]]]] = []
- self.refresh()
-
- def refresh(self):
- self.model_files_dict = {}
- self.model_names = []
- self.current_model = None
- self.models_info = []
-
- model_dirs = [d for d in self.root_dir.iterdir() if d.is_dir()]
- for model_dir in model_dirs:
- model_files = [
- f
- for f in model_dir.iterdir()
- if f.suffix in [".pth", ".pt", ".safetensors"]
- ]
- if len(model_files) == 0:
- logger.warning(f"No model files found in {model_dir}, so skip it")
- continue
- config_path = model_dir / "config.json"
- if not config_path.exists():
- logger.warning(
- f"Config file {config_path} not found, so skip {model_dir}"
- )
- continue
- self.model_files_dict[model_dir.name] = model_files
- self.model_names.append(model_dir.name)
- hps = utils.get_hparams_from_file(config_path)
- style2id: dict[str, int] = hps.data.style2id
- styles = list(style2id.keys())
- spk2id: dict[str, int] = hps.data.spk2id
- speakers = list(spk2id.keys())
- self.models_info.append(
- {
- "name": model_dir.name,
- "files": [str(f) for f in model_files],
- "styles": styles,
- "speakers": speakers,
- }
- )
-
- def load_model(self, model_name: str, model_path_str: str):
- model_path = Path(model_path_str)
- if model_name not in self.model_files_dict:
- raise ValueError(f"Model `{model_name}` is not found")
- if model_path not in self.model_files_dict[model_name]:
- raise ValueError(f"Model file `{model_path}` is not found")
- if self.current_model is None or self.current_model.model_path != model_path:
- self.current_model = Model(
- model_path=model_path,
- config_path=self.root_dir / model_name / "config.json",
- style_vec_path=self.root_dir / model_name / "style_vectors.npy",
- device=self.device,
- )
- return self.current_model
-
- def load_model_gr(
- self, model_name: str, model_path_str: str
- ) -> tuple[gr.Dropdown, gr.Button, gr.Dropdown]:
- model_path = Path(model_path_str)
- if model_name not in self.model_files_dict:
- raise ValueError(f"Model `{model_name}` is not found")
- if model_path not in self.model_files_dict[model_name]:
- raise ValueError(f"Model file `{model_path}` is not found")
- if (
- self.current_model is not None
- and self.current_model.model_path == model_path
- ):
- # Already loaded
- speakers = list(self.current_model.spk2id.keys())
- styles = list(self.current_model.style2id.keys())
- return (
- gr.Dropdown(choices=styles, value=styles[0]),
- gr.Button(interactive=True, value="音声合成"),
- gr.Dropdown(choices=speakers, value=speakers[0]),
- )
- self.current_model = Model(
- model_path=model_path,
- config_path=self.root_dir / model_name / "config.json",
- style_vec_path=self.root_dir / model_name / "style_vectors.npy",
- device=self.device,
- )
- speakers = list(self.current_model.spk2id.keys())
- styles = list(self.current_model.style2id.keys())
- return (
- gr.Dropdown(choices=styles, value=styles[0]),
- gr.Button(interactive=True, value="音声合成"),
- gr.Dropdown(choices=speakers, value=speakers[0]),
- )
-
- def update_model_files_gr(self, model_name: str) -> gr.Dropdown:
- model_files = self.model_files_dict[model_name]
- return gr.Dropdown(choices=model_files, value=model_files[0])
-
- def update_model_names_gr(self) -> tuple[gr.Dropdown, gr.Dropdown, gr.Button]:
- self.refresh()
- initial_model_name = self.model_names[0]
- initial_model_files = self.model_files_dict[initial_model_name]
- return (
- gr.Dropdown(choices=self.model_names, value=initial_model_name),
- gr.Dropdown(choices=initial_model_files, value=initial_model_files[0]),
- gr.Button(interactive=False), # For tts_button
- )
diff --git a/commons.py b/commons.py
deleted file mode 100644
index 081b8a0..0000000
--- a/commons.py
+++ /dev/null
@@ -1,152 +0,0 @@
-import math
-import torch
-from torch.nn import functional as F
-
-
-def init_weights(m, mean=0.0, std=0.01):
- classname = m.__class__.__name__
- if classname.find("Conv") != -1:
- m.weight.data.normal_(mean, std)
-
-
-def get_padding(kernel_size, dilation=1):
- return int((kernel_size * dilation - dilation) / 2)
-
-
-def convert_pad_shape(pad_shape):
- layer = pad_shape[::-1]
- pad_shape = [item for sublist in layer for item in sublist]
- return pad_shape
-
-
-def intersperse(lst, item):
- result = [item] * (len(lst) * 2 + 1)
- result[1::2] = lst
- return result
-
-
-def kl_divergence(m_p, logs_p, m_q, logs_q):
- """KL(P||Q)"""
- kl = (logs_q - logs_p) - 0.5
- kl += (
- 0.5 * (torch.exp(2.0 * logs_p) + ((m_p - m_q) ** 2)) * torch.exp(-2.0 * logs_q)
- )
- return kl
-
-
-def rand_gumbel(shape):
- """Sample from the Gumbel distribution, protect from overflows."""
- uniform_samples = torch.rand(shape) * 0.99998 + 0.00001
- return -torch.log(-torch.log(uniform_samples))
-
-
-def rand_gumbel_like(x):
- g = rand_gumbel(x.size()).to(dtype=x.dtype, device=x.device)
- return g
-
-
-def slice_segments(x, ids_str, segment_size=4):
- gather_indices = ids_str.view(x.size(0), 1, 1).repeat(
- 1, x.size(1), 1
- ) + torch.arange(segment_size, device=x.device)
- return torch.gather(x, 2, gather_indices)
-
-
-def rand_slice_segments(x, x_lengths=None, segment_size=4):
- b, d, t = x.size()
- if x_lengths is None:
- x_lengths = t
- ids_str_max = torch.clamp(x_lengths - segment_size + 1, min=0)
- ids_str = (torch.rand([b], device=x.device) * ids_str_max).to(dtype=torch.long)
- ret = slice_segments(x, ids_str, segment_size)
- return ret, ids_str
-
-
-def get_timing_signal_1d(length, channels, min_timescale=1.0, max_timescale=1.0e4):
- position = torch.arange(length, dtype=torch.float)
- num_timescales = channels // 2
- log_timescale_increment = math.log(float(max_timescale) / float(min_timescale)) / (
- num_timescales - 1
- )
- inv_timescales = min_timescale * torch.exp(
- torch.arange(num_timescales, dtype=torch.float) * -log_timescale_increment
- )
- scaled_time = position.unsqueeze(0) * inv_timescales.unsqueeze(1)
- signal = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], 0)
- signal = F.pad(signal, [0, 0, 0, channels % 2])
- signal = signal.view(1, channels, length)
- return signal
-
-
-def add_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4):
- b, channels, length = x.size()
- signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
- return x + signal.to(dtype=x.dtype, device=x.device)
-
-
-def cat_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4, axis=1):
- b, channels, length = x.size()
- signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
- return torch.cat([x, signal.to(dtype=x.dtype, device=x.device)], axis)
-
-
-def subsequent_mask(length):
- mask = torch.tril(torch.ones(length, length)).unsqueeze(0).unsqueeze(0)
- return mask
-
-
-@torch.jit.script
-def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
- n_channels_int = n_channels[0]
- in_act = input_a + input_b
- t_act = torch.tanh(in_act[:, :n_channels_int, :])
- s_act = torch.sigmoid(in_act[:, n_channels_int:, :])
- acts = t_act * s_act
- return acts
-
-
-def shift_1d(x):
- x = F.pad(x, convert_pad_shape([[0, 0], [0, 0], [1, 0]]))[:, :, :-1]
- return x
-
-
-def sequence_mask(length, max_length=None):
- if max_length is None:
- max_length = length.max()
- x = torch.arange(max_length, dtype=length.dtype, device=length.device)
- return x.unsqueeze(0) < length.unsqueeze(1)
-
-
-def generate_path(duration, mask):
- """
- duration: [b, 1, t_x]
- mask: [b, 1, t_y, t_x]
- """
-
- b, _, t_y, t_x = mask.shape
- cum_duration = torch.cumsum(duration, -1)
-
- cum_duration_flat = cum_duration.view(b * t_x)
- path = sequence_mask(cum_duration_flat, t_y).to(mask.dtype)
- path = path.view(b, t_x, t_y)
- path = path - F.pad(path, convert_pad_shape([[0, 0], [1, 0], [0, 0]]))[:, :-1]
- path = path.unsqueeze(1).transpose(2, 3) * mask
- return path
-
-
-def clip_grad_value_(parameters, clip_value, norm_type=2):
- if isinstance(parameters, torch.Tensor):
- parameters = [parameters]
- parameters = list(filter(lambda p: p.grad is not None, parameters))
- norm_type = float(norm_type)
- if clip_value is not None:
- clip_value = float(clip_value)
-
- total_norm = 0
- for p in parameters:
- param_norm = p.grad.data.norm(norm_type)
- total_norm += param_norm.item() ** norm_type
- if clip_value is not None:
- p.grad.data.clamp_(min=-clip_value, max=clip_value)
- total_norm = total_norm ** (1.0 / norm_type)
- return total_norm
diff --git a/config.py b/config.py
index 056e7f1..40f3f53 100644
--- a/config.py
+++ b/config.py
@@ -9,7 +9,8 @@ from typing import Dict, List
import torch
import yaml
-from common.log import logger
+from style_bert_vits2.logging import logger
+
# If not cuda available, set possible devices to cpu
cuda_available = torch.cuda.is_available()
@@ -237,7 +238,7 @@ class Config:
"If you have no special needs, please do not modify default_config.yml."
)
# sys.exit(0)
- with open(file=config_path, mode="r", encoding="utf-8") as file:
+ with open(config_path, "r", encoding="utf-8") as file:
yaml_config: Dict[str, any] = yaml.safe_load(file.read())
model_name: str = yaml_config["model_name"]
self.model_name: str = model_name
diff --git a/configs/config.json b/configs/config.json
index 2f2ce7f..db0411a 100644
--- a/configs/config.json
+++ b/configs/config.json
@@ -1,5 +1,5 @@
{
- "model_name": "your_model_name",
+ "model_name": "Dummy",
"train": {
"log_interval": 200,
"eval_interval": 1000,
@@ -24,8 +24,9 @@
"freeze_encoder": false
},
"data": {
- "training_files": "Data/your_model_name/filelists/train.list",
- "validation_files": "Data/your_model_name/filelists/val.list",
+ "use_jp_extra": false,
+ "training_files": "Data/Dummy/train.list",
+ "validation_files": "Data/Dummy/val.list",
"max_wav_value": 32768.0,
"sampling_rate": 44100,
"filter_length": 2048,
@@ -68,5 +69,5 @@
"use_spectral_norm": false,
"gin_channels": 256
},
- "version": "2.3.1"
+ "version": "2.4.dev0"
}
diff --git a/configs/configs_jp_extra.json b/configs/configs_jp_extra.json
index b566be1..23c5fe5 100644
--- a/configs/configs_jp_extra.json
+++ b/configs/configs_jp_extra.json
@@ -1,4 +1,5 @@
{
+ "model_name": "Dummy",
"train": {
"log_interval": 200,
"eval_interval": 1000,
@@ -27,8 +28,8 @@
},
"data": {
"use_jp_extra": true,
- "training_files": "filelists/train.list",
- "validation_files": "filelists/val.list",
+ "training_files": "Data/Dummy/train.list",
+ "validation_files": "Data/Dummy/val.list",
"max_wav_value": 32768.0,
"sampling_rate": 44100,
"filter_length": 2048,
@@ -75,5 +76,5 @@
"initial_channel": 64
}
},
- "version": "2.3.1-JP-Extra"
+ "version": "2.4.dev0-JP-Extra"
}
diff --git a/data_utils.py b/data_utils.py
index ac038c2..73d4303 100644
--- a/data_utils.py
+++ b/data_utils.py
@@ -7,12 +7,14 @@ import torch
import torch.utils.data
from tqdm import tqdm
-import commons
from config import config
from mel_processing import mel_spectrogram_torch, spectrogram_torch
-from text import cleaned_text_to_sequence
-from common.log import logger
-from utils import load_filepaths_and_text, load_wav_to_torch
+from style_bert_vits2.logging import logger
+from style_bert_vits2.models import commons
+from style_bert_vits2.models.hyper_parameters import HyperParametersData
+from style_bert_vits2.models.utils import load_filepaths_and_text, load_wav_to_torch
+from style_bert_vits2.nlp import cleaned_text_to_sequence
+
"""Multi speaker version"""
@@ -24,7 +26,7 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
3) computes spectrograms from audio files.
"""
- def __init__(self, audiopaths_sid_text, hparams):
+ def __init__(self, audiopaths_sid_text: str, hparams: HyperParametersData):
self.audiopaths_sid_text = load_filepaths_and_text(audiopaths_sid_text)
self.max_wav_value = hparams.max_wav_value
self.sampling_rate = hparams.sampling_rate
diff --git a/default_style.py b/default_style.py
index 9198ca8..e75257d 100644
--- a/default_style.py
+++ b/default_style.py
@@ -1,9 +1,10 @@
+import json
import os
-from common.log import logger
-from common.constants import DEFAULT_STYLE
import numpy as np
-import json
+
+from style_bert_vits2.constants import DEFAULT_STYLE
+from style_bert_vits2.logging import logger
def set_style_config(json_path, output_path):
diff --git a/gen_yaml.py b/gen_yaml.py
index 91301ac..ac27103 100644
--- a/gen_yaml.py
+++ b/gen_yaml.py
@@ -1,7 +1,9 @@
+import argparse
import os
import shutil
+
import yaml
-import argparse
+
parser = argparse.ArgumentParser(
description="config.ymlの生成。あらかじめ前準備をしたデータをバッチファイルなどで連続で学習する時にtrain_ms.pyより前に使用する。"
diff --git a/infer.py b/infer.py
deleted file mode 100644
index 436ea8b..0000000
--- a/infer.py
+++ /dev/null
@@ -1,315 +0,0 @@
-import torch
-
-import commons
-from text.get_bert import get_bert
-import utils
-from models import SynthesizerTrn
-from models_jp_extra import SynthesizerTrn as SynthesizerTrnJPExtra
-from text import cleaned_text_to_sequence
-from text.cleaner import clean_text
-from text.symbols import symbols
-from common.log import logger
-
-
-class InvalidToneError(ValueError):
- pass
-
-
-def get_net_g(model_path: str, version: str, device: str, hps):
- if version.endswith("JP-Extra"):
- logger.info("Using JP-Extra model")
- net_g = SynthesizerTrnJPExtra(
- len(symbols),
- hps.data.filter_length // 2 + 1,
- hps.train.segment_size // hps.data.hop_length,
- n_speakers=hps.data.n_speakers,
- **hps.model,
- ).to(device)
- else:
- logger.info("Using normal model")
- net_g = SynthesizerTrn(
- len(symbols),
- hps.data.filter_length // 2 + 1,
- hps.train.segment_size // hps.data.hop_length,
- n_speakers=hps.data.n_speakers,
- **hps.model,
- ).to(device)
- net_g.state_dict()
- _ = net_g.eval()
- if model_path.endswith(".pth") or model_path.endswith(".pt"):
- _ = utils.load_checkpoint(model_path, net_g, None, skip_optimizer=True)
- elif model_path.endswith(".safetensors"):
- _ = utils.load_safetensors(model_path, net_g, True)
- else:
- raise ValueError(f"Unknown model format: {model_path}")
- return net_g
-
-
-def get_text(
- text,
- language_str,
- hps,
- device,
- assist_text=None,
- assist_text_weight=0.7,
- given_tone=None,
-):
- use_jp_extra = hps.version.endswith("JP-Extra")
- # 推論のときにのみ呼び出されるので、raise_yomi_errorはFalseに設定
- norm_text, phone, tone, word2ph = clean_text(
- text, language_str, use_jp_extra, raise_yomi_error=False
- )
- if given_tone is not None:
- if len(given_tone) != len(phone):
- raise InvalidToneError(
- f"Length of given_tone ({len(given_tone)}) != length of phone ({len(phone)})"
- )
- tone = given_tone
- phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
-
- if hps.data.add_blank:
- phone = commons.intersperse(phone, 0)
- tone = commons.intersperse(tone, 0)
- language = commons.intersperse(language, 0)
- for i in range(len(word2ph)):
- word2ph[i] = word2ph[i] * 2
- word2ph[0] += 1
- bert_ori = get_bert(
- norm_text,
- word2ph,
- language_str,
- device,
- assist_text,
- assist_text_weight,
- )
- del word2ph
- assert bert_ori.shape[-1] == len(phone), phone
-
- if language_str == "ZH":
- bert = bert_ori
- ja_bert = torch.zeros(1024, len(phone))
- en_bert = torch.zeros(1024, len(phone))
- elif language_str == "JP":
- bert = torch.zeros(1024, len(phone))
- ja_bert = bert_ori
- en_bert = torch.zeros(1024, len(phone))
- elif language_str == "EN":
- bert = torch.zeros(1024, len(phone))
- ja_bert = torch.zeros(1024, len(phone))
- en_bert = bert_ori
- else:
- raise ValueError("language_str should be ZH, JP or EN")
-
- assert bert.shape[-1] == len(
- phone
- ), f"Bert seq len {bert.shape[-1]} != {len(phone)}"
-
- phone = torch.LongTensor(phone)
- tone = torch.LongTensor(tone)
- language = torch.LongTensor(language)
- return bert, ja_bert, en_bert, phone, tone, language
-
-
-def infer(
- text,
- style_vec,
- sdp_ratio,
- noise_scale,
- noise_scale_w,
- length_scale,
- sid: int, # In the original Bert-VITS2, its speaker_name: str, but here it's id
- language,
- hps,
- net_g,
- device,
- skip_start=False,
- skip_end=False,
- assist_text=None,
- assist_text_weight=0.7,
- given_tone=None,
-):
- is_jp_extra = hps.version.endswith("JP-Extra")
- bert, ja_bert, en_bert, phones, tones, lang_ids = get_text(
- text,
- language,
- hps,
- device,
- assist_text=assist_text,
- assist_text_weight=assist_text_weight,
- given_tone=given_tone,
- )
- if skip_start:
- phones = phones[3:]
- tones = tones[3:]
- lang_ids = lang_ids[3:]
- bert = bert[:, 3:]
- ja_bert = ja_bert[:, 3:]
- en_bert = en_bert[:, 3:]
- if skip_end:
- phones = phones[:-2]
- tones = tones[:-2]
- lang_ids = lang_ids[:-2]
- bert = bert[:, :-2]
- ja_bert = ja_bert[:, :-2]
- en_bert = en_bert[:, :-2]
- with torch.no_grad():
- x_tst = phones.to(device).unsqueeze(0)
- tones = tones.to(device).unsqueeze(0)
- lang_ids = lang_ids.to(device).unsqueeze(0)
- bert = bert.to(device).unsqueeze(0)
- ja_bert = ja_bert.to(device).unsqueeze(0)
- en_bert = en_bert.to(device).unsqueeze(0)
- x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
- style_vec = torch.from_numpy(style_vec).to(device).unsqueeze(0)
- del phones
- sid_tensor = torch.LongTensor([sid]).to(device)
- if is_jp_extra:
- output = net_g.infer(
- x_tst,
- x_tst_lengths,
- sid_tensor,
- tones,
- lang_ids,
- ja_bert,
- style_vec=style_vec,
- sdp_ratio=sdp_ratio,
- noise_scale=noise_scale,
- noise_scale_w=noise_scale_w,
- length_scale=length_scale,
- )
- else:
- output = net_g.infer(
- x_tst,
- x_tst_lengths,
- sid_tensor,
- tones,
- lang_ids,
- bert,
- ja_bert,
- en_bert,
- style_vec=style_vec,
- sdp_ratio=sdp_ratio,
- noise_scale=noise_scale,
- noise_scale_w=noise_scale_w,
- length_scale=length_scale,
- )
- audio = output[0][0, 0].data.cpu().float().numpy()
- del (
- x_tst,
- tones,
- lang_ids,
- bert,
- x_tst_lengths,
- sid_tensor,
- ja_bert,
- en_bert,
- style_vec,
- ) # , emo
- if torch.cuda.is_available():
- torch.cuda.empty_cache()
- return audio
-
-
-def infer_multilang(
- text,
- style_vec,
- sdp_ratio,
- noise_scale,
- noise_scale_w,
- length_scale,
- sid,
- language,
- hps,
- net_g,
- device,
- skip_start=False,
- skip_end=False,
-):
- bert, ja_bert, en_bert, phones, tones, lang_ids = [], [], [], [], [], []
- # emo = get_emo_(reference_audio, emotion, sid)
- # if isinstance(reference_audio, np.ndarray):
- # emo = get_clap_audio_feature(reference_audio, device)
- # else:
- # emo = get_clap_text_feature(emotion, device)
- # emo = torch.squeeze(emo, dim=1)
- for idx, (txt, lang) in enumerate(zip(text, language)):
- _skip_start = (idx != 0) or (skip_start and idx == 0)
- _skip_end = (idx != len(language) - 1) or skip_end
- (
- temp_bert,
- temp_ja_bert,
- temp_en_bert,
- temp_phones,
- temp_tones,
- temp_lang_ids,
- ) = get_text(txt, lang, hps, device)
- if _skip_start:
- temp_bert = temp_bert[:, 3:]
- temp_ja_bert = temp_ja_bert[:, 3:]
- temp_en_bert = temp_en_bert[:, 3:]
- temp_phones = temp_phones[3:]
- temp_tones = temp_tones[3:]
- temp_lang_ids = temp_lang_ids[3:]
- if _skip_end:
- temp_bert = temp_bert[:, :-2]
- temp_ja_bert = temp_ja_bert[:, :-2]
- temp_en_bert = temp_en_bert[:, :-2]
- temp_phones = temp_phones[:-2]
- temp_tones = temp_tones[:-2]
- temp_lang_ids = temp_lang_ids[:-2]
- bert.append(temp_bert)
- ja_bert.append(temp_ja_bert)
- en_bert.append(temp_en_bert)
- phones.append(temp_phones)
- tones.append(temp_tones)
- lang_ids.append(temp_lang_ids)
- bert = torch.concatenate(bert, dim=1)
- ja_bert = torch.concatenate(ja_bert, dim=1)
- en_bert = torch.concatenate(en_bert, dim=1)
- phones = torch.concatenate(phones, dim=0)
- tones = torch.concatenate(tones, dim=0)
- lang_ids = torch.concatenate(lang_ids, dim=0)
- with torch.no_grad():
- x_tst = phones.to(device).unsqueeze(0)
- tones = tones.to(device).unsqueeze(0)
- lang_ids = lang_ids.to(device).unsqueeze(0)
- bert = bert.to(device).unsqueeze(0)
- ja_bert = ja_bert.to(device).unsqueeze(0)
- en_bert = en_bert.to(device).unsqueeze(0)
- # emo = emo.to(device).unsqueeze(0)
- x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
- del phones
- speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
- audio = (
- net_g.infer(
- x_tst,
- x_tst_lengths,
- speakers,
- tones,
- lang_ids,
- bert,
- ja_bert,
- en_bert,
- style_vec=style_vec,
- sdp_ratio=sdp_ratio,
- noise_scale=noise_scale,
- noise_scale_w=noise_scale_w,
- length_scale=length_scale,
- )[0][0, 0]
- .data.cpu()
- .float()
- .numpy()
- )
- del (
- x_tst,
- tones,
- lang_ids,
- bert,
- x_tst_lengths,
- speakers,
- ja_bert,
- en_bert,
- ) # , emo
- if torch.cuda.is_available():
- torch.cuda.empty_cache()
- return audio
diff --git a/initialize.py b/initialize.py
index 5e35061..d2680aa 100644
--- a/initialize.py
+++ b/initialize.py
@@ -5,23 +5,18 @@ from pathlib import Path
import yaml
from huggingface_hub import hf_hub_download
-from common.log import logger
+from style_bert_vits2.logging import logger
def download_bert_models():
- with open("bert/bert_models.json", "r") as fp:
+ with open("bert/bert_models.json", "r", encoding="utf-8") as fp:
models = json.load(fp)
for k, v in models.items():
local_path = Path("bert").joinpath(k)
for file in v["files"]:
if not Path(local_path).joinpath(file).exists():
logger.info(f"Downloading {k} {file}")
- hf_hub_download(
- v["repo_id"],
- file,
- local_dir=local_path,
- local_dir_use_symlinks=False,
- )
+ hf_hub_download(v["repo_id"], file, local_dir=local_path)
def download_slm_model():
@@ -29,12 +24,7 @@ def download_slm_model():
file = "pytorch_model.bin"
if not Path(local_path).joinpath(file).exists():
logger.info(f"Downloading wavlm-base-plus {file}")
- hf_hub_download(
- "microsoft/wavlm-base-plus",
- file,
- local_dir=local_path,
- local_dir_use_symlinks=False,
- )
+ hf_hub_download("microsoft/wavlm-base-plus", file, local_dir=local_path)
def download_pretrained_models():
@@ -44,10 +34,7 @@ def download_pretrained_models():
if not Path(local_path).joinpath(file).exists():
logger.info(f"Downloading pretrained {file}")
hf_hub_download(
- "litagin/Style-Bert-VITS2-1.0-base",
- file,
- local_dir=local_path,
- local_dir_use_symlinks=False,
+ "litagin/Style-Bert-VITS2-1.0-base", file, local_dir=local_path
)
@@ -58,10 +45,7 @@ def download_jp_extra_pretrained_models():
if not Path(local_path).joinpath(file).exists():
logger.info(f"Downloading JP-Extra pretrained {file}")
hf_hub_download(
- "litagin/Style-Bert-VITS2-2.0-base-JP-Extra",
- file,
- local_dir=local_path,
- local_dir_use_symlinks=False,
+ "litagin/Style-Bert-VITS2-2.0-base-JP-Extra", file, local_dir=local_path
)
diff --git a/losses.py b/losses.py
index 763cc02..9bb50af 100644
--- a/losses.py
+++ b/losses.py
@@ -2,8 +2,6 @@ import torch
import torchaudio
from transformers import AutoModel
-from common.log import logger
-
def feature_loss(fmap_r, fmap_g):
loss = 0
diff --git a/mel_processing.py b/mel_processing.py
index e9e7ec3..02cd7d4 100644
--- a/mel_processing.py
+++ b/mel_processing.py
@@ -1,7 +1,9 @@
+import warnings
+
import torch
import torch.utils.data
from librosa.filters import mel as librosa_mel_fn
-import warnings
+
# warnings.simplefilter(action='ignore', category=FutureWarning)
warnings.filterwarnings(action="ignore")
diff --git a/monotonic_align/__init__.py b/monotonic_align/__init__.py
deleted file mode 100644
index 15d8e60..0000000
--- a/monotonic_align/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-from numpy import zeros, int32, float32
-from torch import from_numpy
-
-from .core import maximum_path_jit
-
-
-def maximum_path(neg_cent, mask):
- device = neg_cent.device
- dtype = neg_cent.dtype
- neg_cent = neg_cent.data.cpu().numpy().astype(float32)
- path = zeros(neg_cent.shape, dtype=int32)
-
- t_t_max = mask.sum(1)[:, 0].data.cpu().numpy().astype(int32)
- t_s_max = mask.sum(2)[:, 0].data.cpu().numpy().astype(int32)
- maximum_path_jit(path, neg_cent, t_t_max, t_s_max)
- return from_numpy(path).to(device=device, dtype=dtype)
diff --git a/monotonic_align/core.py b/monotonic_align/core.py
deleted file mode 100644
index ffa489d..0000000
--- a/monotonic_align/core.py
+++ /dev/null
@@ -1,46 +0,0 @@
-import numba
-
-
-@numba.jit(
- numba.void(
- numba.int32[:, :, ::1],
- numba.float32[:, :, ::1],
- numba.int32[::1],
- numba.int32[::1],
- ),
- nopython=True,
- nogil=True,
-)
-def maximum_path_jit(paths, values, t_ys, t_xs):
- b = paths.shape[0]
- max_neg_val = -1e9
- for i in range(int(b)):
- path = paths[i]
- value = values[i]
- t_y = t_ys[i]
- t_x = t_xs[i]
-
- v_prev = v_cur = 0.0
- index = t_x - 1
-
- for y in range(t_y):
- for x in range(max(0, t_x + y - t_y), min(t_x, y + 1)):
- if x == y:
- v_cur = max_neg_val
- else:
- v_cur = value[y - 1, x]
- if x == 0:
- if y == 0:
- v_prev = 0.0
- else:
- v_prev = max_neg_val
- else:
- v_prev = value[y - 1, x - 1]
- value[y, x] += max(v_prev, v_cur)
-
- for y in range(t_y - 1, -1, -1):
- path[y, index] = 1
- if index != 0 and (
- index == y or value[y - 1, index] < value[y - 1, index - 1]
- ):
- index = index - 1
diff --git a/preprocess_all.py b/preprocess_all.py
index 62c0b4f..2eaf846 100644
--- a/preprocess_all.py
+++ b/preprocess_all.py
@@ -1,7 +1,17 @@
import argparse
-from webui.train import preprocess_all
from multiprocessing import cpu_count
+from style_bert_vits2.nlp.japanese import pyopenjtalk_worker
+from style_bert_vits2.nlp.japanese.user_dict import update_dict
+from webui.train import preprocess_all
+
+
+# このプロセスからはワーカーを起動して辞書を使いたいので、ここで初期化
+pyopenjtalk_worker.initialize_worker()
+
+# dict_data/ 以下の辞書データを pyopenjtalk に適用
+update_dict()
+
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
diff --git a/preprocess_text.py b/preprocess_text.py
index b3aaa17..6bc8585 100644
--- a/preprocess_text.py
+++ b/preprocess_text.py
@@ -7,10 +7,19 @@ from typing import Optional
import click
from tqdm import tqdm
-from common.log import logger
-from common.stdout_wrapper import SAFE_STDOUT
from config import config
-from text.cleaner import clean_text
+from style_bert_vits2.logging import logger
+from style_bert_vits2.nlp import clean_text
+from style_bert_vits2.nlp.japanese import pyopenjtalk_worker
+from style_bert_vits2.nlp.japanese.user_dict import update_dict
+from style_bert_vits2.utils.stdout_wrapper import SAFE_STDOUT
+
+
+# このプロセスからはワーカーを起動して辞書を使いたいので、ここで初期化
+pyopenjtalk_worker.initialize_worker()
+
+# dict_data/ 以下の辞書データを pyopenjtalk に適用
+update_dict()
preprocess_text_config = config.preprocess_text_config
@@ -72,7 +81,7 @@ def preprocess(
utt, spk, language, text = line.strip().split("|")
norm_text, phones, tones, word2ph = clean_text(
text=text,
- language=language,
+ language=language, # type: ignore
use_jp_extra=use_jp_extra,
raise_yomi_error=(yomi_error != "use"),
)
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..3045f03
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,133 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "style-bert-vits2"
+dynamic = ["version"]
+description = 'Style-Bert-VITS2: Bert-VITS2 with more controllable voice styles.'
+readme = "README.md"
+requires-python = ">=3.9"
+license = "AGPL-3.0"
+keywords = []
+authors = [
+ { name = "litagin02", email = "139731664+litagin02@users.noreply.github.com" },
+]
+classifiers = [
+ "Development Status :: 4 - Beta",
+ "Programming Language :: Python",
+ "Programming Language :: Python :: 3.9",
+ "Programming Language :: Python :: 3.10",
+ "Programming Language :: Python :: 3.11",
+ "Programming Language :: Python :: Implementation :: CPython",
+]
+dependencies = [
+ 'cmudict',
+ 'cn2an',
+ 'g2p_en',
+ 'gradio',
+ 'jieba',
+ 'librosa==0.9.2',
+ 'loguru',
+ 'num2words',
+ 'numba',
+ 'numpy',
+ 'pyannote.audio>=3.1.0',
+ 'pydantic>=2.0',
+ 'pyopenjtalk-dict',
+ 'pypinyin',
+ 'pyworld-prebuilt',
+ 'safetensors',
+ 'scipy',
+ 'torch>=2.1,<2.2',
+ 'transformers',
+]
+
+[project.urls]
+Documentation = "https://github.com/litagin02/Style-Bert-VITS2#readme"
+Issues = "https://github.com/litagin02/Style-Bert-VITS2/issues"
+Source = "https://github.com/litagin02/Style-Bert-VITS2"
+
+[tool.hatch.version]
+path = "style_bert_vits2/constants.py"
+
+[tool.hatch.build.targets.sdist]
+only-include = [
+ ".vscode",
+ "dict_data/default.csv",
+ "docs",
+ "style_bert_vits2",
+ "tests",
+ "LGPL_LICENSE",
+ "LICENSE",
+ "pyproject.toml",
+ "README.md",
+]
+exclude = [
+ ".git",
+ ".gitignore",
+ ".gitattributes",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["style_bert_vits2"]
+
+[tool.hatch.envs.test]
+dependencies = [
+ "coverage[toml]>=6.5",
+ "pytest",
+]
+[tool.hatch.envs.test.scripts]
+# Usage: `hatch run test:test`
+test = "pytest {args:tests}"
+# Usage: `hatch run test:coverage`
+test-cov = "coverage run -m pytest {args:tests}"
+# Usage: `hatch run test:cov-report`
+cov-report = [
+ "- coverage combine",
+ "coverage report",
+]
+# Usage: `hatch run test:cov`
+cov = [
+ "test-cov",
+ "cov-report",
+]
+
+[tool.hatch.envs.style]
+detached = true
+dependencies = [
+ "black",
+ "isort",
+]
+[tool.hatch.envs.style.scripts]
+check = [
+ "black --check --diff .",
+ "isort --check-only --diff --profile black --gitignore --lai 2 .",
+]
+fmt = [
+ "black .",
+ "isort --profile black --gitignore --lai 2 .",
+ "check",
+]
+
+[[tool.hatch.envs.test.matrix]]
+python = ["3.9", "3.10", "3.11"]
+
+[tool.coverage.run]
+source_pkgs = ["style_bert_vits2", "tests"]
+branch = true
+parallel = true
+omit = [
+ "style_bert_vits2/constants.py",
+]
+
+[tool.coverage.paths]
+style_bert_vits2 = ["style_bert_vits2", "*/style-bert-vits2/style_bert_vits2"]
+tests = ["tests", "*/style-bert-vits2/tests"]
+
+[tool.coverage.report]
+exclude_lines = [
+ "no cov",
+ "if __name__ == .__main__.:",
+ "if TYPE_CHECKING:",
+]
diff --git a/re_matching.py b/re_matching.py
deleted file mode 100644
index dd464a5..0000000
--- a/re_matching.py
+++ /dev/null
@@ -1,81 +0,0 @@
-import re
-
-
-def extract_language_and_text_updated(speaker, dialogue):
- # 使用正则表达式匹配<语言>标签和其后的文本
- pattern_language_text = r"<(\S+?)>([^<]+)"
- matches = re.findall(pattern_language_text, dialogue, re.DOTALL)
- speaker = speaker[1:-1]
- # 清理文本：去除两边的空白字符
- matches_cleaned = [(lang.upper(), text.strip()) for lang, text in matches]
- matches_cleaned.append(speaker)
- return matches_cleaned
-
-
-def validate_text(input_text):
- # 验证说话人的正则表达式
- pattern_speaker = r"(\[\S+?\])((?:\s*<\S+?>[^<\[\]]+?)+)"
-
- # 使用re.DOTALL标志使.匹配包括换行符在内的所有字符
- matches = re.findall(pattern_speaker, input_text, re.DOTALL)
-
- # 对每个匹配到的说话人内容进行进一步验证
- for _, dialogue in matches:
- language_text_matches = extract_language_and_text_updated(_, dialogue)
- if not language_text_matches:
- return (
- False,
- "Error: Invalid format detected in dialogue content. Please check your input.",
- )
-
- # 如果输入的文本中没有找到任何匹配项
- if not matches:
- return (
- False,
- "Error: No valid speaker format detected. Please check your input.",
- )
-
- return True, "Input is valid."
-
-
-def text_matching(text: str) -> list:
- speaker_pattern = r"(\[\S+?\])(.+?)(?=\[\S+?\]|$)"
- matches = re.findall(speaker_pattern, text, re.DOTALL)
- result = []
- for speaker, dialogue in matches:
- result.append(extract_language_and_text_updated(speaker, dialogue))
- return result
-
-
-def cut_para(text):
- splitted_para = re.split("[\n]", text) # 按段分
- splitted_para = [
- sentence.strip() for sentence in splitted_para if sentence.strip()
- ] # 删除空字符串
- return splitted_para
-
-
-def cut_sent(para):
- para = re.sub("([。！;？\?])([^”’])", r"\1\n\2", para) # 单字符断句符
- para = re.sub("(\.{6})([^”’])", r"\1\n\2", para) # 英文省略号
- para = re.sub("(\…{2})([^”’])", r"\1\n\2", para) # 中文省略号
- para = re.sub("([。！？\?][”’])([^，。！？\?])", r"\1\n\2", para)
- para = para.rstrip() # 段尾如果有多余的\n就去掉它
- return para.split("\n")
-
-
-if __name__ == "__main__":
- text = """
- [说话人1]
- [说话人2]你好吗？元気ですか？こんにちは，世界。你好吗？
- [说话人3]谢谢。どういたしまして。
- """
- text_matching(text)
- # 测试函数
- test_text = """
- [说话人1]你好，こんにちは！こんにちは，世界。
- [说话人2]你好吗？
- """
- text_matching(test_text)
- res = validate_text(test_text)
- print(res)
diff --git a/requirements.txt b/requirements.txt
index 6bf329f..bf47ec0 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -14,11 +14,12 @@ numba
numpy
psutil
pyannote.audio>=3.1.0
+pydantic>=2.0
pyloudnorm
# pyopenjtalk-prebuilt # Should be manually uninstalled
pyopenjtalk-dict
pypinyin
-# pyworld # Not supported on Windows without Cython...
+pyworld-prebuilt
PyYAML
requests
safetensors
diff --git a/resample.py b/resample.py
index 5aff5ef..5c3cc79 100644
--- a/resample.py
+++ b/resample.py
@@ -1,15 +1,17 @@
import argparse
import os
from concurrent.futures import ThreadPoolExecutor
+from multiprocessing import cpu_count
import librosa
import pyloudnorm as pyln
import soundfile
from tqdm import tqdm
-from common.log import logger
-from common.stdout_wrapper import SAFE_STDOUT
from config import config
+from style_bert_vits2.logging import logger
+from style_bert_vits2.utils.stdout_wrapper import SAFE_STDOUT
+
DEFAULT_BLOCK_SIZE: float = 0.400 # seconds
diff --git a/scripts/Update-Style-Bert-VITS2.bat b/scripts/Update-Style-Bert-VITS2.bat
index b4f8b8d..04f5937 100644
--- a/scripts/Update-Style-Bert-VITS2.bat
+++ b/scripts/Update-Style-Bert-VITS2.bat
@@ -10,12 +10,12 @@ if not exist %CURL_CMD% (
pause & popd & exit /b 1
)
-@REM Style-Bert-VITS2.zip をGitHubのmasterの最新のものをダウンロード
+@REM Style-Bert-VITS2.zip をGitHubのdev-refactorの最新のものをダウンロード
%CURL_CMD% -Lo Style-Bert-VITS2.zip^
- https://github.com/litagin02/Style-Bert-VITS2/archive/refs/heads/master.zip
+ https://github.com/litagin02/Style-Bert-VITS2/archive/refs/heads/dev-refactor.zip
if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% )
-@REM Style-Bert-VITS2.zip を解凍（フォルダ名前がBert-VITS2-masterになる）
+@REM Style-Bert-VITS2.zip を解凍（フォルダ名前がBert-VITS2-dev-refactorになる）
%PS_CMD% Expand-Archive -Path Style-Bert-VITS2.zip -DestinationPath . -Force
if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% )
@@ -23,9 +23,9 @@ if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% )
del Style-Bert-VITS2.zip
if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% )
-@REM Bert-VITS2-masterの中身をStyle-Bert-VITS2に上書き移動
-xcopy /QSY .\Style-Bert-VITS2-master\ .\Style-Bert-VITS2\
-rmdir /s /q Style-Bert-VITS2-master
+@REM Bert-VITS2-dev-refactorの中身をStyle-Bert-VITS2に上書き移動
+xcopy /QSY .\Style-Bert-VITS2-dev-refactor\ .\Style-Bert-VITS2\
+rmdir /s /q Style-Bert-VITS2-dev-refactor
if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% )
@REM 仮想環境のpip requirements.txtを更新
@@ -39,9 +39,14 @@ if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% )
echo Update completed. Running Style-Bert-VITS2 Editor...
+@REM Style-Bert-VITS2フォルダに移動
+pushd Style-Bert-VITS2
+
@REM Style-Bert-VITS2 Editorを起動
python server_editor.py --inbrowser
pause
-popd
\ No newline at end of file
+popd
+
+popd
diff --git a/server_editor.py b/server_editor.py
index 402116f..c6f856e 100644
--- a/server_editor.py
+++ b/server_editor.py
@@ -30,20 +30,30 @@ from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel
from scipy.io import wavfile
-from common.constants import (
+from style_bert_vits2.constants import (
DEFAULT_ASSIST_TEXT_WEIGHT,
DEFAULT_NOISE,
DEFAULT_NOISEW,
DEFAULT_SDP_RATIO,
DEFAULT_STYLE,
DEFAULT_STYLE_WEIGHT,
- LATEST_VERSION,
+ VERSION,
Languages,
)
-from common.log import logger
-from common.tts_model import ModelHolder
-from text.japanese import g2kata_tone, kata_tone2phone_tone, text_normalize
-from text.user_dict import apply_word, delete_word, read_dict, rewrite_word, update_dict
+from style_bert_vits2.logging import logger
+from style_bert_vits2.nlp import bert_models
+from style_bert_vits2.nlp.japanese import pyopenjtalk_worker as pyopenjtalk
+from style_bert_vits2.nlp.japanese.g2p_utils import g2kata_tone, kata_tone2phone_tone
+from style_bert_vits2.nlp.japanese.normalizer import normalize_text
+from style_bert_vits2.nlp.japanese.user_dict import (
+ apply_word,
+ delete_word,
+ read_dict,
+ rewrite_word,
+ update_dict,
+)
+from style_bert_vits2.tts_model import TTSModelHolder, TTSModelInfo
+
# ---フロントエンド部分に関する処理---
@@ -139,6 +149,19 @@ def save_last_download(latest_release):
# ---フロントエンド部分に関する処理ここまで---
# 以降はAPIの設定
+# pyopenjtalk_worker を起動
+## pyopenjtalk_worker は TCP ソケットサーバーのため、ここで起動する
+pyopenjtalk.initialize_worker()
+
+# pyopenjtalk の辞書を更新
+update_dict()
+
+# 事前に BERT モデル/トークナイザーをロードしておく
+## ここでロードしなくても必要になった際に自動ロードされるが、時間がかかるため事前にロードしておいた方が体験が良い
+## server_editor.py は日本語にしか対応していないため、日本語の BERT モデル/トークナイザーのみロードする
+bert_models.load_model(Languages.JP)
+bert_models.load_tokenizer(Languages.JP)
+
class AudioResponse(Response):
media_type = "audio/wav"
@@ -175,7 +198,7 @@ if device == "cuda" and not torch.cuda.is_available():
model_dir = Path(args.model_dir)
port = int(args.port)
-model_holder = ModelHolder(model_dir, device)
+model_holder = TTSModelHolder(model_dir, device)
if len(model_holder.model_names) == 0:
logger.error(f"Models not found in {model_dir}.")
sys.exit(1)
@@ -196,7 +219,7 @@ router = APIRouter()
@router.get("/version")
def version() -> str:
- return LATEST_VERSION
+ return VERSION
class MoraTone(BaseModel):
@@ -212,7 +235,7 @@ class TextRequest(BaseModel):
async def read_item(item: TextRequest):
try:
# 最初に正規化しないと整合性がとれない
- text = text_normalize(item.text)
+ text = normalize_text(item.text)
kata_tone_list = g2kata_tone(text)
except Exception as e:
raise HTTPException(
@@ -223,11 +246,11 @@ async def read_item(item: TextRequest):
@router.post("/normalize")
-async def normalize_text(item: TextRequest):
- return text_normalize(item.text)
+async def normalize(item: TextRequest):
+ return normalize_text(item.text)
-@router.get("/models_info")
+@router.get("/models_info", response_model=list[TTSModelInfo])
def models_info():
return model_holder.models_info
@@ -260,7 +283,7 @@ def synthesis(request: SynthesisRequest):
detail=f"1行の文字数は{args.line_length}文字以下にしてください。",
)
try:
- model = model_holder.load_model(
+ model = model_holder.get_model(
model_name=request.model, model_path_str=request.modelFile
)
except Exception as e:
@@ -284,10 +307,10 @@ def synthesis(request: SynthesisRequest):
)
sr, audio = model.infer(
text=text,
- language=request.language.value,
+ language=request.language,
sdp_ratio=request.sdpRatio,
noise=request.noise,
- noisew=request.noisew,
+ noise_w=request.noisew,
length=1 / request.speed,
given_tone=tone,
style=request.style,
@@ -298,7 +321,7 @@ def synthesis(request: SynthesisRequest):
line_split=False,
pitch_scale=request.pitchScale,
intonation_scale=request.intonationScale,
- sid=sid,
+ speaker_id=sid,
)
with BytesIO() as wavContent:
@@ -319,6 +342,7 @@ def multi_synthesis(request: MultiSynthesisRequest):
detail=f"行数は{args.line_count}行以下にしてください。",
)
audios = []
+ sr = None
for i, req in enumerate(lines):
if args.line_length is not None and len(req.text) > args.line_length:
raise HTTPException(
@@ -326,7 +350,7 @@ def multi_synthesis(request: MultiSynthesisRequest):
detail=f"1行の文字数は{args.line_length}文字以下にしてください。",
)
try:
- model = model_holder.load_model(
+ model = model_holder.get_model(
model_name=req.model, model_path_str=req.modelFile
)
except Exception as e:
@@ -343,10 +367,10 @@ def multi_synthesis(request: MultiSynthesisRequest):
tone = [t for _, t in phone_tone]
sr, audio = model.infer(
text=text,
- language=req.language.value,
+ language=req.language,
sdp_ratio=req.sdpRatio,
noise=req.noise,
- noisew=req.noisew,
+ noise_w=req.noisew,
length=1 / req.speed,
given_tone=tone,
style=req.style,
diff --git a/server_fastapi.py b/server_fastapi.py
index 5a68b8b..8c5a7cc 100644
--- a/server_fastapi.py
+++ b/server_fastapi.py
@@ -8,7 +8,7 @@ import os
import sys
from io import BytesIO
from pathlib import Path
-from typing import Dict, Optional, Union
+from typing import Any, Optional
from urllib.parse import unquote
import GPUtil
@@ -20,7 +20,8 @@ from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import FileResponse, Response
from scipy.io import wavfile
-from common.constants import (
+from config import config
+from style_bert_vits2.constants import (
DEFAULT_ASSIST_TEXT_WEIGHT,
DEFAULT_LENGTH,
DEFAULT_LINE_SPLIT,
@@ -32,13 +33,33 @@ from common.constants import (
DEFAULT_STYLE_WEIGHT,
Languages,
)
-from common.log import logger
-from common.tts_model import Model, ModelHolder
-from config import config
+from style_bert_vits2.logging import logger
+from style_bert_vits2.nlp import bert_models
+from style_bert_vits2.nlp.japanese import pyopenjtalk_worker as pyopenjtalk
+from style_bert_vits2.nlp.japanese.user_dict import update_dict
+from style_bert_vits2.tts_model import TTSModel, TTSModelHolder
+
ln = config.server_config.language
+# pyopenjtalk_worker を起動
+## pyopenjtalk_worker は TCP ソケットサーバーのため、ここで起動する
+pyopenjtalk.initialize_worker()
+
+# dict_data/ 以下の辞書データを pyopenjtalk に適用
+update_dict()
+
+# 事前に BERT モデル/トークナイザーをロードしておく
+## ここでロードしなくても必要になった際に自動ロードされるが、時間がかかるため事前にロードしておいた方が体験が良い
+bert_models.load_model(Languages.JP)
+bert_models.load_tokenizer(Languages.JP)
+bert_models.load_model(Languages.EN)
+bert_models.load_tokenizer(Languages.EN)
+bert_models.load_model(Languages.ZH)
+bert_models.load_tokenizer(Languages.ZH)
+
+
def raise_validation_error(msg: str, param: str):
logger.warning(f"Validation error: {msg}")
raise HTTPException(
@@ -51,17 +72,22 @@ class AudioResponse(Response):
media_type = "audio/wav"
-def load_models(model_holder: ModelHolder):
- model_holder.models = []
+loaded_models: list[TTSModel] = []
+
+
+def load_models(model_holder: TTSModelHolder):
+ global loaded_models
+ loaded_models = []
for model_name, model_paths in model_holder.model_files_dict.items():
- model = Model(
+ model = TTSModel(
model_path=model_paths[0],
config_path=model_holder.root_dir / model_name / "config.json",
style_vec_path=model_holder.root_dir / model_name / "style_vectors.npy",
device=model_holder.device,
)
- model.load_net_g()
- model_holder.models.append(model)
+ # 起動時に全てのモデルを読み込むのは時間がかかりメモリを食うのでやめる
+ # model.load()
+ loaded_models.append(model)
if __name__ == "__main__":
@@ -78,13 +104,14 @@ if __name__ == "__main__":
device = "cuda" if torch.cuda.is_available() else "cpu"
model_dir = Path(args.dir)
- model_holder = ModelHolder(model_dir, device)
+ model_holder = TTSModelHolder(model_dir, device)
if len(model_holder.model_names) == 0:
logger.error(f"Models not found in {model_dir}.")
sys.exit(1)
logger.info("Loading models...")
load_models(model_holder)
+
limit = config.server_config.limit
app = FastAPI()
allow_origins = config.server_config.origins
@@ -99,12 +126,13 @@ if __name__ == "__main__":
allow_methods=["*"],
allow_headers=["*"],
)
- app.logger = logger
+ # app.logger = logger
+ # ↑効いていなさそう。loggerをどうやって上書きするかはよく分からなかった。
@app.post("/voice", response_class=AudioResponse)
async def voice(
request: Request,
- text: str = Query(..., min_length=1, max_length=limit, description=f"セリフ"),
+ text: str = Query(..., min_length=1, max_length=limit, description="セリフ"),
encoding: str = Query(None, description="textをURLデコードする(ex, `utf-8`)"),
model_id: int = Query(
0, description="モデルID。`GET /models/info`のkeyの値を指定ください"
@@ -132,7 +160,7 @@ if __name__ == "__main__":
DEFAULT_LENGTH,
description="話速。基準は1で大きくするほど音声は長くなり読み上げが遅まる",
),
- language: Languages = Query(ln, description=f"textの言語"),
+ language: Languages = Query(ln, description="textの言語"),
auto_split: bool = Query(DEFAULT_LINE_SPLIT, description="改行で分けて生成"),
split_interval: float = Query(
DEFAULT_SPLIT_INTERVAL, description="分けた場合に挟む無音の長さ（秒）"
@@ -144,7 +172,7 @@ if __name__ == "__main__":
assist_text_weight: float = Query(
DEFAULT_ASSIST_TEXT_WEIGHT, description="assist_textの強さ"
),
- style: Optional[Union[int, str]] = Query(DEFAULT_STYLE, description="スタイル"),
+ style: Optional[str] = Query(DEFAULT_STYLE, description="スタイル"),
style_weight: float = Query(DEFAULT_STYLE_WEIGHT, description="スタイルの強さ"),
reference_audio_path: Optional[str] = Query(
None, description="スタイルを音声ファイルで行う"
@@ -155,11 +183,11 @@ if __name__ == "__main__":
f"{request.client.host}:{request.client.port}/voice { unquote(str(request.query_params) )}"
)
if model_id >= len(
- model_holder.models
+ model_holder.model_names
): # /models/refresh があるためQuery(le)で表現不可
raise_validation_error(f"model_id={model_id} not found", "model_id")
- model = model_holder.models[model_id]
+ model = loaded_models[model_id]
if speaker_name is None:
if speaker_id not in model.id2spk.keys():
raise_validation_error(
@@ -173,16 +201,17 @@ if __name__ == "__main__":
speaker_id = model.spk2id[speaker_name]
if style not in model.style2id.keys():
raise_validation_error(f"style={style} not found", "style")
+ assert style is not None
if encoding is not None:
text = unquote(text, encoding=encoding)
sr, audio = model.infer(
text=text,
language=language,
- sid=speaker_id,
+ speaker_id=speaker_id,
reference_audio_path=reference_audio_path,
sdp_ratio=sdp_ratio,
noise=noise,
- noisew=noisew,
+ noise_w=noisew,
length=length,
line_split=auto_split,
split_interval=split_interval,
@@ -201,8 +230,8 @@ if __name__ == "__main__":
def get_loaded_models_info():
"""ロードされたモデル情報の取得"""
- result: Dict[str, Dict] = dict()
- for model_id, model in enumerate(model_holder.models):
+ result: dict[str, dict[str, Any]] = dict()
+ for model_id, model in enumerate(loaded_models):
result[str(model_id)] = {
"config_path": model.config_path,
"model_path": model.model_path,
diff --git a/slice.py b/slice.py
index 2d56427..4a86a3d 100644
--- a/slice.py
+++ b/slice.py
@@ -1,5 +1,4 @@
import argparse
-import os
import shutil
from pathlib import Path
@@ -8,8 +7,11 @@ import torch
import yaml
from tqdm import tqdm
-from common.log import logger
-from common.stdout_wrapper import SAFE_STDOUT
+from style_bert_vits2.logging import logger
+from style_bert_vits2.utils.stdout_wrapper import SAFE_STDOUT
+
+
+# TODO: 並列処理による高速化
vad_model, utils = torch.hub.load(
repo_or_dir="snakers4/silero-vad",
@@ -22,7 +24,10 @@ vad_model, utils = torch.hub.load(
def get_stamps(
- audio_file, min_silence_dur_ms: int = 700, min_sec: float = 2, max_sec: float = 12
+ audio_file: Path,
+ min_silence_dur_ms: int = 700,
+ min_sec: float = 2,
+ max_sec: float = 12,
):
"""
min_silence_dur_ms: int (ミリ秒):
@@ -41,7 +46,7 @@ def get_stamps(
min_ms = int(min_sec * 1000)
- wav = read_audio(audio_file, sampling_rate=sampling_rate)
+ wav = read_audio(str(audio_file), sampling_rate=sampling_rate)
speech_timestamps = get_speech_timestamps(
wav,
vad_model,
@@ -55,13 +60,13 @@ def get_stamps(
def split_wav(
- audio_file,
- target_dir="raw",
- min_sec=2,
- max_sec=12,
- min_silence_dur_ms=700,
-):
- margin = 200 # ミリ秒単位で、音声の前後に余裕を持たせる
+ audio_file: Path,
+ target_dir: Path,
+ min_sec: float = 2,
+ max_sec: float = 12,
+ min_silence_dur_ms: int = 700,
+) -> tuple[float, int]:
+ margin: int = 200 # ミリ秒単位で、音声の前後に余裕を持たせる
speech_timestamps = get_stamps(
audio_file,
min_silence_dur_ms=min_silence_dur_ms,
@@ -73,10 +78,10 @@ def split_wav(
total_ms = len(data) / sr * 1000
- file_name = os.path.basename(audio_file).split(".")[0]
- os.makedirs(target_dir, exist_ok=True)
+ file_name = audio_file.stem
+ target_dir.mkdir(parents=True, exist_ok=True)
- total_time_ms = 0
+ total_time_ms: float = 0
count = 0
# タイムスタンプに従って分割し、ファイルに保存
@@ -88,7 +93,7 @@ def split_wav(
end_sample = int(end_ms / 1000 * sr)
segment = data[start_sample:end_sample]
- sf.write(os.path.join(target_dir, f"{file_name}-{i}.wav"), segment, sr)
+ sf.write(str(target_dir / f"{file_name}-{i}.wav"), segment, sr)
total_time_ms += end_ms - start_ms
count += 1
@@ -125,20 +130,21 @@ if __name__ == "__main__":
)
args = parser.parse_args()
- with open(os.path.join("configs", "paths.yml"), "r", encoding="utf-8") as f:
+ with open(Path("configs/paths.yml"), "r", encoding="utf-8") as f:
path_config: dict[str, str] = yaml.safe_load(f.read())
dataset_root = path_config["dataset_root"]
- input_dir = args.input_dir
- output_dir = os.path.join(dataset_root, args.model_name, "raw")
- min_sec = args.min_sec
- max_sec = args.max_sec
- min_silence_dur_ms = args.min_silence_dur_ms
+ model_name = str(args.model_name)
+ input_dir = Path(args.input_dir)
+ output_dir = Path(dataset_root) / model_name / "raw"
+ min_sec: float = args.min_sec
+ max_sec: float = args.max_sec
+ min_silence_dur_ms: int = args.min_silence_dur_ms
wav_files = Path(input_dir).glob("**/*.wav")
wav_files = list(wav_files)
logger.info(f"Found {len(wav_files)} wav files.")
- if os.path.exists(output_dir):
+ if output_dir.exists():
logger.warning(f"Output directory {output_dir} already exists, deleting...")
shutil.rmtree(output_dir)
@@ -146,7 +152,7 @@ if __name__ == "__main__":
total_count = 0
for wav_file in tqdm(wav_files, file=SAFE_STDOUT):
time_sec, count = split_wav(
- audio_file=str(wav_file),
+ audio_file=wav_file,
target_dir=output_dir,
min_sec=min_sec,
max_sec=max_sec,
diff --git a/spec_gen.py b/spec_gen.py
deleted file mode 100644
index b6715fa..0000000
--- a/spec_gen.py
+++ /dev/null
@@ -1,87 +0,0 @@
-import torch
-from tqdm import tqdm
-from multiprocessing import Pool
-from mel_processing import spectrogram_torch, mel_spectrogram_torch
-from utils import load_wav_to_torch
-
-
-class AudioProcessor:
- def __init__(
- self,
- max_wav_value,
- use_mel_spec_posterior,
- filter_length,
- n_mel_channels,
- sampling_rate,
- hop_length,
- win_length,
- mel_fmin,
- mel_fmax,
- ):
- self.max_wav_value = max_wav_value
- self.use_mel_spec_posterior = use_mel_spec_posterior
- self.filter_length = filter_length
- self.n_mel_channels = n_mel_channels
- self.sampling_rate = sampling_rate
- self.hop_length = hop_length
- self.win_length = win_length
- self.mel_fmin = mel_fmin
- self.mel_fmax = mel_fmax
-
- def process_audio(self, filename):
- audio, sampling_rate = load_wav_to_torch(filename)
- audio_norm = audio / self.max_wav_value
- audio_norm = audio_norm.unsqueeze(0)
- spec_filename = filename.replace(".wav", ".spec.pt")
- if self.use_mel_spec_posterior:
- spec_filename = spec_filename.replace(".spec.pt", ".mel.pt")
- try:
- spec = torch.load(spec_filename)
- except:
- if self.use_mel_spec_posterior:
- spec = mel_spectrogram_torch(
- audio_norm,
- self.filter_length,
- self.n_mel_channels,
- self.sampling_rate,
- self.hop_length,
- self.win_length,
- self.mel_fmin,
- self.mel_fmax,
- center=False,
- )
- else:
- spec = spectrogram_torch(
- audio_norm,
- self.filter_length,
- self.sampling_rate,
- self.hop_length,
- self.win_length,
- center=False,
- )
- spec = torch.squeeze(spec, 0)
- torch.save(spec, spec_filename)
- return spec, audio_norm
-
-
-# 使用示例
-processor = AudioProcessor(
- max_wav_value=32768.0,
- use_mel_spec_posterior=False,
- filter_length=2048,
- n_mel_channels=128,
- sampling_rate=44100,
- hop_length=512,
- win_length=2048,
- mel_fmin=0.0,
- mel_fmax="null",
-)
-
-with open("filelists/train.list", "r") as f:
- filepaths = [line.split("|")[0] for line in f] # 取每一行的第一部分作为audiopath
-
-# 使用多进程处理
-with Pool(processes=32) as pool: # 使用4个进程
- with tqdm(total=len(filepaths)) as pbar:
- for i, _ in enumerate(pool.imap_unordered(processor.process_audio, filepaths)):
- pbar.update()
diff --git a/speech_mos.py b/speech_mos.py
index d69a23a..453b7d3 100644
--- a/speech_mos.py
+++ b/speech_mos.py
@@ -10,9 +10,10 @@ import pandas as pd
import torch
from tqdm import tqdm
-from common.log import logger
-from common.tts_model import Model
from config import config
+from style_bert_vits2.logging import logger
+from style_bert_vits2.tts_model import TTSModel
+
warnings.filterwarnings("ignore")
@@ -54,7 +55,7 @@ safetensors_files = model_path.glob("*.safetensors")
def get_model(model_file: Path):
- return Model(
+ return TTSModel(
model_path=str(model_file),
config_path=str(model_file.parent / "config.json"),
style_vec_path=str(model_file.parent / "style_vectors.npy"),
diff --git a/style_bert_vits2/.editorconfig b/style_bert_vits2/.editorconfig
new file mode 100644
index 0000000..bbc8a70
--- /dev/null
+++ b/style_bert_vits2/.editorconfig
@@ -0,0 +1,15 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+indent_size = 4
+indent_style = space
+trim_trailing_whitespace = true
+
+[*.md]
+trim_trailing_whitespace = false
+
+[*.yml]
+indent_size = 2
diff --git a/style_bert_vits2/__init__.py b/style_bert_vits2/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/style_bert_vits2/constants.py b/style_bert_vits2/constants.py
new file mode 100644
index 0000000..aa14331
--- /dev/null
+++ b/style_bert_vits2/constants.py
@@ -0,0 +1,48 @@
+from pathlib import Path
+
+from style_bert_vits2.utils.strenum import StrEnum
+
+
+# Style-Bert-VITS2 のバージョン
+VERSION = "2.4.dev0"
+
+# Style-Bert-VITS2 のベースディレクトリ
+BASE_DIR = Path(__file__).parent.parent
+
+
+# 利用可能な言語
+## JP-Extra モデル利用時は JP 以外の言語の音声合成はできない
+class Languages(StrEnum):
+ JP = "JP"
+ EN = "EN"
+ ZH = "ZH"
+
+
+# 言語ごとのデフォルトの BERT トークナイザーのパス
+DEFAULT_BERT_TOKENIZER_PATHS = {
+ Languages.JP: BASE_DIR / "bert" / "deberta-v2-large-japanese-char-wwm",
+ Languages.EN: BASE_DIR / "bert" / "deberta-v3-large",
+ Languages.ZH: BASE_DIR / "bert" / "chinese-roberta-wwm-ext-large",
+}
+
+# デフォルトのユーザー辞書ディレクトリ
+## style_bert_vits2.nlp.japanese.user_dict モジュールのデフォルト値として利用される
+## ライブラリとしての利用などで外部のユーザー辞書を指定したい場合は、user_dict 以下の各関数の実行時、引数に辞書データファイルのパスを指定する
+DEFAULT_USER_DICT_DIR = BASE_DIR / "dict_data"
+
+# デフォルトの推論パラメータ
+DEFAULT_STYLE = "Neutral"
+DEFAULT_STYLE_WEIGHT = 5.0
+DEFAULT_SDP_RATIO = 0.2
+DEFAULT_NOISE = 0.6
+DEFAULT_NOISEW = 0.8
+DEFAULT_LENGTH = 1.0
+DEFAULT_LINE_SPLIT = True
+DEFAULT_SPLIT_INTERVAL = 0.5
+DEFAULT_ASSIST_TEXT_WEIGHT = 0.7
+DEFAULT_ASSIST_TEXT_WEIGHT = 1.0
+
+# Gradio のテーマ
+## Built-in theme: "default", "base", "monochrome", "soft", "glass"
+## See https://huggingface.co/spaces/gradio/theme-gallery for more themes
+GRADIO_THEME = "NoCrypt/miku"
diff --git a/style_bert_vits2/logging.py b/style_bert_vits2/logging.py
new file mode 100644
index 0000000..e5c216a
--- /dev/null
+++ b/style_bert_vits2/logging.py
@@ -0,0 +1,15 @@
+from loguru import logger
+
+from style_bert_vits2.utils.stdout_wrapper import SAFE_STDOUT
+
+
+# Remove all default handlers
+logger.remove()
+
+# Add a new handler
+logger.add(
+ SAFE_STDOUT,
+ format="{time:MM-DD HH:mm:ss} |{level:^8}| {file}:{line} | {message}",
+ backtrace=True,
+ diagnose=True,
+)
diff --git a/style_bert_vits2/models/__init__.py b/style_bert_vits2/models/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/attentions.py b/style_bert_vits2/models/attentions.py
similarity index 82%
rename from attentions.py
rename to style_bert_vits2/models/attentions.py
index 9a4bba9..b851b53 100644
--- a/attentions.py
+++ b/style_bert_vits2/models/attentions.py
@@ -1,14 +1,15 @@
import math
+from typing import Any, Optional
+
import torch
from torch import nn
from torch.nn import functional as F
-import commons
-from common.log import logger as logging
+from style_bert_vits2.models import commons
class LayerNorm(nn.Module):
- def __init__(self, channels, eps=1e-5):
+ def __init__(self, channels: int, eps: float = 1e-5) -> None:
super().__init__()
self.channels = channels
self.eps = eps
@@ -16,14 +17,16 @@ class LayerNorm(nn.Module):
self.gamma = nn.Parameter(torch.ones(channels))
self.beta = nn.Parameter(torch.zeros(channels))
- def forward(self, x):
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
x = x.transpose(1, -1)
x = F.layer_norm(x, (self.channels,), self.gamma, self.beta, self.eps)
return x.transpose(1, -1)
-@torch.jit.script
-def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
+@torch.jit.script # type: ignore
+def fused_add_tanh_sigmoid_multiply(
+ input_a: torch.Tensor, input_b: torch.Tensor, n_channels: list[int]
+) -> torch.Tensor:
n_channels_int = n_channels[0]
in_act = input_a + input_b
t_act = torch.tanh(in_act[:, :n_channels_int, :])
@@ -35,16 +38,16 @@ def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
class Encoder(nn.Module):
def __init__(
self,
- hidden_channels,
- filter_channels,
- n_heads,
- n_layers,
- kernel_size=1,
- p_dropout=0.0,
- window_size=4,
- isflow=True,
- **kwargs
- ):
+ hidden_channels: int,
+ filter_channels: int,
+ n_heads: int,
+ n_layers: int,
+ kernel_size: int = 1,
+ p_dropout: float = 0.0,
+ window_size: int = 4,
+ isflow: bool = True,
+ **kwargs: Any,
+ ) -> None:
super().__init__()
self.hidden_channels = hidden_channels
self.filter_channels = filter_channels
@@ -67,7 +70,7 @@ class Encoder(nn.Module):
self.cond_layer_idx = (
kwargs["cond_layer_idx"] if "cond_layer_idx" in kwargs else 2
)
- # logging.debug(self.gin_channels, self.cond_layer_idx)
+ # logger.debug(self.gin_channels, self.cond_layer_idx)
assert (
self.cond_layer_idx < self.n_layers
), "cond_layer_idx should be less than n_layers"
@@ -98,12 +101,15 @@ class Encoder(nn.Module):
)
self.norm_layers_2.append(LayerNorm(hidden_channels))
- def forward(self, x, x_mask, g=None):
+ def forward(
+ self, x: torch.Tensor, x_mask: torch.Tensor, g: Optional[torch.Tensor] = None
+ ) -> torch.Tensor:
attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
x = x * x_mask
for i in range(self.n_layers):
if i == self.cond_layer_idx and g is not None:
g = self.spk_emb_linear(g.transpose(1, 2))
+ assert g is not None
g = g.transpose(1, 2)
x = x + g
x = x * x_mask
@@ -121,16 +127,16 @@ class Encoder(nn.Module):
class Decoder(nn.Module):
def __init__(
self,
- hidden_channels,
- filter_channels,
- n_heads,
- n_layers,
- kernel_size=1,
- p_dropout=0.0,
- proximal_bias=False,
- proximal_init=True,
- **kwargs
- ):
+ hidden_channels: int,
+ filter_channels: int,
+ n_heads: int,
+ n_layers: int,
+ kernel_size: int = 1,
+ p_dropout: float = 0.0,
+ proximal_bias: bool = False,
+ proximal_init: bool = True,
+ **kwargs: Any,
+ ) -> None:
super().__init__()
self.hidden_channels = hidden_channels
self.filter_channels = filter_channels
@@ -178,7 +184,13 @@ class Decoder(nn.Module):
)
self.norm_layers_2.append(LayerNorm(hidden_channels))
- def forward(self, x, x_mask, h, h_mask):
+ def forward(
+ self,
+ x: torch.Tensor,
+ x_mask: torch.Tensor,
+ h: torch.Tensor,
+ h_mask: torch.Tensor,
+ ) -> torch.Tensor:
"""
x: decoder input
h: encoder output
@@ -207,16 +219,16 @@ class Decoder(nn.Module):
class MultiHeadAttention(nn.Module):
def __init__(
self,
- channels,
- out_channels,
- n_heads,
- p_dropout=0.0,
- window_size=None,
- heads_share=True,
- block_length=None,
- proximal_bias=False,
- proximal_init=False,
- ):
+ channels: int,
+ out_channels: int,
+ n_heads: int,
+ p_dropout: float = 0.0,
+ window_size: Optional[int] = None,
+ heads_share: bool = True,
+ block_length: Optional[int] = None,
+ proximal_bias: bool = False,
+ proximal_init: bool = False,
+ ) -> None:
super().__init__()
assert channels % n_heads == 0
@@ -256,9 +268,13 @@ class MultiHeadAttention(nn.Module):
if proximal_init:
with torch.no_grad():
self.conv_k.weight.copy_(self.conv_q.weight)
+ assert self.conv_k.bias is not None
+ assert self.conv_q.bias is not None
self.conv_k.bias.copy_(self.conv_q.bias)
- def forward(self, x, c, attn_mask=None):
+ def forward(
+ self, x: torch.Tensor, c: torch.Tensor, attn_mask: Optional[torch.Tensor] = None
+ ) -> torch.Tensor:
q = self.conv_q(x)
k = self.conv_k(c)
v = self.conv_v(c)
@@ -268,7 +284,13 @@ class MultiHeadAttention(nn.Module):
x = self.conv_o(x)
return x
- def attention(self, query, key, value, mask=None):
+ def attention(
+ self,
+ query: torch.Tensor,
+ key: torch.Tensor,
+ value: torch.Tensor,
+ mask: Optional[torch.Tensor] = None,
+ ) -> tuple[torch.Tensor, torch.Tensor]:
# reshape [b, d, t] -> [b, n_h, t, d_k]
b, d, t_s, t_t = (*key.size(), query.size(2))
query = query.view(b, self.n_heads, self.k_channels, t_t).transpose(2, 3)
@@ -319,7 +341,9 @@ class MultiHeadAttention(nn.Module):
) # [b, n_h, t_t, d_k] -> [b, d, t_t]
return output, p_attn
- def _matmul_with_relative_values(self, x, y):
+ def _matmul_with_relative_values(
+ self, x: torch.Tensor, y: torch.Tensor
+ ) -> torch.Tensor:
"""
x: [b, h, l, m]
y: [h or 1, m, d]
@@ -328,7 +352,9 @@ class MultiHeadAttention(nn.Module):
ret = torch.matmul(x, y.unsqueeze(0))
return ret
- def _matmul_with_relative_keys(self, x, y):
+ def _matmul_with_relative_keys(
+ self, x: torch.Tensor, y: torch.Tensor
+ ) -> torch.Tensor:
"""
x: [b, h, l, d]
y: [h or 1, m, d]
@@ -337,8 +363,11 @@ class MultiHeadAttention(nn.Module):
ret = torch.matmul(x, y.unsqueeze(0).transpose(-2, -1))
return ret
- def _get_relative_embeddings(self, relative_embeddings, length):
- 2 * self.window_size + 1
+ def _get_relative_embeddings(
+ self, relative_embeddings: torch.Tensor, length: int
+ ) -> torch.Tensor:
+ assert self.window_size is not None
+ 2 * self.window_size + 1 # type: ignore
# Pad first before slice to avoid using cond ops.
pad_length = max(length - (self.window_size + 1), 0)
slice_start_position = max((self.window_size + 1) - length, 0)
@@ -355,7 +384,7 @@ class MultiHeadAttention(nn.Module):
]
return used_relative_embeddings
- def _relative_position_to_absolute_position(self, x):
+ def _relative_position_to_absolute_position(self, x: torch.Tensor) -> torch.Tensor:
"""
x: [b, h, l, 2*l-1]
ret: [b, h, l, l]
@@ -376,7 +405,7 @@ class MultiHeadAttention(nn.Module):
]
return x_final
- def _absolute_position_to_relative_position(self, x):
+ def _absolute_position_to_relative_position(self, x: torch.Tensor) -> torch.Tensor:
"""
x: [b, h, l, l]
ret: [b, h, l, 2*l-1]
@@ -392,7 +421,7 @@ class MultiHeadAttention(nn.Module):
x_final = x_flat.view([batch, heads, length, 2 * length])[:, :, :, 1:]
return x_final
- def _attention_bias_proximal(self, length):
+ def _attention_bias_proximal(self, length: int) -> torch.Tensor:
"""Bias for self-attention to encourage attention to close positions.
Args:
length: an integer scalar.
@@ -407,14 +436,14 @@ class MultiHeadAttention(nn.Module):
class FFN(nn.Module):
def __init__(
self,
- in_channels,
- out_channels,
- filter_channels,
- kernel_size,
- p_dropout=0.0,
- activation=None,
- causal=False,
- ):
+ in_channels: int,
+ out_channels: int,
+ filter_channels: int,
+ kernel_size: int,
+ p_dropout: float = 0.0,
+ activation: Optional[str] = None,
+ causal: bool = False,
+ ) -> None:
super().__init__()
self.in_channels = in_channels
self.out_channels = out_channels
@@ -433,7 +462,7 @@ class FFN(nn.Module):
self.conv_2 = nn.Conv1d(filter_channels, out_channels, kernel_size)
self.drop = nn.Dropout(p_dropout)
- def forward(self, x, x_mask):
+ def forward(self, x: torch.Tensor, x_mask: torch.Tensor) -> torch.Tensor:
x = self.conv_1(self.padding(x * x_mask))
if self.activation == "gelu":
x = x * torch.sigmoid(1.702 * x)
@@ -443,7 +472,7 @@ class FFN(nn.Module):
x = self.conv_2(self.padding(x * x_mask))
return x * x_mask
- def _causal_padding(self, x):
+ def _causal_padding(self, x: torch.Tensor) -> torch.Tensor:
if self.kernel_size == 1:
return x
pad_l = self.kernel_size - 1
@@ -452,7 +481,7 @@ class FFN(nn.Module):
x = F.pad(x, commons.convert_pad_shape(padding))
return x
- def _same_padding(self, x):
+ def _same_padding(self, x: torch.Tensor) -> torch.Tensor:
if self.kernel_size == 1:
return x
pad_l = (self.kernel_size - 1) // 2
diff --git a/style_bert_vits2/models/commons.py b/style_bert_vits2/models/commons.py
new file mode 100644
index 0000000..38f548c
--- /dev/null
+++ b/style_bert_vits2/models/commons.py
@@ -0,0 +1,223 @@
+"""
+以下に記述されている関数のコメントはリファクタリング時に GPT-4 に生成させたもので、
+コードと完全に一致している保証はない。あくまで参考程度とすること。
+"""
+
+from typing import Any, Optional, Union
+
+import torch
+from torch.nn import functional as F
+
+
+def init_weights(m: torch.nn.Module, mean: float = 0.0, std: float = 0.01) -> None:
+ """
+ モジュールの重みを初期化する
+
+ Args:
+ m (torch.nn.Module): 重みを初期化する対象のモジュール
+ mean (float): 正規分布の平均
+ std (float): 正規分布の標準偏差
+ """
+ classname = m.__class__.__name__
+ if classname.find("Conv") != -1:
+ m.weight.data.normal_(mean, std)
+
+
+def get_padding(kernel_size: int, dilation: int = 1) -> int:
+ """
+ カーネルサイズと膨張率からパディングの大きさを計算する
+
+ Args:
+ kernel_size (int): カーネルのサイズ
+ dilation (int): 膨張率
+
+ Returns:
+ int: 計算されたパディングの大きさ
+ """
+ return int((kernel_size * dilation - dilation) / 2)
+
+
+def convert_pad_shape(pad_shape: list[list[Any]]) -> list[Any]:
+ """
+ パディングの形状を変換する
+
+ Args:
+ pad_shape (list[list[Any]]): 変換前のパディングの形状
+
+ Returns:
+ list[Any]: 変換後のパディングの形状
+ """
+ layer = pad_shape[::-1]
+ new_pad_shape = [item for sublist in layer for item in sublist]
+ return new_pad_shape
+
+
+def intersperse(lst: list[Any], item: Any) -> list[Any]:
+ """
+ リストの要素の間に特定のアイテムを挿入する
+
+ Args:
+ lst (list[Any]): 元のリスト
+ item (Any): 挿入するアイテム
+
+ Returns:
+ list[Any]: 新しいリスト
+ """
+ result = [item] * (len(lst) * 2 + 1)
+ result[1::2] = lst
+ return result
+
+
+def slice_segments(
+ x: torch.Tensor, ids_str: torch.Tensor, segment_size: int = 4
+) -> torch.Tensor:
+ """
+ テンソルからセグメントをスライスする
+
+ Args:
+ x (torch.Tensor): 入力テンソル
+ ids_str (torch.Tensor): スライスを開始するインデックス
+ segment_size (int, optional): スライスのサイズ (デフォルト: 4)
+
+ Returns:
+ torch.Tensor: スライスされたセグメント
+ """
+ gather_indices = ids_str.view(x.size(0), 1, 1).repeat(
+ 1, x.size(1), 1
+ ) + torch.arange(segment_size, device=x.device)
+ return torch.gather(x, 2, gather_indices)
+
+
+def rand_slice_segments(
+ x: torch.Tensor, x_lengths: Optional[torch.Tensor] = None, segment_size: int = 4
+) -> tuple[torch.Tensor, torch.Tensor]:
+ """
+ ランダムなセグメントをスライスする
+
+ Args:
+ x (torch.Tensor): 入力テンソル
+ x_lengths (Optional[torch.Tensor], optional): 各バッチの長さ (デフォルト: None)
+ segment_size (int, optional): スライスのサイズ (デフォルト: 4)
+
+ Returns:
+ tuple[torch.Tensor, torch.Tensor]: スライスされたセグメントと開始インデックス
+ """
+ b, d, t = x.size()
+ if x_lengths is None:
+ x_lengths = t # type: ignore
+ ids_str_max = torch.clamp(x_lengths - segment_size + 1, min=0) # type: ignore
+ ids_str = (torch.rand([b], device=x.device) * ids_str_max).to(dtype=torch.long)
+ ret = slice_segments(x, ids_str, segment_size)
+ return ret, ids_str
+
+
+def subsequent_mask(length: int) -> torch.Tensor:
+ """
+ 後続のマスクを生成する
+
+ Args:
+ length (int): マスクのサイズ
+
+ Returns:
+ torch.Tensor: 生成されたマスク
+ """
+ mask = torch.tril(torch.ones(length, length)).unsqueeze(0).unsqueeze(0)
+ return mask
+
+
+@torch.jit.script # type: ignore
+def fused_add_tanh_sigmoid_multiply(
+ input_a: torch.Tensor, input_b: torch.Tensor, n_channels: torch.Tensor
+) -> torch.Tensor:
+ """
+ 加算、tanh、sigmoid の活性化関数を組み合わせた演算を行う
+
+ Args:
+ input_a (torch.Tensor): 入力テンソル A
+ input_b (torch.Tensor): 入力テンソル B
+ n_channels (torch.Tensor): チャネル数
+
+ Returns:
+ torch.Tensor: 演算結果
+ """
+ n_channels_int = n_channels[0]
+ in_act = input_a + input_b
+ t_act = torch.tanh(in_act[:, :n_channels_int, :])
+ s_act = torch.sigmoid(in_act[:, n_channels_int:, :])
+ acts = t_act * s_act
+ return acts
+
+
+def sequence_mask(
+ length: torch.Tensor, max_length: Optional[int] = None
+) -> torch.Tensor:
+ """
+ シーケンスマスクを生成する
+
+ Args:
+ length (torch.Tensor): 各シーケンスの長さ
+ max_length (Optional[int]): 最大のシーケンス長さ。指定されていない場合は length の最大値を使用
+
+ Returns:
+ torch.Tensor: 生成されたシーケンスマスク
+ """
+ if max_length is None:
+ max_length = length.max() # type: ignore
+ x = torch.arange(max_length, dtype=length.dtype, device=length.device) # type: ignore
+ return x.unsqueeze(0) < length.unsqueeze(1)
+
+
+def generate_path(duration: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
+ """
+ パスを生成する
+
+ Args:
+ duration (torch.Tensor): 各時間ステップの持続時間
+ mask (torch.Tensor): マスクテンソル
+
+ Returns:
+ torch.Tensor: 生成されたパス
+ """
+ b, _, t_y, t_x = mask.shape
+ cum_duration = torch.cumsum(duration, -1)
+
+ cum_duration_flat = cum_duration.view(b * t_x)
+ path = sequence_mask(cum_duration_flat, t_y).to(mask.dtype)
+ path = path.view(b, t_x, t_y)
+ path = path - F.pad(path, convert_pad_shape([[0, 0], [1, 0], [0, 0]]))[:, :-1]
+ path = path.unsqueeze(1).transpose(2, 3) * mask
+ return path
+
+
+def clip_grad_value_(
+ parameters: Union[torch.Tensor, list[torch.Tensor]],
+ clip_value: Optional[float],
+ norm_type: float = 2.0,
+) -> float:
+ """
+ 勾配の値をクリップする
+
+ Args:
+ parameters (Union[torch.Tensor, list[torch.Tensor]]): クリップするパラメータ
+ clip_value (Optional[float]): クリップする値。None の場合はクリップしない
+ norm_type (float): ノルムの種類
+
+ Returns:
+ float: 総ノルム
+ """
+ if isinstance(parameters, torch.Tensor):
+ parameters = [parameters]
+ parameters = list(filter(lambda p: p.grad is not None, parameters))
+ norm_type = float(norm_type)
+ if clip_value is not None:
+ clip_value = float(clip_value)
+
+ total_norm = 0.0
+ for p in parameters:
+ assert p.grad is not None
+ param_norm = p.grad.data.norm(norm_type)
+ total_norm += param_norm.item() ** norm_type
+ if clip_value is not None:
+ p.grad.data.clamp_(min=-clip_value, max=clip_value)
+ total_norm = total_norm ** (1.0 / norm_type)
+ return total_norm
diff --git a/style_bert_vits2/models/hyper_parameters.py b/style_bert_vits2/models/hyper_parameters.py
new file mode 100644
index 0000000..254b285
--- /dev/null
+++ b/style_bert_vits2/models/hyper_parameters.py
@@ -0,0 +1,129 @@
+"""
+Style-Bert-VITS2 モデルのハイパーパラメータを表す Pydantic モデル。
+デフォルト値は configs/configs_jp_extra.json 内の定義と概ね同一で、
+万が一ロードした config.json に存在しないキーがあった際のフェイルセーフとして適用される。
+"""
+
+from pathlib import Path
+from typing import Optional, Union
+
+from pydantic import BaseModel, ConfigDict
+
+
+class HyperParametersTrain(BaseModel):
+ log_interval: int = 200
+ eval_interval: int = 1000
+ seed: int = 42
+ epochs: int = 1000
+ learning_rate: float = 0.0001
+ betas: tuple[float, float] = (0.8, 0.99)
+ eps: float = 1e-9
+ batch_size: int = 2
+ bf16_run: bool = False
+ fp16_run: bool = False
+ lr_decay: float = 0.99996
+ segment_size: int = 16384
+ init_lr_ratio: int = 1
+ warmup_epochs: int = 0
+ c_mel: int = 45
+ c_kl: float = 1.0
+ c_commit: int = 100
+ skip_optimizer: bool = False
+ freeze_ZH_bert: bool = False
+ freeze_JP_bert: bool = False
+ freeze_EN_bert: bool = False
+ freeze_emo: bool = False
+ freeze_style: bool = False
+ freeze_decoder: bool = False
+
+
+class HyperParametersData(BaseModel):
+ use_jp_extra: bool = True
+ training_files: str = "Data/Dummy/train.list"
+ validation_files: str = "Data/Dummy/val.list"
+ max_wav_value: float = 32768.0
+ sampling_rate: int = 44100
+ filter_length: int = 2048
+ hop_length: int = 512
+ win_length: int = 2048
+ n_mel_channels: int = 128
+ mel_fmin: float = 0.0
+ mel_fmax: Optional[float] = None
+ add_blank: bool = True
+ n_speakers: int = 1
+ cleaned_text: bool = True
+ spk2id: dict[str, int] = {
+ "Dummy": 0,
+ }
+ num_styles: int = 1
+ style2id: dict[str, int] = {
+ "Neutral": 0,
+ }
+
+
+class HyperParametersModelSLM(BaseModel):
+ model: str = "./slm/wavlm-base-plus"
+ sr: int = 16000
+ hidden: int = 768
+ nlayers: int = 13
+ initial_channel: int = 64
+
+
+class HyperParametersModel(BaseModel):
+ use_spk_conditioned_encoder: bool = True
+ use_noise_scaled_mas: bool = True
+ use_mel_posterior_encoder: bool = False
+ use_duration_discriminator: bool = False
+ use_wavlm_discriminator: bool = True
+ inter_channels: int = 192
+ hidden_channels: int = 192
+ filter_channels: int = 768
+ n_heads: int = 2
+ n_layers: int = 6
+ kernel_size: int = 3
+ p_dropout: float = 0.1
+ resblock: str = "1"
+ resblock_kernel_sizes: list[int] = [3, 7, 11]
+ resblock_dilation_sizes: list[list[int]] = [
+ [1, 3, 5],
+ [1, 3, 5],
+ [1, 3, 5],
+ ]
+ upsample_rates: list[int] = [8, 8, 2, 2, 2]
+ upsample_initial_channel: int = 512
+ upsample_kernel_sizes: list[int] = [16, 16, 8, 2, 2]
+ n_layers_q: int = 3
+ use_spectral_norm: bool = False
+ gin_channels: int = 512
+ slm: HyperParametersModelSLM = HyperParametersModelSLM()
+
+
+class HyperParameters(BaseModel):
+ model_name: str = "Dummy"
+ version: str = "2.0-JP-Extra"
+ train: HyperParametersTrain = HyperParametersTrain()
+ data: HyperParametersData = HyperParametersData()
+ model: HyperParametersModel = HyperParametersModel()
+
+ # 以下は学習時にのみ動的に設定されるパラメータ (通常 config.json には存在しない)
+ model_dir: Optional[str] = None
+ speedup: bool = False
+ repo_id: Optional[str] = None
+
+ # model_ 以下を Pydantic の保護対象から除外する
+ model_config = ConfigDict(protected_namespaces=())
+
+ @staticmethod
+ def load_from_json(json_path: Union[str, Path]) -> "HyperParameters":
+ """
+ 与えられた JSON ファイルからハイパーパラメータを読み込む。
+
+ Args:
+ json_path (Union[str, Path]): JSON ファイルのパス
+
+ Returns:
+ HyperParameters: ハイパーパラメータ
+ """
+
+ with open(json_path, "r", encoding="utf-8") as f:
+ return HyperParameters.model_validate_json(f.read())
diff --git a/style_bert_vits2/models/infer.py b/style_bert_vits2/models/infer.py
new file mode 100644
index 0000000..a9bcbf6
--- /dev/null
+++ b/style_bert_vits2/models/infer.py
@@ -0,0 +1,267 @@
+from typing import Any, Optional, Union, cast
+
+import torch
+from numpy.typing import NDArray
+
+from style_bert_vits2.constants import Languages
+from style_bert_vits2.logging import logger
+from style_bert_vits2.models import commons, utils
+from style_bert_vits2.models.hyper_parameters import HyperParameters
+from style_bert_vits2.models.models import SynthesizerTrn
+from style_bert_vits2.models.models_jp_extra import (
+ SynthesizerTrn as SynthesizerTrnJPExtra,
+)
+from style_bert_vits2.nlp import (
+ clean_text,
+ cleaned_text_to_sequence,
+ extract_bert_feature,
+)
+from style_bert_vits2.nlp.symbols import SYMBOLS
+
+
+def get_net_g(model_path: str, version: str, device: str, hps: HyperParameters):
+ if version.endswith("JP-Extra"):
+ logger.info("Using JP-Extra model")
+ net_g = SynthesizerTrnJPExtra(
+ n_vocab=len(SYMBOLS),
+ spec_channels=hps.data.filter_length // 2 + 1,
+ segment_size=hps.train.segment_size // hps.data.hop_length,
+ n_speakers=hps.data.n_speakers,
+ # hps.model 以下のすべての値を引数に渡す
+ use_spk_conditioned_encoder=hps.model.use_spk_conditioned_encoder,
+ use_noise_scaled_mas=hps.model.use_noise_scaled_mas,
+ use_mel_posterior_encoder=hps.model.use_mel_posterior_encoder,
+ use_duration_discriminator=hps.model.use_duration_discriminator,
+ use_wavlm_discriminator=hps.model.use_wavlm_discriminator,
+ inter_channels=hps.model.inter_channels,
+ hidden_channels=hps.model.hidden_channels,
+ filter_channels=hps.model.filter_channels,
+ n_heads=hps.model.n_heads,
+ n_layers=hps.model.n_layers,
+ kernel_size=hps.model.kernel_size,
+ p_dropout=hps.model.p_dropout,
+ resblock=hps.model.resblock,
+ resblock_kernel_sizes=hps.model.resblock_kernel_sizes,
+ resblock_dilation_sizes=hps.model.resblock_dilation_sizes,
+ upsample_rates=hps.model.upsample_rates,
+ upsample_initial_channel=hps.model.upsample_initial_channel,
+ upsample_kernel_sizes=hps.model.upsample_kernel_sizes,
+ n_layers_q=hps.model.n_layers_q,
+ use_spectral_norm=hps.model.use_spectral_norm,
+ gin_channels=hps.model.gin_channels,
+ slm=hps.model.slm,
+ ).to(device)
+ else:
+ logger.info("Using normal model")
+ net_g = SynthesizerTrn(
+ n_vocab=len(SYMBOLS),
+ spec_channels=hps.data.filter_length // 2 + 1,
+ segment_size=hps.train.segment_size // hps.data.hop_length,
+ n_speakers=hps.data.n_speakers,
+ # hps.model 以下のすべての値を引数に渡す
+ use_spk_conditioned_encoder=hps.model.use_spk_conditioned_encoder,
+ use_noise_scaled_mas=hps.model.use_noise_scaled_mas,
+ use_mel_posterior_encoder=hps.model.use_mel_posterior_encoder,
+ use_duration_discriminator=hps.model.use_duration_discriminator,
+ use_wavlm_discriminator=hps.model.use_wavlm_discriminator,
+ inter_channels=hps.model.inter_channels,
+ hidden_channels=hps.model.hidden_channels,
+ filter_channels=hps.model.filter_channels,
+ n_heads=hps.model.n_heads,
+ n_layers=hps.model.n_layers,
+ kernel_size=hps.model.kernel_size,
+ p_dropout=hps.model.p_dropout,
+ resblock=hps.model.resblock,
+ resblock_kernel_sizes=hps.model.resblock_kernel_sizes,
+ resblock_dilation_sizes=hps.model.resblock_dilation_sizes,
+ upsample_rates=hps.model.upsample_rates,
+ upsample_initial_channel=hps.model.upsample_initial_channel,
+ upsample_kernel_sizes=hps.model.upsample_kernel_sizes,
+ n_layers_q=hps.model.n_layers_q,
+ use_spectral_norm=hps.model.use_spectral_norm,
+ gin_channels=hps.model.gin_channels,
+ slm=hps.model.slm,
+ ).to(device)
+ net_g.state_dict()
+ _ = net_g.eval()
+ if model_path.endswith(".pth") or model_path.endswith(".pt"):
+ _ = utils.checkpoints.load_checkpoint(
+ model_path, net_g, None, skip_optimizer=True
+ )
+ elif model_path.endswith(".safetensors"):
+ _ = utils.safetensors.load_safetensors(model_path, net_g, True)
+ else:
+ raise ValueError(f"Unknown model format: {model_path}")
+ return net_g
+
+
+def get_text(
+ text: str,
+ language_str: Languages,
+ hps: HyperParameters,
+ device: str,
+ assist_text: Optional[str] = None,
+ assist_text_weight: float = 0.7,
+ given_tone: Optional[list[int]] = None,
+):
+ use_jp_extra = hps.version.endswith("JP-Extra")
+ # 推論時のみ呼び出されるので、raise_yomi_error は False に設定
+ norm_text, phone, tone, word2ph = clean_text(
+ text,
+ language_str,
+ use_jp_extra=use_jp_extra,
+ raise_yomi_error=False,
+ )
+ if given_tone is not None:
+ if len(given_tone) != len(phone):
+ raise InvalidToneError(
+ f"Length of given_tone ({len(given_tone)}) != length of phone ({len(phone)})"
+ )
+ tone = given_tone
+ phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
+
+ if hps.data.add_blank:
+ phone = commons.intersperse(phone, 0)
+ tone = commons.intersperse(tone, 0)
+ language = commons.intersperse(language, 0)
+ for i in range(len(word2ph)):
+ word2ph[i] = word2ph[i] * 2
+ word2ph[0] += 1
+ bert_ori = extract_bert_feature(
+ norm_text,
+ word2ph,
+ language_str,
+ device,
+ assist_text,
+ assist_text_weight,
+ )
+ del word2ph
+ assert bert_ori.shape[-1] == len(phone), phone
+
+ if language_str == Languages.ZH:
+ bert = bert_ori
+ ja_bert = torch.zeros(1024, len(phone))
+ en_bert = torch.zeros(1024, len(phone))
+ elif language_str == Languages.JP:
+ bert = torch.zeros(1024, len(phone))
+ ja_bert = bert_ori
+ en_bert = torch.zeros(1024, len(phone))
+ elif language_str == Languages.EN:
+ bert = torch.zeros(1024, len(phone))
+ ja_bert = torch.zeros(1024, len(phone))
+ en_bert = bert_ori
+ else:
+ raise ValueError("language_str should be ZH, JP or EN")
+
+ assert bert.shape[-1] == len(
+ phone
+ ), f"Bert seq len {bert.shape[-1]} != {len(phone)}"
+
+ phone = torch.LongTensor(phone)
+ tone = torch.LongTensor(tone)
+ language = torch.LongTensor(language)
+ return bert, ja_bert, en_bert, phone, tone, language
+
+
+def infer(
+ text: str,
+ style_vec: NDArray[Any],
+ sdp_ratio: float,
+ noise_scale: float,
+ noise_scale_w: float,
+ length_scale: float,
+ sid: int, # In the original Bert-VITS2, its speaker_name: str, but here it's id
+ language: Languages,
+ hps: HyperParameters,
+ net_g: Union[SynthesizerTrn, SynthesizerTrnJPExtra],
+ device: str,
+ skip_start: bool = False,
+ skip_end: bool = False,
+ assist_text: Optional[str] = None,
+ assist_text_weight: float = 0.7,
+ given_tone: Optional[list[int]] = None,
+):
+ is_jp_extra = hps.version.endswith("JP-Extra")
+ bert, ja_bert, en_bert, phones, tones, lang_ids = get_text(
+ text,
+ language,
+ hps,
+ device,
+ assist_text=assist_text,
+ assist_text_weight=assist_text_weight,
+ given_tone=given_tone,
+ )
+ if skip_start:
+ phones = phones[3:]
+ tones = tones[3:]
+ lang_ids = lang_ids[3:]
+ bert = bert[:, 3:]
+ ja_bert = ja_bert[:, 3:]
+ en_bert = en_bert[:, 3:]
+ if skip_end:
+ phones = phones[:-2]
+ tones = tones[:-2]
+ lang_ids = lang_ids[:-2]
+ bert = bert[:, :-2]
+ ja_bert = ja_bert[:, :-2]
+ en_bert = en_bert[:, :-2]
+ with torch.no_grad():
+ x_tst = phones.to(device).unsqueeze(0)
+ tones = tones.to(device).unsqueeze(0)
+ lang_ids = lang_ids.to(device).unsqueeze(0)
+ bert = bert.to(device).unsqueeze(0)
+ ja_bert = ja_bert.to(device).unsqueeze(0)
+ en_bert = en_bert.to(device).unsqueeze(0)
+ x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
+ style_vec_tensor = torch.from_numpy(style_vec).to(device).unsqueeze(0)
+ del phones
+ sid_tensor = torch.LongTensor([sid]).to(device)
+ if is_jp_extra:
+ output = cast(SynthesizerTrnJPExtra, net_g).infer(
+ x_tst,
+ x_tst_lengths,
+ sid_tensor,
+ tones,
+ lang_ids,
+ ja_bert,
+ style_vec=style_vec_tensor,
+ sdp_ratio=sdp_ratio,
+ noise_scale=noise_scale,
+ noise_scale_w=noise_scale_w,
+ length_scale=length_scale,
+ )
+ else:
+ output = cast(SynthesizerTrn, net_g).infer(
+ x_tst,
+ x_tst_lengths,
+ sid_tensor,
+ tones,
+ lang_ids,
+ bert,
+ ja_bert,
+ en_bert,
+ style_vec=style_vec_tensor,
+ sdp_ratio=sdp_ratio,
+ noise_scale=noise_scale,
+ noise_scale_w=noise_scale_w,
+ length_scale=length_scale,
+ )
+ audio = output[0][0, 0].data.cpu().float().numpy()
+ del (
+ x_tst,
+ tones,
+ lang_ids,
+ bert,
+ x_tst_lengths,
+ sid_tensor,
+ ja_bert,
+ en_bert,
+ style_vec,
+ ) # , emo
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+ return audio
+
+
+class InvalidToneError(ValueError):
+ pass
diff --git a/models.py b/style_bert_vits2/models/models.py
similarity index 78%
rename from models.py
rename to style_bert_vits2/models/models.py
index eb706fd..56fb27c 100644
--- a/models.py
+++ b/style_bert_vits2/models/models.py
@@ -1,5 +1,5 @@
import math
-import warnings
+from typing import Any, Optional
import torch
from torch import nn
@@ -7,18 +7,19 @@ from torch.nn import Conv1d, Conv2d, ConvTranspose1d
from torch.nn import functional as F
from torch.nn.utils import remove_weight_norm, spectral_norm, weight_norm
-import attentions
-import commons
-import modules
-import monotonic_align
-from commons import get_padding, init_weights
-from text import num_languages, num_tones, symbols
+from style_bert_vits2.models import attentions, commons, modules, monotonic_alignment
+from style_bert_vits2.nlp.symbols import NUM_LANGUAGES, NUM_TONES, SYMBOLS
class DurationDiscriminator(nn.Module): # vits2
def __init__(
- self, in_channels, filter_channels, kernel_size, p_dropout, gin_channels=0
- ):
+ self,
+ in_channels: int,
+ filter_channels: int,
+ kernel_size: int,
+ p_dropout: float,
+ gin_channels: int = 0,
+ ) -> None:
super().__init__()
self.in_channels = in_channels
@@ -52,7 +53,13 @@ class DurationDiscriminator(nn.Module): # vits2
self.output_layer = nn.Sequential(nn.Linear(filter_channels, 1), nn.Sigmoid())
- def forward_probability(self, x, x_mask, dur, g=None):
+ def forward_probability(
+ self,
+ x: torch.Tensor,
+ x_mask: torch.Tensor,
+ dur: torch.Tensor,
+ g: Optional[torch.Tensor] = None,
+ ) -> torch.Tensor:
dur = self.dur_proj(dur)
x = torch.cat([x, dur], dim=1)
x = self.pre_out_conv_1(x * x_mask)
@@ -68,7 +75,14 @@ class DurationDiscriminator(nn.Module): # vits2
output_prob = self.output_layer(x)
return output_prob
- def forward(self, x, x_mask, dur_r, dur_hat, g=None):
+ def forward(
+ self,
+ x: torch.Tensor,
+ x_mask: torch.Tensor,
+ dur_r: torch.Tensor,
+ dur_hat: torch.Tensor,
+ g: Optional[torch.Tensor] = None,
+ ) -> list[torch.Tensor]:
x = torch.detach(x)
if g is not None:
g = torch.detach(g)
@@ -93,17 +107,17 @@ class DurationDiscriminator(nn.Module): # vits2
class TransformerCouplingBlock(nn.Module):
def __init__(
self,
- channels,
- hidden_channels,
- filter_channels,
- n_heads,
- n_layers,
- kernel_size,
- p_dropout,
- n_flows=4,
- gin_channels=0,
- share_parameter=False,
- ):
+ channels: int,
+ hidden_channels: int,
+ filter_channels: int,
+ n_heads: int,
+ n_layers: int,
+ kernel_size: int,
+ p_dropout: float,
+ n_flows: int = 4,
+ gin_channels: int = 0,
+ share_parameter: bool = False,
+ ) -> None:
super().__init__()
self.channels = channels
self.hidden_channels = hidden_channels
@@ -115,16 +129,17 @@ class TransformerCouplingBlock(nn.Module):
self.flows = nn.ModuleList()
self.wn = (
- attentions.FFT(
- hidden_channels,
- filter_channels,
- n_heads,
- n_layers,
- kernel_size,
- p_dropout,
- isflow=True,
- gin_channels=self.gin_channels,
- )
+ # attentions.FFT(
+ # hidden_channels,
+ # filter_channels,
+ # n_heads,
+ # n_layers,
+ # kernel_size,
+ # p_dropout,
+ # isflow=True,
+ # gin_channels=self.gin_channels,
+ # )
+ None
if share_parameter
else None
)
@@ -146,7 +161,13 @@ class TransformerCouplingBlock(nn.Module):
)
self.flows.append(modules.Flip())
- def forward(self, x, x_mask, g=None, reverse=False):
+ def forward(
+ self,
+ x: torch.Tensor,
+ x_mask: torch.Tensor,
+ g: Optional[torch.Tensor] = None,
+ reverse: bool = False,
+ ) -> torch.Tensor:
if not reverse:
for flow in self.flows:
x, _ = flow(x, x_mask, g=g, reverse=reverse)
@@ -159,13 +180,13 @@ class TransformerCouplingBlock(nn.Module):
class StochasticDurationPredictor(nn.Module):
def __init__(
self,
- in_channels,
- filter_channels,
- kernel_size,
- p_dropout,
- n_flows=4,
- gin_channels=0,
- ):
+ in_channels: int,
+ filter_channels: int,
+ kernel_size: int,
+ p_dropout: float,
+ n_flows: int = 4,
+ gin_channels: int = 0,
+ ) -> None:
super().__init__()
filter_channels = in_channels # it needs to be removed from future version.
self.in_channels = in_channels
@@ -205,7 +226,15 @@ class StochasticDurationPredictor(nn.Module):
if gin_channels != 0:
self.cond = nn.Conv1d(gin_channels, filter_channels, 1)
- def forward(self, x, x_mask, w=None, g=None, reverse=False, noise_scale=1.0):
+ def forward(
+ self,
+ x: torch.Tensor,
+ x_mask: torch.Tensor,
+ w: Optional[torch.Tensor] = None,
+ g: Optional[torch.Tensor] = None,
+ reverse: bool = False,
+ noise_scale: float = 1.0,
+ ) -> torch.Tensor:
x = torch.detach(x)
x = self.pre(x)
if g is not None:
@@ -269,8 +298,13 @@ class StochasticDurationPredictor(nn.Module):
class DurationPredictor(nn.Module):
def __init__(
- self, in_channels, filter_channels, kernel_size, p_dropout, gin_channels=0
- ):
+ self,
+ in_channels: int,
+ filter_channels: int,
+ kernel_size: int,
+ p_dropout: float,
+ gin_channels: int = 0,
+ ) -> None:
super().__init__()
self.in_channels = in_channels
@@ -293,7 +327,9 @@ class DurationPredictor(nn.Module):
if gin_channels != 0:
self.cond = nn.Conv1d(gin_channels, in_channels, 1)
- def forward(self, x, x_mask, g=None):
+ def forward(
+ self, x: torch.Tensor, x_mask: torch.Tensor, g: Optional[torch.Tensor] = None
+ ) -> torch.Tensor:
x = torch.detach(x)
if g is not None:
g = torch.detach(g)
@@ -313,17 +349,17 @@ class DurationPredictor(nn.Module):
class TextEncoder(nn.Module):
def __init__(
self,
- n_vocab,
- out_channels,
- hidden_channels,
- filter_channels,
- n_heads,
- n_layers,
- kernel_size,
- p_dropout,
- n_speakers,
- gin_channels=0,
- ):
+ n_vocab: int,
+ out_channels: int,
+ hidden_channels: int,
+ filter_channels: int,
+ n_heads: int,
+ n_layers: int,
+ kernel_size: int,
+ p_dropout: float,
+ n_speakers: int,
+ gin_channels: int = 0,
+ ) -> None:
super().__init__()
self.n_vocab = n_vocab
self.out_channels = out_channels
@@ -334,11 +370,11 @@ class TextEncoder(nn.Module):
self.kernel_size = kernel_size
self.p_dropout = p_dropout
self.gin_channels = gin_channels
- self.emb = nn.Embedding(len(symbols), hidden_channels)
+ self.emb = nn.Embedding(len(SYMBOLS), hidden_channels)
nn.init.normal_(self.emb.weight, 0.0, hidden_channels**-0.5)
- self.tone_emb = nn.Embedding(num_tones, hidden_channels)
+ self.tone_emb = nn.Embedding(NUM_TONES, hidden_channels)
nn.init.normal_(self.tone_emb.weight, 0.0, hidden_channels**-0.5)
- self.language_emb = nn.Embedding(num_languages, hidden_channels)
+ self.language_emb = nn.Embedding(NUM_LANGUAGES, hidden_channels)
nn.init.normal_(self.language_emb.weight, 0.0, hidden_channels**-0.5)
self.bert_proj = nn.Conv1d(1024, hidden_channels, 1)
self.ja_bert_proj = nn.Conv1d(1024, hidden_channels, 1)
@@ -358,17 +394,17 @@ class TextEncoder(nn.Module):
def forward(
self,
- x,
- x_lengths,
- tone,
- language,
- bert,
- ja_bert,
- en_bert,
- style_vec,
- sid,
- g=None,
- ):
+ x: torch.Tensor,
+ x_lengths: torch.Tensor,
+ tone: torch.Tensor,
+ language: torch.Tensor,
+ bert: torch.Tensor,
+ ja_bert: torch.Tensor,
+ en_bert: torch.Tensor,
+ style_vec: torch.Tensor,
+ sid: torch.Tensor,
+ g: Optional[torch.Tensor] = None,
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
bert_emb = self.bert_proj(bert).transpose(1, 2)
ja_bert_emb = self.ja_bert_proj(ja_bert).transpose(1, 2)
en_bert_emb = self.en_bert_proj(en_bert).transpose(1, 2)
@@ -400,14 +436,14 @@ class TextEncoder(nn.Module):
class ResidualCouplingBlock(nn.Module):
def __init__(
self,
- channels,
- hidden_channels,
- kernel_size,
- dilation_rate,
- n_layers,
- n_flows=4,
- gin_channels=0,
- ):
+ channels: int,
+ hidden_channels: int,
+ kernel_size: int,
+ dilation_rate: int,
+ n_layers: int,
+ n_flows: int = 4,
+ gin_channels: int = 0,
+ ) -> None:
super().__init__()
self.channels = channels
self.hidden_channels = hidden_channels
@@ -432,7 +468,13 @@ class ResidualCouplingBlock(nn.Module):
)
self.flows.append(modules.Flip())
- def forward(self, x, x_mask, g=None, reverse=False):
+ def forward(
+ self,
+ x: torch.Tensor,
+ x_mask: torch.Tensor,
+ g: Optional[torch.Tensor] = None,
+ reverse: bool = False,
+ ) -> torch.Tensor:
if not reverse:
for flow in self.flows:
x, _ = flow(x, x_mask, g=g, reverse=reverse)
@@ -445,14 +487,14 @@ class ResidualCouplingBlock(nn.Module):
class PosteriorEncoder(nn.Module):
def __init__(
self,
- in_channels,
- out_channels,
- hidden_channels,
- kernel_size,
- dilation_rate,
- n_layers,
- gin_channels=0,
- ):
+ in_channels: int,
+ out_channels: int,
+ hidden_channels: int,
+ kernel_size: int,
+ dilation_rate: int,
+ n_layers: int,
+ gin_channels: int = 0,
+ ) -> None:
super().__init__()
self.in_channels = in_channels
self.out_channels = out_channels
@@ -472,7 +514,12 @@ class PosteriorEncoder(nn.Module):
)
self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
- def forward(self, x, x_lengths, g=None):
+ def forward(
+ self,
+ x: torch.Tensor,
+ x_lengths: torch.Tensor,
+ g: Optional[torch.Tensor] = None,
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(
x.dtype
)
@@ -487,22 +534,22 @@ class PosteriorEncoder(nn.Module):
class Generator(torch.nn.Module):
def __init__(
self,
- initial_channel,
- resblock,
- resblock_kernel_sizes,
- resblock_dilation_sizes,
- upsample_rates,
- upsample_initial_channel,
- upsample_kernel_sizes,
- gin_channels=0,
- ):
+ initial_channel: int,
+ resblock_str: str,
+ resblock_kernel_sizes: list[int],
+ resblock_dilation_sizes: list[list[int]],
+ upsample_rates: list[int],
+ upsample_initial_channel: int,
+ upsample_kernel_sizes: list[int],
+ gin_channels: int = 0,
+ ) -> None:
super(Generator, self).__init__()
self.num_kernels = len(resblock_kernel_sizes)
self.num_upsamples = len(upsample_rates)
self.conv_pre = Conv1d(
initial_channel, upsample_initial_channel, 7, 1, padding=3
)
- resblock = modules.ResBlock1 if resblock == "1" else modules.ResBlock2
+ resblock = modules.ResBlock1 if resblock_str == "1" else modules.ResBlock2
self.ups = nn.ModuleList()
for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):
@@ -519,20 +566,24 @@ class Generator(torch.nn.Module):
)
self.resblocks = nn.ModuleList()
+ ch = None
for i in range(len(self.ups)):
ch = upsample_initial_channel // (2 ** (i + 1))
for j, (k, d) in enumerate(
zip(resblock_kernel_sizes, resblock_dilation_sizes)
):
- self.resblocks.append(resblock(ch, k, d))
+ self.resblocks.append(resblock(ch, k, d)) # type: ignore
+ assert ch is not None
self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False)
- self.ups.apply(init_weights)
+ self.ups.apply(commons.init_weights)
if gin_channels != 0:
self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1)
- def forward(self, x, g=None):
+ def forward(
+ self, x: torch.Tensor, g: Optional[torch.Tensor] = None
+ ) -> torch.Tensor:
x = self.conv_pre(x)
if g is not None:
x = x + self.cond(g)
@@ -546,6 +597,7 @@ class Generator(torch.nn.Module):
xs = self.resblocks[i * self.num_kernels + j](x)
else:
xs += self.resblocks[i * self.num_kernels + j](x)
+ assert xs is not None
x = xs / self.num_kernels
x = F.leaky_relu(x)
x = self.conv_post(x)
@@ -553,7 +605,7 @@ class Generator(torch.nn.Module):
return x
- def remove_weight_norm(self):
+ def remove_weight_norm(self) -> None:
print("Removing weight norm...")
for layer in self.ups:
remove_weight_norm(layer)
@@ -562,7 +614,13 @@ class Generator(torch.nn.Module):
class DiscriminatorP(torch.nn.Module):
- def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False):
+ def __init__(
+ self,
+ period: int,
+ kernel_size: int = 5,
+ stride: int = 3,
+ use_spectral_norm: bool = False,
+ ) -> None:
super(DiscriminatorP, self).__init__()
self.period = period
self.use_spectral_norm = use_spectral_norm
@@ -575,7 +633,7 @@ class DiscriminatorP(torch.nn.Module):
32,
(kernel_size, 1),
(stride, 1),
- padding=(get_padding(kernel_size, 1), 0),
+ padding=(commons.get_padding(kernel_size, 1), 0),
)
),
norm_f(
@@ -584,7 +642,7 @@ class DiscriminatorP(torch.nn.Module):
128,
(kernel_size, 1),
(stride, 1),
- padding=(get_padding(kernel_size, 1), 0),
+ padding=(commons.get_padding(kernel_size, 1), 0),
)
),
norm_f(
@@ -593,7 +651,7 @@ class DiscriminatorP(torch.nn.Module):
512,
(kernel_size, 1),
(stride, 1),
- padding=(get_padding(kernel_size, 1), 0),
+ padding=(commons.get_padding(kernel_size, 1), 0),
)
),
norm_f(
@@ -602,7 +660,7 @@ class DiscriminatorP(torch.nn.Module):
1024,
(kernel_size, 1),
(stride, 1),
- padding=(get_padding(kernel_size, 1), 0),
+ padding=(commons.get_padding(kernel_size, 1), 0),
)
),
norm_f(
@@ -611,14 +669,14 @@ class DiscriminatorP(torch.nn.Module):
1024,
(kernel_size, 1),
1,
- padding=(get_padding(kernel_size, 1), 0),
+ padding=(commons.get_padding(kernel_size, 1), 0),
)
),
]
)
self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0)))
- def forward(self, x):
+ def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, list[torch.Tensor]]:
fmap = []
# 1d to 2d
@@ -641,7 +699,7 @@ class DiscriminatorP(torch.nn.Module):
class DiscriminatorS(torch.nn.Module):
- def __init__(self, use_spectral_norm=False):
+ def __init__(self, use_spectral_norm: bool = False) -> None:
super(DiscriminatorS, self).__init__()
norm_f = weight_norm if use_spectral_norm is False else spectral_norm
self.convs = nn.ModuleList(
@@ -656,7 +714,7 @@ class DiscriminatorS(torch.nn.Module):
)
self.conv_post = norm_f(Conv1d(1024, 1, 3, 1, padding=1))
- def forward(self, x):
+ def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, list[torch.Tensor]]:
fmap = []
for layer in self.convs:
@@ -671,7 +729,7 @@ class DiscriminatorS(torch.nn.Module):
class MultiPeriodDiscriminator(torch.nn.Module):
- def __init__(self, use_spectral_norm=False):
+ def __init__(self, use_spectral_norm: bool = False) -> None:
super(MultiPeriodDiscriminator, self).__init__()
periods = [2, 3, 5, 7, 11]
@@ -681,7 +739,13 @@ class MultiPeriodDiscriminator(torch.nn.Module):
]
self.discriminators = nn.ModuleList(discs)
- def forward(self, y, y_hat):
+ def forward(
+ self,
+ y: torch.Tensor,
+ y_hat: torch.Tensor,
+ ) -> tuple[
+ list[torch.Tensor], list[torch.Tensor], list[torch.Tensor], list[torch.Tensor]
+ ]:
y_d_rs = []
y_d_gs = []
fmap_rs = []
@@ -703,7 +767,7 @@ class ReferenceEncoder(nn.Module):
outputs --- [N, ref_enc_gru_size]
"""
- def __init__(self, spec_channels, gin_channels=0):
+ def __init__(self, spec_channels: int, gin_channels: int = 0) -> None:
super().__init__()
self.spec_channels = spec_channels
ref_enc_filters = [32, 32, 64, 64, 128, 128]
@@ -732,7 +796,9 @@ class ReferenceEncoder(nn.Module):
)
self.proj = nn.Linear(128, gin_channels)
- def forward(self, inputs, mask=None):
+ def forward(
+ self, inputs: torch.Tensor, mask: Optional[torch.Tensor] = None
+ ) -> torch.Tensor:
N = inputs.size(0)
out = inputs.view(N, 1, -1, self.spec_channels) # [N, 1, Ty, n_freqs]
for conv in self.convs:
@@ -750,7 +816,9 @@ class ReferenceEncoder(nn.Module):
return self.proj(out.squeeze(0))
- def calculate_channels(self, L, kernel_size, stride, pad, n_convs):
+ def calculate_channels(
+ self, L: int, kernel_size: int, stride: int, pad: int, n_convs: int
+ ) -> int:
for i in range(n_convs):
L = (L - kernel_size + 2 * pad) // stride + 1
return L
@@ -763,31 +831,31 @@ class SynthesizerTrn(nn.Module):
def __init__(
self,
- n_vocab,
- spec_channels,
- segment_size,
- inter_channels,
- hidden_channels,
- filter_channels,
- n_heads,
- n_layers,
- kernel_size,
- p_dropout,
- resblock,
- resblock_kernel_sizes,
- resblock_dilation_sizes,
- upsample_rates,
- upsample_initial_channel,
- upsample_kernel_sizes,
- n_speakers=256,
- gin_channels=256,
- use_sdp=True,
- n_flow_layer=4,
- n_layers_trans_flow=4,
- flow_share_parameter=False,
- use_transformer_flow=True,
- **kwargs,
- ):
+ n_vocab: int,
+ spec_channels: int,
+ segment_size: int,
+ inter_channels: int,
+ hidden_channels: int,
+ filter_channels: int,
+ n_heads: int,
+ n_layers: int,
+ kernel_size: int,
+ p_dropout: float,
+ resblock: str,
+ resblock_kernel_sizes: list[int],
+ resblock_dilation_sizes: list[list[int]],
+ upsample_rates: list[int],
+ upsample_initial_channel: int,
+ upsample_kernel_sizes: list[int],
+ n_speakers: int = 256,
+ gin_channels: int = 256,
+ use_sdp: bool = True,
+ n_flow_layer: int = 4,
+ n_layers_trans_flow: int = 4,
+ flow_share_parameter: bool = False,
+ use_transformer_flow: bool = True,
+ **kwargs: Any,
+ ) -> None:
super().__init__()
self.n_vocab = n_vocab
self.spec_channels = spec_channels
@@ -885,18 +953,27 @@ class SynthesizerTrn(nn.Module):
def forward(
self,
- x,
- x_lengths,
- y,
- y_lengths,
- sid,
- tone,
- language,
- bert,
- ja_bert,
- en_bert,
- style_vec,
- ):
+ x: torch.Tensor,
+ x_lengths: torch.Tensor,
+ y: torch.Tensor,
+ y_lengths: torch.Tensor,
+ sid: torch.Tensor,
+ tone: torch.Tensor,
+ language: torch.Tensor,
+ bert: torch.Tensor,
+ ja_bert: torch.Tensor,
+ en_bert: torch.Tensor,
+ style_vec: torch.Tensor,
+ ) -> tuple[
+ torch.Tensor,
+ torch.Tensor,
+ torch.Tensor,
+ torch.Tensor,
+ torch.Tensor,
+ torch.Tensor,
+ tuple[torch.Tensor, ...],
+ tuple[torch.Tensor, ...],
+ ]:
if self.n_speakers > 0:
g = self.emb_g(sid).unsqueeze(-1) # [b, h, 1]
else:
@@ -933,7 +1010,7 @@ class SynthesizerTrn(nn.Module):
attn_mask = torch.unsqueeze(x_mask, 2) * torch.unsqueeze(y_mask, -1)
attn = (
- monotonic_align.maximum_path(neg_cent, attn_mask.squeeze(1))
+ monotonic_alignment.maximum_path(neg_cent, attn_mask.squeeze(1))
.unsqueeze(1)
.detach()
)
@@ -974,27 +1051,28 @@ class SynthesizerTrn(nn.Module):
def infer(
self,
- x,
- x_lengths,
- sid,
- tone,
- language,
- bert,
- ja_bert,
- en_bert,
- style_vec,
- noise_scale=0.667,
- length_scale=1,
- noise_scale_w=0.8,
- max_len=None,
- sdp_ratio=0,
- y=None,
- ):
+ x: torch.Tensor,
+ x_lengths: torch.Tensor,
+ sid: torch.Tensor,
+ tone: torch.Tensor,
+ language: torch.Tensor,
+ bert: torch.Tensor,
+ ja_bert: torch.Tensor,
+ en_bert: torch.Tensor,
+ style_vec: torch.Tensor,
+ noise_scale: float = 0.667,
+ length_scale: float = 1.0,
+ noise_scale_w: float = 0.8,
+ max_len: Optional[int] = None,
+ sdp_ratio: float = 0.0,
+ y: Optional[torch.Tensor] = None,
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, tuple[torch.Tensor, ...]]:
# x, m_p, logs_p, x_mask = self.enc_p(x, x_lengths, tone, language, bert)
# g = self.gst(y)
if self.n_speakers > 0:
g = self.emb_g(sid).unsqueeze(-1) # [b, h, 1]
else:
+ assert y is not None
g = self.ref_enc(y.transpose(1, 2)).unsqueeze(-1)
x, m_p, logs_p, x_mask = self.enc_p(
x, x_lengths, tone, language, bert, ja_bert, en_bert, style_vec, sid, g=g
diff --git a/models_jp_extra.py b/style_bert_vits2/models/models_jp_extra.py
similarity index 77%
rename from models_jp_extra.py
rename to style_bert_vits2/models/models_jp_extra.py
index 1bb2dd2..2850baf 100644
--- a/models_jp_extra.py
+++ b/style_bert_vits2/models/models_jp_extra.py
@@ -1,24 +1,25 @@
import math
+from typing import Any, Optional
+
import torch
from torch import nn
+from torch.nn import Conv1d, Conv2d, ConvTranspose1d
from torch.nn import functional as F
+from torch.nn.utils import remove_weight_norm, spectral_norm, weight_norm
-import commons
-import modules
-import attentions
-import monotonic_align
-
-from torch.nn import Conv1d, ConvTranspose1d, Conv2d
-from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm
-
-from commons import init_weights, get_padding
-from text import symbols, num_tones, num_languages
+from style_bert_vits2.models import attentions, commons, modules, monotonic_alignment
+from style_bert_vits2.nlp.symbols import NUM_LANGUAGES, NUM_TONES, SYMBOLS
class DurationDiscriminator(nn.Module): # vits2
def __init__(
- self, in_channels, filter_channels, kernel_size, p_dropout, gin_channels=0
- ):
+ self,
+ in_channels: int,
+ filter_channels: int,
+ kernel_size: int,
+ p_dropout: float,
+ gin_channels: int = 0,
+ ) -> None:
super().__init__()
self.in_channels = in_channels
@@ -49,7 +50,7 @@ class DurationDiscriminator(nn.Module): # vits2
nn.Linear(2 * filter_channels, 1), nn.Sigmoid()
)
- def forward_probability(self, x, dur):
+ def forward_probability(self, x: torch.Tensor, dur: torch.Tensor) -> torch.Tensor:
dur = self.dur_proj(dur)
x = torch.cat([x, dur], dim=1)
x = x.transpose(1, 2)
@@ -57,7 +58,14 @@ class DurationDiscriminator(nn.Module): # vits2
output_prob = self.output_layer(x)
return output_prob
- def forward(self, x, x_mask, dur_r, dur_hat, g=None):
+ def forward(
+ self,
+ x: torch.Tensor,
+ x_mask: torch.Tensor,
+ dur_r: torch.Tensor,
+ dur_hat: torch.Tensor,
+ g: Optional[torch.Tensor] = None,
+ ) -> list[torch.Tensor]:
x = torch.detach(x)
if g is not None:
g = torch.detach(g)
@@ -82,17 +90,17 @@ class DurationDiscriminator(nn.Module): # vits2
class TransformerCouplingBlock(nn.Module):
def __init__(
self,
- channels,
- hidden_channels,
- filter_channels,
- n_heads,
- n_layers,
- kernel_size,
- p_dropout,
- n_flows=4,
- gin_channels=0,
- share_parameter=False,
- ):
+ channels: int,
+ hidden_channels: int,
+ filter_channels: int,
+ n_heads: int,
+ n_layers: int,
+ kernel_size: int,
+ p_dropout: float,
+ n_flows: int = 4,
+ gin_channels: int = 0,
+ share_parameter: bool = False,
+ ) -> None:
super().__init__()
self.channels = channels
self.hidden_channels = hidden_channels
@@ -104,16 +112,17 @@ class TransformerCouplingBlock(nn.Module):
self.flows = nn.ModuleList()
self.wn = (
- attentions.FFT(
- hidden_channels,
- filter_channels,
- n_heads,
- n_layers,
- kernel_size,
- p_dropout,
- isflow=True,
- gin_channels=self.gin_channels,
- )
+ # attentions.FFT(
+ # hidden_channels,
+ # filter_channels,
+ # n_heads,
+ # n_layers,
+ # kernel_size,
+ # p_dropout,
+ # isflow=True,
+ # gin_channels=self.gin_channels,
+ # )
+ None
if share_parameter
else None
)
@@ -135,7 +144,13 @@ class TransformerCouplingBlock(nn.Module):
)
self.flows.append(modules.Flip())
- def forward(self, x, x_mask, g=None, reverse=False):
+ def forward(
+ self,
+ x: torch.Tensor,
+ x_mask: torch.Tensor,
+ g: Optional[torch.Tensor] = None,
+ reverse: bool = False,
+ ) -> torch.Tensor:
if not reverse:
for flow in self.flows:
x, _ = flow(x, x_mask, g=g, reverse=reverse)
@@ -148,13 +163,13 @@ class TransformerCouplingBlock(nn.Module):
class StochasticDurationPredictor(nn.Module):
def __init__(
self,
- in_channels,
- filter_channels,
- kernel_size,
- p_dropout,
- n_flows=4,
- gin_channels=0,
- ):
+ in_channels: int,
+ filter_channels: int,
+ kernel_size: int,
+ p_dropout: float,
+ n_flows: int = 4,
+ gin_channels: int = 0,
+ ) -> None:
super().__init__()
filter_channels = in_channels # it needs to be removed from future version.
self.in_channels = in_channels
@@ -194,7 +209,15 @@ class StochasticDurationPredictor(nn.Module):
if gin_channels != 0:
self.cond = nn.Conv1d(gin_channels, filter_channels, 1)
- def forward(self, x, x_mask, w=None, g=None, reverse=False, noise_scale=1.0):
+ def forward(
+ self,
+ x: torch.Tensor,
+ x_mask: torch.Tensor,
+ w: Optional[torch.Tensor] = None,
+ g: Optional[torch.Tensor] = None,
+ reverse: bool = False,
+ noise_scale: float = 1.0,
+ ) -> torch.Tensor:
x = torch.detach(x)
x = self.pre(x)
if g is not None:
@@ -258,8 +281,13 @@ class StochasticDurationPredictor(nn.Module):
class DurationPredictor(nn.Module):
def __init__(
- self, in_channels, filter_channels, kernel_size, p_dropout, gin_channels=0
- ):
+ self,
+ in_channels: int,
+ filter_channels: int,
+ kernel_size: int,
+ p_dropout: float,
+ gin_channels: int = 0,
+ ) -> None:
super().__init__()
self.in_channels = in_channels
@@ -282,7 +310,9 @@ class DurationPredictor(nn.Module):
if gin_channels != 0:
self.cond = nn.Conv1d(gin_channels, in_channels, 1)
- def forward(self, x, x_mask, g=None):
+ def forward(
+ self, x: torch.Tensor, x_mask: torch.Tensor, g: Optional[torch.Tensor] = None
+ ) -> torch.Tensor:
x = torch.detach(x)
if g is not None:
g = torch.detach(g)
@@ -300,14 +330,14 @@ class DurationPredictor(nn.Module):
class Bottleneck(nn.Sequential):
- def __init__(self, in_dim, hidden_dim):
+ def __init__(self, in_dim: int, hidden_dim: int) -> None:
c_fc1 = nn.Linear(in_dim, hidden_dim, bias=False)
c_fc2 = nn.Linear(in_dim, hidden_dim, bias=False)
- super().__init__(*[c_fc1, c_fc2])
+ super().__init__(c_fc1, c_fc2)
class Block(nn.Module):
- def __init__(self, in_dim, hidden_dim) -> None:
+ def __init__(self, in_dim: int, hidden_dim: int) -> None:
super().__init__()
self.norm = nn.LayerNorm(in_dim)
self.mlp = MLP(in_dim, hidden_dim)
@@ -318,13 +348,13 @@ class Block(nn.Module):
class MLP(nn.Module):
- def __init__(self, in_dim, hidden_dim):
+ def __init__(self, in_dim: int, hidden_dim: int) -> None:
super().__init__()
self.c_fc1 = nn.Linear(in_dim, hidden_dim, bias=False)
self.c_fc2 = nn.Linear(in_dim, hidden_dim, bias=False)
self.c_proj = nn.Linear(hidden_dim, in_dim, bias=False)
- def forward(self, x: torch.Tensor):
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
x = F.silu(self.c_fc1(x)) * self.c_fc2(x)
x = self.c_proj(x)
return x
@@ -333,16 +363,16 @@ class MLP(nn.Module):
class TextEncoder(nn.Module):
def __init__(
self,
- n_vocab,
- out_channels,
- hidden_channels,
- filter_channels,
- n_heads,
- n_layers,
- kernel_size,
- p_dropout,
- gin_channels=0,
- ):
+ n_vocab: int,
+ out_channels: int,
+ hidden_channels: int,
+ filter_channels: int,
+ n_heads: int,
+ n_layers: int,
+ kernel_size: int,
+ p_dropout: float,
+ gin_channels: int = 0,
+ ) -> None:
super().__init__()
self.n_vocab = n_vocab
self.out_channels = out_channels
@@ -353,11 +383,11 @@ class TextEncoder(nn.Module):
self.kernel_size = kernel_size
self.p_dropout = p_dropout
self.gin_channels = gin_channels
- self.emb = nn.Embedding(len(symbols), hidden_channels)
+ self.emb = nn.Embedding(len(SYMBOLS), hidden_channels)
nn.init.normal_(self.emb.weight, 0.0, hidden_channels**-0.5)
- self.tone_emb = nn.Embedding(num_tones, hidden_channels)
+ self.tone_emb = nn.Embedding(NUM_TONES, hidden_channels)
nn.init.normal_(self.tone_emb.weight, 0.0, hidden_channels**-0.5)
- self.language_emb = nn.Embedding(num_languages, hidden_channels)
+ self.language_emb = nn.Embedding(NUM_LANGUAGES, hidden_channels)
nn.init.normal_(self.language_emb.weight, 0.0, hidden_channels**-0.5)
self.bert_proj = nn.Conv1d(1024, hidden_channels, 1)
@@ -375,7 +405,16 @@ class TextEncoder(nn.Module):
)
self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
- def forward(self, x, x_lengths, tone, language, bert, style_vec, g=None):
+ def forward(
+ self,
+ x: torch.Tensor,
+ x_lengths: torch.Tensor,
+ tone: torch.Tensor,
+ language: torch.Tensor,
+ bert: torch.Tensor,
+ style_vec: torch.Tensor,
+ g: Optional[torch.Tensor] = None,
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
bert_emb = self.bert_proj(bert).transpose(1, 2)
style_emb = self.style_proj(style_vec.unsqueeze(1))
x = (
@@ -402,14 +441,14 @@ class TextEncoder(nn.Module):
class ResidualCouplingBlock(nn.Module):
def __init__(
self,
- channels,
- hidden_channels,
- kernel_size,
- dilation_rate,
- n_layers,
- n_flows=4,
- gin_channels=0,
- ):
+ channels: int,
+ hidden_channels: int,
+ kernel_size: int,
+ dilation_rate: int,
+ n_layers: int,
+ n_flows: int = 4,
+ gin_channels: int = 0,
+ ) -> None:
super().__init__()
self.channels = channels
self.hidden_channels = hidden_channels
@@ -434,7 +473,13 @@ class ResidualCouplingBlock(nn.Module):
)
self.flows.append(modules.Flip())
- def forward(self, x, x_mask, g=None, reverse=False):
+ def forward(
+ self,
+ x: torch.Tensor,
+ x_mask: torch.Tensor,
+ g: Optional[torch.Tensor] = None,
+ reverse: bool = False,
+ ) -> torch.Tensor:
if not reverse:
for flow in self.flows:
x, _ = flow(x, x_mask, g=g, reverse=reverse)
@@ -447,14 +492,14 @@ class ResidualCouplingBlock(nn.Module):
class PosteriorEncoder(nn.Module):
def __init__(
self,
- in_channels,
- out_channels,
- hidden_channels,
- kernel_size,
- dilation_rate,
- n_layers,
- gin_channels=0,
- ):
+ in_channels: int,
+ out_channels: int,
+ hidden_channels: int,
+ kernel_size: int,
+ dilation_rate: int,
+ n_layers: int,
+ gin_channels: int = 0,
+ ) -> None:
super().__init__()
self.in_channels = in_channels
self.out_channels = out_channels
@@ -474,7 +519,12 @@ class PosteriorEncoder(nn.Module):
)
self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
- def forward(self, x, x_lengths, g=None):
+ def forward(
+ self,
+ x: torch.Tensor,
+ x_lengths: torch.Tensor,
+ g: Optional[torch.Tensor] = None,
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(
x.dtype
)
@@ -489,22 +539,22 @@ class PosteriorEncoder(nn.Module):
class Generator(torch.nn.Module):
def __init__(
self,
- initial_channel,
- resblock,
- resblock_kernel_sizes,
- resblock_dilation_sizes,
- upsample_rates,
- upsample_initial_channel,
- upsample_kernel_sizes,
- gin_channels=0,
- ):
+ initial_channel: int,
+ resblock_str: str,
+ resblock_kernel_sizes: list[int],
+ resblock_dilation_sizes: list[list[int]],
+ upsample_rates: list[int],
+ upsample_initial_channel: int,
+ upsample_kernel_sizes: list[int],
+ gin_channels: int = 0,
+ ) -> None:
super(Generator, self).__init__()
self.num_kernels = len(resblock_kernel_sizes)
self.num_upsamples = len(upsample_rates)
self.conv_pre = Conv1d(
initial_channel, upsample_initial_channel, 7, 1, padding=3
)
- resblock = modules.ResBlock1 if resblock == "1" else modules.ResBlock2
+ resblock = modules.ResBlock1 if resblock_str == "1" else modules.ResBlock2
self.ups = nn.ModuleList()
for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):
@@ -521,20 +571,24 @@ class Generator(torch.nn.Module):
)
self.resblocks = nn.ModuleList()
+ ch = None
for i in range(len(self.ups)):
ch = upsample_initial_channel // (2 ** (i + 1))
for j, (k, d) in enumerate(
zip(resblock_kernel_sizes, resblock_dilation_sizes)
):
- self.resblocks.append(resblock(ch, k, d))
+ self.resblocks.append(resblock(ch, k, d)) # type: ignore
+ assert ch is not None
self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False)
- self.ups.apply(init_weights)
+ self.ups.apply(commons.init_weights)
if gin_channels != 0:
self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1)
- def forward(self, x, g=None):
+ def forward(
+ self, x: torch.Tensor, g: Optional[torch.Tensor] = None
+ ) -> torch.Tensor:
x = self.conv_pre(x)
if g is not None:
x = x + self.cond(g)
@@ -548,6 +602,7 @@ class Generator(torch.nn.Module):
xs = self.resblocks[i * self.num_kernels + j](x)
else:
xs += self.resblocks[i * self.num_kernels + j](x)
+ assert xs is not None
x = xs / self.num_kernels
x = F.leaky_relu(x)
x = self.conv_post(x)
@@ -555,7 +610,7 @@ class Generator(torch.nn.Module):
return x
- def remove_weight_norm(self):
+ def remove_weight_norm(self) -> None:
print("Removing weight norm...")
for layer in self.ups:
remove_weight_norm(layer)
@@ -564,7 +619,13 @@ class Generator(torch.nn.Module):
class DiscriminatorP(torch.nn.Module):
- def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False):
+ def __init__(
+ self,
+ period: int,
+ kernel_size: int = 5,
+ stride: int = 3,
+ use_spectral_norm: bool = False,
+ ) -> None:
super(DiscriminatorP, self).__init__()
self.period = period
self.use_spectral_norm = use_spectral_norm
@@ -577,7 +638,7 @@ class DiscriminatorP(torch.nn.Module):
32,
(kernel_size, 1),
(stride, 1),
- padding=(get_padding(kernel_size, 1), 0),
+ padding=(commons.get_padding(kernel_size, 1), 0),
)
),
norm_f(
@@ -586,7 +647,7 @@ class DiscriminatorP(torch.nn.Module):
128,
(kernel_size, 1),
(stride, 1),
- padding=(get_padding(kernel_size, 1), 0),
+ padding=(commons.get_padding(kernel_size, 1), 0),
)
),
norm_f(
@@ -595,7 +656,7 @@ class DiscriminatorP(torch.nn.Module):
512,
(kernel_size, 1),
(stride, 1),
- padding=(get_padding(kernel_size, 1), 0),
+ padding=(commons.get_padding(kernel_size, 1), 0),
)
),
norm_f(
@@ -604,7 +665,7 @@ class DiscriminatorP(torch.nn.Module):
1024,
(kernel_size, 1),
(stride, 1),
- padding=(get_padding(kernel_size, 1), 0),
+ padding=(commons.get_padding(kernel_size, 1), 0),
)
),
norm_f(
@@ -613,14 +674,14 @@ class DiscriminatorP(torch.nn.Module):
1024,
(kernel_size, 1),
1,
- padding=(get_padding(kernel_size, 1), 0),
+ padding=(commons.get_padding(kernel_size, 1), 0),
)
),
]
)
self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0)))
- def forward(self, x):
+ def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, list[torch.Tensor]]:
fmap = []
# 1d to 2d
@@ -643,7 +704,7 @@ class DiscriminatorP(torch.nn.Module):
class DiscriminatorS(torch.nn.Module):
- def __init__(self, use_spectral_norm=False):
+ def __init__(self, use_spectral_norm: bool = False) -> None:
super(DiscriminatorS, self).__init__()
norm_f = weight_norm if use_spectral_norm is False else spectral_norm
self.convs = nn.ModuleList(
@@ -658,7 +719,7 @@ class DiscriminatorS(torch.nn.Module):
)
self.conv_post = norm_f(Conv1d(1024, 1, 3, 1, padding=1))
- def forward(self, x):
+ def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, list[torch.Tensor]]:
fmap = []
for layer in self.convs:
@@ -673,7 +734,7 @@ class DiscriminatorS(torch.nn.Module):
class MultiPeriodDiscriminator(torch.nn.Module):
- def __init__(self, use_spectral_norm=False):
+ def __init__(self, use_spectral_norm: bool = False) -> None:
super(MultiPeriodDiscriminator, self).__init__()
periods = [2, 3, 5, 7, 11]
@@ -683,7 +744,13 @@ class MultiPeriodDiscriminator(torch.nn.Module):
]
self.discriminators = nn.ModuleList(discs)
- def forward(self, y, y_hat):
+ def forward(
+ self,
+ y: torch.Tensor,
+ y_hat: torch.Tensor,
+ ) -> tuple[
+ list[torch.Tensor], list[torch.Tensor], list[torch.Tensor], list[torch.Tensor]
+ ]:
y_d_rs = []
y_d_gs = []
fmap_rs = []
@@ -703,8 +770,12 @@ class WavLMDiscriminator(nn.Module):
"""docstring for Discriminator."""
def __init__(
- self, slm_hidden=768, slm_layers=13, initial_channel=64, use_spectral_norm=False
- ):
+ self,
+ slm_hidden: int = 768,
+ slm_layers: int = 13,
+ initial_channel: int = 64,
+ use_spectral_norm: bool = False,
+ ) -> None:
super(WavLMDiscriminator, self).__init__()
norm_f = weight_norm if use_spectral_norm == False else spectral_norm
self.pre = norm_f(
@@ -734,7 +805,7 @@ class WavLMDiscriminator(nn.Module):
self.conv_post = norm_f(Conv1d(initial_channel * 4, 1, 3, 1, padding=1))
- def forward(self, x):
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
x = self.pre(x)
fmap = []
@@ -754,7 +825,7 @@ class ReferenceEncoder(nn.Module):
outputs --- [N, ref_enc_gru_size]
"""
- def __init__(self, spec_channels, gin_channels=0):
+ def __init__(self, spec_channels: int, gin_channels: int = 0) -> None:
super().__init__()
self.spec_channels = spec_channels
ref_enc_filters = [32, 32, 64, 64, 128, 128]
@@ -783,7 +854,9 @@ class ReferenceEncoder(nn.Module):
)
self.proj = nn.Linear(128, gin_channels)
- def forward(self, inputs, mask=None):
+ def forward(
+ self, inputs: torch.Tensor, mask: Optional[torch.Tensor] = None
+ ) -> torch.Tensor:
N = inputs.size(0)
out = inputs.view(N, 1, -1, self.spec_channels) # [N, 1, Ty, n_freqs]
for conv in self.convs:
@@ -801,7 +874,9 @@ class ReferenceEncoder(nn.Module):
return self.proj(out.squeeze(0))
- def calculate_channels(self, L, kernel_size, stride, pad, n_convs):
+ def calculate_channels(
+ self, L: int, kernel_size: int, stride: int, pad: int, n_convs: int
+ ) -> int:
for i in range(n_convs):
L = (L - kernel_size + 2 * pad) // stride + 1
return L
@@ -814,31 +889,31 @@ class SynthesizerTrn(nn.Module):
def __init__(
self,
- n_vocab,
- spec_channels,
- segment_size,
- inter_channels,
- hidden_channels,
- filter_channels,
- n_heads,
- n_layers,
- kernel_size,
- p_dropout,
- resblock,
- resblock_kernel_sizes,
- resblock_dilation_sizes,
- upsample_rates,
- upsample_initial_channel,
- upsample_kernel_sizes,
- n_speakers=256,
- gin_channels=256,
- use_sdp=True,
- n_flow_layer=4,
- n_layers_trans_flow=6,
- flow_share_parameter=False,
- use_transformer_flow=True,
- **kwargs
- ):
+ n_vocab: int,
+ spec_channels: int,
+ segment_size: int,
+ inter_channels: int,
+ hidden_channels: int,
+ filter_channels: int,
+ n_heads: int,
+ n_layers: int,
+ kernel_size: int,
+ p_dropout: float,
+ resblock: str,
+ resblock_kernel_sizes: list[int],
+ resblock_dilation_sizes: list[list[int]],
+ upsample_rates: list[int],
+ upsample_initial_channel: int,
+ upsample_kernel_sizes: list[int],
+ n_speakers: int = 256,
+ gin_channels: int = 256,
+ use_sdp: bool = True,
+ n_flow_layer: int = 4,
+ n_layers_trans_flow: int = 6,
+ flow_share_parameter: bool = False,
+ use_transformer_flow: bool = True,
+ **kwargs: Any,
+ ) -> None:
super().__init__()
self.n_vocab = n_vocab
self.spec_channels = spec_channels
@@ -935,16 +1010,26 @@ class SynthesizerTrn(nn.Module):
def forward(
self,
- x,
- x_lengths,
- y,
- y_lengths,
- sid,
- tone,
- language,
- bert,
- style_vec,
- ):
+ x: torch.Tensor,
+ x_lengths: torch.Tensor,
+ y: torch.Tensor,
+ y_lengths: torch.Tensor,
+ sid: torch.Tensor,
+ tone: torch.Tensor,
+ language: torch.Tensor,
+ bert: torch.Tensor,
+ style_vec: torch.Tensor,
+ ) -> tuple[
+ torch.Tensor,
+ torch.Tensor,
+ torch.Tensor,
+ torch.Tensor,
+ torch.Tensor,
+ torch.Tensor,
+ torch.Tensor,
+ tuple[torch.Tensor, ...],
+ tuple[torch.Tensor, ...],
+ ]:
if self.n_speakers > 0:
g = self.emb_g(sid).unsqueeze(-1) # [b, h, 1]
else:
@@ -981,7 +1066,7 @@ class SynthesizerTrn(nn.Module):
attn_mask = torch.unsqueeze(x_mask, 2) * torch.unsqueeze(y_mask, -1)
attn = (
- monotonic_align.maximum_path(neg_cent, attn_mask.squeeze(1))
+ monotonic_alignment.maximum_path(neg_cent, attn_mask.squeeze(1))
.unsqueeze(1)
.detach()
)
@@ -1016,32 +1101,33 @@ class SynthesizerTrn(nn.Module):
ids_slice,
x_mask,
y_mask,
- (z, z_p, m_p, logs_p, m_q, logs_q),
+ (z, z_p, m_p, logs_p, m_q, logs_q), # type: ignore
(x, logw, logw_), # , logw_sdp),
g,
)
def infer(
self,
- x,
- x_lengths,
- sid,
- tone,
- language,
- bert,
- style_vec,
- noise_scale=0.667,
- length_scale=1,
- noise_scale_w=0.8,
- max_len=None,
- sdp_ratio=0,
- y=None,
- ):
+ x: torch.Tensor,
+ x_lengths: torch.Tensor,
+ sid: torch.Tensor,
+ tone: torch.Tensor,
+ language: torch.Tensor,
+ bert: torch.Tensor,
+ style_vec: torch.Tensor,
+ noise_scale: float = 0.667,
+ length_scale: float = 1.0,
+ noise_scale_w: float = 0.8,
+ max_len: Optional[int] = None,
+ sdp_ratio: float = 0.0,
+ y: Optional[torch.Tensor] = None,
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, tuple[torch.Tensor, ...]]:
# x, m_p, logs_p, x_mask = self.enc_p(x, x_lengths, tone, language, bert)
# g = self.gst(y)
if self.n_speakers > 0:
g = self.emb_g(sid).unsqueeze(-1) # [b, h, 1]
else:
+ assert y is not None
g = self.ref_enc(y.transpose(1, 2)).unsqueeze(-1)
x, m_p, logs_p, x_mask = self.enc_p(
x, x_lengths, tone, language, bert, style_vec, g=g
diff --git a/modules.py b/style_bert_vits2/models/modules.py
similarity index 76%
rename from modules.py
rename to style_bert_vits2/models/modules.py
index 86b93b5..ebc5273 100644
--- a/modules.py
+++ b/style_bert_vits2/models/modules.py
@@ -1,5 +1,5 @@
import math
-import warnings
+from typing import Any, Optional, Union
import torch
from torch import nn
@@ -7,16 +7,16 @@ from torch.nn import Conv1d
from torch.nn import functional as F
from torch.nn.utils import remove_weight_norm, weight_norm
-import commons
-from attentions import Encoder
-from commons import get_padding, init_weights
-from transforms import piecewise_rational_quadratic_transform
+from style_bert_vits2.models import commons
+from style_bert_vits2.models.attentions import Encoder
+from style_bert_vits2.models.transforms import piecewise_rational_quadratic_transform
+
LRELU_SLOPE = 0.1
class LayerNorm(nn.Module):
- def __init__(self, channels, eps=1e-5):
+ def __init__(self, channels: int, eps: float = 1e-5) -> None:
super().__init__()
self.channels = channels
self.eps = eps
@@ -24,7 +24,7 @@ class LayerNorm(nn.Module):
self.gamma = nn.Parameter(torch.ones(channels))
self.beta = nn.Parameter(torch.zeros(channels))
- def forward(self, x):
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
x = x.transpose(1, -1)
x = F.layer_norm(x, (self.channels,), self.gamma, self.beta, self.eps)
return x.transpose(1, -1)
@@ -33,13 +33,13 @@ class LayerNorm(nn.Module):
class ConvReluNorm(nn.Module):
def __init__(
self,
- in_channels,
- hidden_channels,
- out_channels,
- kernel_size,
- n_layers,
- p_dropout,
- ):
+ in_channels: int,
+ hidden_channels: int,
+ out_channels: int,
+ kernel_size: int,
+ n_layers: int,
+ p_dropout: float,
+ ) -> None:
super().__init__()
self.in_channels = in_channels
self.hidden_channels = hidden_channels
@@ -70,9 +70,10 @@ class ConvReluNorm(nn.Module):
self.norm_layers.append(LayerNorm(hidden_channels))
self.proj = nn.Conv1d(hidden_channels, out_channels, 1)
self.proj.weight.data.zero_()
+ assert self.proj.bias is not None
self.proj.bias.data.zero_()
- def forward(self, x, x_mask):
+ def forward(self, x: torch.Tensor, x_mask: torch.Tensor) -> torch.Tensor:
x_org = x
for i in range(self.n_layers):
x = self.conv_layers[i](x * x_mask)
@@ -87,7 +88,9 @@ class DDSConv(nn.Module):
Dialted and Depth-Separable Convolution
"""
- def __init__(self, channels, kernel_size, n_layers, p_dropout=0.0):
+ def __init__(
+ self, channels: int, kernel_size: int, n_layers: int, p_dropout: float = 0.0
+ ) -> None:
super().__init__()
self.channels = channels
self.kernel_size = kernel_size
@@ -116,7 +119,9 @@ class DDSConv(nn.Module):
self.norms_1.append(LayerNorm(channels))
self.norms_2.append(LayerNorm(channels))
- def forward(self, x, x_mask, g=None):
+ def forward(
+ self, x: torch.Tensor, x_mask: torch.Tensor, g: Optional[torch.Tensor] = None
+ ) -> torch.Tensor:
if g is not None:
x = x + g
for i in range(self.n_layers):
@@ -134,13 +139,13 @@ class DDSConv(nn.Module):
class WN(torch.nn.Module):
def __init__(
self,
- hidden_channels,
- kernel_size,
- dilation_rate,
- n_layers,
- gin_channels=0,
- p_dropout=0,
- ):
+ hidden_channels: int,
+ kernel_size: int,
+ dilation_rate: int,
+ n_layers: int,
+ gin_channels: int = 0,
+ p_dropout: float = 0,
+ ) -> None:
super(WN, self).__init__()
assert kernel_size % 2 == 1
self.hidden_channels = hidden_channels
@@ -183,7 +188,13 @@ class WN(torch.nn.Module):
res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name="weight")
self.res_skip_layers.append(res_skip_layer)
- def forward(self, x, x_mask, g=None, **kwargs):
+ def forward(
+ self,
+ x: torch.Tensor,
+ x_mask: torch.Tensor,
+ g: Optional[torch.Tensor] = None,
+ **kwargs: Any,
+ ) -> torch.Tensor:
output = torch.zeros_like(x)
n_channels_tensor = torch.IntTensor([self.hidden_channels])
@@ -210,7 +221,7 @@ class WN(torch.nn.Module):
output = output + res_skip_acts
return output * x_mask
- def remove_weight_norm(self):
+ def remove_weight_norm(self) -> None:
if self.gin_channels != 0:
torch.nn.utils.remove_weight_norm(self.cond_layer)
for l in self.in_layers:
@@ -220,7 +231,12 @@ class WN(torch.nn.Module):
class ResBlock1(torch.nn.Module):
- def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5)):
+ def __init__(
+ self,
+ channels: int,
+ kernel_size: int = 3,
+ dilation: tuple[int, int, int] = (1, 3, 5),
+ ) -> None:
super(ResBlock1, self).__init__()
self.convs1 = nn.ModuleList(
[
@@ -231,7 +247,7 @@ class ResBlock1(torch.nn.Module):
kernel_size,
1,
dilation=dilation[0],
- padding=get_padding(kernel_size, dilation[0]),
+ padding=commons.get_padding(kernel_size, dilation[0]),
)
),
weight_norm(
@@ -241,7 +257,7 @@ class ResBlock1(torch.nn.Module):
kernel_size,
1,
dilation=dilation[1],
- padding=get_padding(kernel_size, dilation[1]),
+ padding=commons.get_padding(kernel_size, dilation[1]),
)
),
weight_norm(
@@ -251,12 +267,12 @@ class ResBlock1(torch.nn.Module):
kernel_size,
1,
dilation=dilation[2],
- padding=get_padding(kernel_size, dilation[2]),
+ padding=commons.get_padding(kernel_size, dilation[2]),
)
),
]
)
- self.convs1.apply(init_weights)
+ self.convs1.apply(commons.init_weights)
self.convs2 = nn.ModuleList(
[
@@ -267,7 +283,7 @@ class ResBlock1(torch.nn.Module):
kernel_size,
1,
dilation=1,
- padding=get_padding(kernel_size, 1),
+ padding=commons.get_padding(kernel_size, 1),
)
),
weight_norm(
@@ -277,7 +293,7 @@ class ResBlock1(torch.nn.Module):
kernel_size,
1,
dilation=1,
- padding=get_padding(kernel_size, 1),
+ padding=commons.get_padding(kernel_size, 1),
)
),
weight_norm(
@@ -287,14 +303,16 @@ class ResBlock1(torch.nn.Module):
kernel_size,
1,
dilation=1,
- padding=get_padding(kernel_size, 1),
+ padding=commons.get_padding(kernel_size, 1),
)
),
]
)
- self.convs2.apply(init_weights)
+ self.convs2.apply(commons.init_weights)
- def forward(self, x, x_mask=None):
+ def forward(
+ self, x: torch.Tensor, x_mask: Optional[torch.Tensor] = None
+ ) -> torch.Tensor:
for c1, c2 in zip(self.convs1, self.convs2):
xt = F.leaky_relu(x, LRELU_SLOPE)
if x_mask is not None:
@@ -309,7 +327,7 @@ class ResBlock1(torch.nn.Module):
x = x * x_mask
return x
- def remove_weight_norm(self):
+ def remove_weight_norm(self) -> None:
for l in self.convs1:
remove_weight_norm(l)
for l in self.convs2:
@@ -317,7 +335,9 @@ class ResBlock1(torch.nn.Module):
class ResBlock2(torch.nn.Module):
- def __init__(self, channels, kernel_size=3, dilation=(1, 3)):
+ def __init__(
+ self, channels: int, kernel_size: int = 3, dilation: tuple[int, int] = (1, 3)
+ ) -> None:
super(ResBlock2, self).__init__()
self.convs = nn.ModuleList(
[
@@ -328,7 +348,7 @@ class ResBlock2(torch.nn.Module):
kernel_size,
1,
dilation=dilation[0],
- padding=get_padding(kernel_size, dilation[0]),
+ padding=commons.get_padding(kernel_size, dilation[0]),
)
),
weight_norm(
@@ -338,14 +358,16 @@ class ResBlock2(torch.nn.Module):
kernel_size,
1,
dilation=dilation[1],
- padding=get_padding(kernel_size, dilation[1]),
+ padding=commons.get_padding(kernel_size, dilation[1]),
)
),
]
)
- self.convs.apply(init_weights)
+ self.convs.apply(commons.init_weights)
- def forward(self, x, x_mask=None):
+ def forward(
+ self, x: torch.Tensor, x_mask: Optional[torch.Tensor] = None
+ ) -> torch.Tensor:
for c in self.convs:
xt = F.leaky_relu(x, LRELU_SLOPE)
if x_mask is not None:
@@ -356,13 +378,19 @@ class ResBlock2(torch.nn.Module):
x = x * x_mask
return x
- def remove_weight_norm(self):
+ def remove_weight_norm(self) -> None:
for l in self.convs:
remove_weight_norm(l)
class Log(nn.Module):
- def forward(self, x, x_mask, reverse=False, **kwargs):
+ def forward(
+ self,
+ x: torch.Tensor,
+ x_mask: torch.Tensor,
+ reverse: bool = False,
+ **kwargs: Any,
+ ) -> Union[tuple[torch.Tensor, torch.Tensor], torch.Tensor]:
if not reverse:
y = torch.log(torch.clamp_min(x, 1e-5)) * x_mask
logdet = torch.sum(-y, [1, 2])
@@ -373,7 +401,13 @@ class Log(nn.Module):
class Flip(nn.Module):
- def forward(self, x, *args, reverse=False, **kwargs):
+ def forward(
+ self,
+ x: torch.Tensor,
+ *args: Any,
+ reverse: bool = False,
+ **kwargs: Any,
+ ) -> Union[tuple[torch.Tensor, torch.Tensor], torch.Tensor]:
x = torch.flip(x, [1])
if not reverse:
logdet = torch.zeros(x.size(0)).to(dtype=x.dtype, device=x.device)
@@ -383,13 +417,19 @@ class Flip(nn.Module):
class ElementwiseAffine(nn.Module):
- def __init__(self, channels):
+ def __init__(self, channels: int) -> None:
super().__init__()
self.channels = channels
self.m = nn.Parameter(torch.zeros(channels, 1))
self.logs = nn.Parameter(torch.zeros(channels, 1))
- def forward(self, x, x_mask, reverse=False, **kwargs):
+ def forward(
+ self,
+ x: torch.Tensor,
+ x_mask: torch.Tensor,
+ reverse: bool = False,
+ **kwargs: Any,
+ ) -> Union[tuple[torch.Tensor, torch.Tensor], torch.Tensor]:
if not reverse:
y = self.m + torch.exp(self.logs) * x
y = y * x_mask
@@ -403,15 +443,15 @@ class ElementwiseAffine(nn.Module):
class ResidualCouplingLayer(nn.Module):
def __init__(
self,
- channels,
- hidden_channels,
- kernel_size,
- dilation_rate,
- n_layers,
- p_dropout=0,
- gin_channels=0,
- mean_only=False,
- ):
+ channels: int,
+ hidden_channels: int,
+ kernel_size: int,
+ dilation_rate: int,
+ n_layers: int,
+ p_dropout: float = 0,
+ gin_channels: int = 0,
+ mean_only: bool = False,
+ ) -> None:
assert channels % 2 == 0, "channels should be divisible by 2"
super().__init__()
self.channels = channels
@@ -433,9 +473,16 @@ class ResidualCouplingLayer(nn.Module):
)
self.post = nn.Conv1d(hidden_channels, self.half_channels * (2 - mean_only), 1)
self.post.weight.data.zero_()
+ assert self.post.bias is not None
self.post.bias.data.zero_()
- def forward(self, x, x_mask, g=None, reverse=False):
+ def forward(
+ self,
+ x: torch.Tensor,
+ x_mask: torch.Tensor,
+ g: Optional[torch.Tensor] = None,
+ reverse: bool = False,
+ ) -> Union[tuple[torch.Tensor, torch.Tensor], torch.Tensor]:
x0, x1 = torch.split(x, [self.half_channels] * 2, 1)
h = self.pre(x0) * x_mask
h = self.enc(h, x_mask, g=g)
@@ -460,13 +507,13 @@ class ResidualCouplingLayer(nn.Module):
class ConvFlow(nn.Module):
def __init__(
self,
- in_channels,
- filter_channels,
- kernel_size,
- n_layers,
- num_bins=10,
- tail_bound=5.0,
- ):
+ in_channels: int,
+ filter_channels: int,
+ kernel_size: int,
+ n_layers: int,
+ num_bins: int = 10,
+ tail_bound: float = 5.0,
+ ) -> None:
super().__init__()
self.in_channels = in_channels
self.filter_channels = filter_channels
@@ -482,9 +529,16 @@ class ConvFlow(nn.Module):
filter_channels, self.half_channels * (num_bins * 3 - 1), 1
)
self.proj.weight.data.zero_()
+ assert self.proj.bias is not None
self.proj.bias.data.zero_()
- def forward(self, x, x_mask, g=None, reverse=False):
+ def forward(
+ self,
+ x: torch.Tensor,
+ x_mask: torch.Tensor,
+ g: Optional[torch.Tensor] = None,
+ reverse: bool = False,
+ ) -> Union[tuple[torch.Tensor, torch.Tensor], torch.Tensor]:
x0, x1 = torch.split(x, [self.half_channels] * 2, 1)
h = self.pre(x0)
h = self.convs(h, x_mask, g=g)
@@ -520,17 +574,17 @@ class ConvFlow(nn.Module):
class TransformerCouplingLayer(nn.Module):
def __init__(
self,
- channels,
- hidden_channels,
- kernel_size,
- n_layers,
- n_heads,
- p_dropout=0,
- filter_channels=0,
- mean_only=False,
- wn_sharing_parameter=None,
- gin_channels=0,
- ):
+ channels: int,
+ hidden_channels: int,
+ kernel_size: int,
+ n_layers: int,
+ n_heads: int,
+ p_dropout: float = 0,
+ filter_channels: int = 0,
+ mean_only: bool = False,
+ wn_sharing_parameter: Optional[nn.Module] = None,
+ gin_channels: int = 0,
+ ) -> None:
assert channels % 2 == 0, "channels should be divisible by 2"
super().__init__()
self.channels = channels
@@ -557,9 +611,16 @@ class TransformerCouplingLayer(nn.Module):
)
self.post = nn.Conv1d(hidden_channels, self.half_channels * (2 - mean_only), 1)
self.post.weight.data.zero_()
+ assert self.post.bias is not None
self.post.bias.data.zero_()
- def forward(self, x, x_mask, g=None, reverse=False):
+ def forward(
+ self,
+ x: torch.Tensor,
+ x_mask: torch.Tensor,
+ g: Optional[torch.Tensor] = None,
+ reverse: bool = False,
+ ) -> Union[tuple[torch.Tensor, torch.Tensor], torch.Tensor]:
x0, x1 = torch.split(x, [self.half_channels] * 2, 1)
h = self.pre(x0) * x_mask
h = self.enc(h, x_mask, g=g)
diff --git a/style_bert_vits2/models/monotonic_alignment.py b/style_bert_vits2/models/monotonic_alignment.py
new file mode 100644
index 0000000..8ec9d24
--- /dev/null
+++ b/style_bert_vits2/models/monotonic_alignment.py
@@ -0,0 +1,89 @@
+"""
+以下に記述されている関数のコメントはリファクタリング時に GPT-4 に生成させたもので、
+コードと完全に一致している保証はない。あくまで参考程度とすること。
+"""
+
+from typing import Any
+
+import numba
+import torch
+from numpy import float32, int32, zeros
+
+
+def maximum_path(neg_cent: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
+ """
+ 与えられた負の中心とマスクを使用して最大パスを計算する
+
+ Args:
+ neg_cent (torch.Tensor): 負の中心を表すテンソル
+ mask (torch.Tensor): マスクを表すテンソル
+
+ Returns:
+ Tensor: 計算された最大パスを表すテンソル
+ """
+
+ device = neg_cent.device
+ dtype = neg_cent.dtype
+ neg_cent = neg_cent.data.cpu().numpy().astype(float32)
+ path = zeros(neg_cent.shape, dtype=int32)
+
+ t_t_max = mask.sum(1)[:, 0].data.cpu().numpy().astype(int32)
+ t_s_max = mask.sum(2)[:, 0].data.cpu().numpy().astype(int32)
+ __maximum_path_jit(path, neg_cent, t_t_max, t_s_max)
+
+ return torch.from_numpy(path).to(device=device, dtype=dtype)
+
+
+@numba.jit(
+ numba.void(
+ numba.int32[:, :, ::1],
+ numba.float32[:, :, ::1],
+ numba.int32[::1],
+ numba.int32[::1],
+ ),
+ nopython=True,
+ nogil=True,
+) # type: ignore
+def __maximum_path_jit(paths: Any, values: Any, t_ys: Any, t_xs: Any) -> None:
+ """
+ 与えられたパス、値、およびターゲットの y と x 座標を使用して JIT で最大パスを計算する
+
+ Args:
+ paths: 計算されたパスを格納するための整数型の 3 次元配列
+ values: 値を格納するための浮動小数点型の 3 次元配列
+ t_ys: ターゲットの y 座標を格納するための整数型の 1 次元配列
+ t_xs: ターゲットの x 座標を格納するための整数型の 1 次元配列
+ """
+
+ b = paths.shape[0]
+ max_neg_val = -1e9
+ for i in range(int(b)):
+ path = paths[i]
+ value = values[i]
+ t_y = t_ys[i]
+ t_x = t_xs[i]
+
+ v_prev = v_cur = 0.0
+ index = t_x - 1
+
+ for y in range(t_y):
+ for x in range(max(0, t_x + y - t_y), min(t_x, y + 1)):
+ if x == y:
+ v_cur = max_neg_val
+ else:
+ v_cur = value[y - 1, x]
+ if x == 0:
+ if y == 0:
+ v_prev = 0.0
+ else:
+ v_prev = max_neg_val
+ else:
+ v_prev = value[y - 1, x - 1]
+ value[y, x] += max(v_prev, v_cur)
+
+ for y in range(t_y - 1, -1, -1):
+ path[y, index] = 1
+ if index != 0 and (
+ index == y or value[y - 1, index] < value[y - 1, index - 1]
+ ):
+ index = index - 1
diff --git a/transforms.py b/style_bert_vits2/models/transforms.py
similarity index 79%
rename from transforms.py
rename to style_bert_vits2/models/transforms.py
index a11f799..b6f4420 100644
--- a/transforms.py
+++ b/style_bert_vits2/models/transforms.py
@@ -1,7 +1,8 @@
-import torch
-from torch.nn import functional as F
+from typing import Optional
import numpy as np
+import torch
+from torch.nn import functional as F
DEFAULT_MIN_BIN_WIDTH = 1e-3
@@ -10,17 +11,18 @@ DEFAULT_MIN_DERIVATIVE = 1e-3
def piecewise_rational_quadratic_transform(
- inputs,
- unnormalized_widths,
- unnormalized_heights,
- unnormalized_derivatives,
- inverse=False,
- tails=None,
- tail_bound=1.0,
- min_bin_width=DEFAULT_MIN_BIN_WIDTH,
- min_bin_height=DEFAULT_MIN_BIN_HEIGHT,
- min_derivative=DEFAULT_MIN_DERIVATIVE,
-):
+ inputs: torch.Tensor,
+ unnormalized_widths: torch.Tensor,
+ unnormalized_heights: torch.Tensor,
+ unnormalized_derivatives: torch.Tensor,
+ inverse: bool = False,
+ tails: Optional[str] = None,
+ tail_bound: float = 1.0,
+ min_bin_width: float = DEFAULT_MIN_BIN_WIDTH,
+ min_bin_height: float = DEFAULT_MIN_BIN_HEIGHT,
+ min_derivative: float = DEFAULT_MIN_DERIVATIVE,
+) -> tuple[torch.Tensor, torch.Tensor]:
+
if tails is None:
spline_fn = rational_quadratic_spline
spline_kwargs = {}
@@ -37,28 +39,31 @@ def piecewise_rational_quadratic_transform(
min_bin_width=min_bin_width,
min_bin_height=min_bin_height,
min_derivative=min_derivative,
- **spline_kwargs
+ **spline_kwargs, # type: ignore
)
return outputs, logabsdet
-def searchsorted(bin_locations, inputs, eps=1e-6):
+def searchsorted(
+ bin_locations: torch.Tensor, inputs: torch.Tensor, eps: float = 1e-6
+) -> torch.Tensor:
bin_locations[..., -1] += eps
return torch.sum(inputs[..., None] >= bin_locations, dim=-1) - 1
def unconstrained_rational_quadratic_spline(
- inputs,
- unnormalized_widths,
- unnormalized_heights,
- unnormalized_derivatives,
- inverse=False,
- tails="linear",
- tail_bound=1.0,
- min_bin_width=DEFAULT_MIN_BIN_WIDTH,
- min_bin_height=DEFAULT_MIN_BIN_HEIGHT,
- min_derivative=DEFAULT_MIN_DERIVATIVE,
-):
+ inputs: torch.Tensor,
+ unnormalized_widths: torch.Tensor,
+ unnormalized_heights: torch.Tensor,
+ unnormalized_derivatives: torch.Tensor,
+ inverse: bool = False,
+ tails: str = "linear",
+ tail_bound: float = 1.0,
+ min_bin_width: float = DEFAULT_MIN_BIN_WIDTH,
+ min_bin_height: float = DEFAULT_MIN_BIN_HEIGHT,
+ min_derivative: float = DEFAULT_MIN_DERIVATIVE,
+) -> tuple[torch.Tensor, torch.Tensor]:
+
inside_interval_mask = (inputs >= -tail_bound) & (inputs <= tail_bound)
outside_interval_mask = ~inside_interval_mask
@@ -74,7 +79,7 @@ def unconstrained_rational_quadratic_spline(
outputs[outside_interval_mask] = inputs[outside_interval_mask]
logabsdet[outside_interval_mask] = 0
else:
- raise RuntimeError("{} tails are not implemented.".format(tails))
+ raise RuntimeError(f"{tails} tails are not implemented.")
(
outputs[inside_interval_mask],
@@ -98,19 +103,20 @@ def unconstrained_rational_quadratic_spline(
def rational_quadratic_spline(
- inputs,
- unnormalized_widths,
- unnormalized_heights,
- unnormalized_derivatives,
- inverse=False,
- left=0.0,
- right=1.0,
- bottom=0.0,
- top=1.0,
- min_bin_width=DEFAULT_MIN_BIN_WIDTH,
- min_bin_height=DEFAULT_MIN_BIN_HEIGHT,
- min_derivative=DEFAULT_MIN_DERIVATIVE,
-):
+ inputs: torch.Tensor,
+ unnormalized_widths: torch.Tensor,
+ unnormalized_heights: torch.Tensor,
+ unnormalized_derivatives: torch.Tensor,
+ inverse: bool = False,
+ left: float = 0.0,
+ right: float = 1.0,
+ bottom: float = 0.0,
+ top: float = 1.0,
+ min_bin_width: float = DEFAULT_MIN_BIN_WIDTH,
+ min_bin_height: float = DEFAULT_MIN_BIN_HEIGHT,
+ min_derivative: float = DEFAULT_MIN_DERIVATIVE,
+) -> tuple[torch.Tensor, torch.Tensor]:
+
if torch.min(inputs) < left or torch.max(inputs) > right:
raise ValueError("Input to a transform is not within its domain")
diff --git a/style_bert_vits2/models/utils/__init__.py b/style_bert_vits2/models/utils/__init__.py
new file mode 100644
index 0000000..edd51cc
--- /dev/null
+++ b/style_bert_vits2/models/utils/__init__.py
@@ -0,0 +1,262 @@
+import glob
+import logging
+import os
+import re
+import subprocess
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Optional, Union
+
+import numpy as np
+import torch
+from numpy.typing import NDArray
+from scipy.io.wavfile import read
+
+from style_bert_vits2.logging import logger
+from style_bert_vits2.models.utils import checkpoints # type: ignore
+from style_bert_vits2.models.utils import safetensors # type: ignore
+
+
+if TYPE_CHECKING:
+ # tensorboard はライブラリとしてインストールされている場合は依存関係に含まれないため、型チェック時のみインポートする
+ from torch.utils.tensorboard import SummaryWriter
+
+
+__is_matplotlib_imported = False
+
+
+def summarize(
+ writer: "SummaryWriter",
+ global_step: int,
+ scalars: dict[str, float] = {},
+ histograms: dict[str, Any] = {},
+ images: dict[str, Any] = {},
+ audios: dict[str, Any] = {},
+ audio_sampling_rate: int = 22050,
+) -> None:
+ """
+ 指定されたデータを TensorBoard にまとめて追加する
+
+ Args:
+ writer (SummaryWriter): TensorBoard への書き込みを行うオブジェクト
+ global_step (int): グローバルステップ数
+ scalars (dict[str, float]): スカラー値の辞書
+ histograms (dict[str, Any]): ヒストグラムの辞書
+ images (dict[str, Any]): 画像データの辞書
+ audios (dict[str, Any]): 音声データの辞書
+ audio_sampling_rate (int): 音声データのサンプリングレート
+ """
+ for k, v in scalars.items():
+ writer.add_scalar(k, v, global_step)
+ for k, v in histograms.items():
+ writer.add_histogram(k, v, global_step)
+ for k, v in images.items():
+ writer.add_image(k, v, global_step, dataformats="HWC")
+ for k, v in audios.items():
+ writer.add_audio(k, v, global_step, audio_sampling_rate)
+
+
+def is_resuming(dir_path: Union[str, Path]) -> bool:
+ """
+ 指定されたディレクトリパスに再開可能なモデルが存在するかどうかを返す
+
+ Args:
+ dir_path: チェックするディレクトリのパス
+
+ Returns:
+ bool: 再開可能なモデルが存在するかどうか
+ """
+ # JP-ExtraバージョンではDURがなくWDがあったり変わるため、Gのみで判断する
+ g_list = glob.glob(os.path.join(dir_path, "G_*.pth"))
+ # d_list = glob.glob(os.path.join(dir_path, "D_*.pth"))
+ # dur_list = glob.glob(os.path.join(dir_path, "DUR_*.pth"))
+ return len(g_list) > 0
+
+
+def plot_spectrogram_to_numpy(spectrogram: NDArray[Any]) -> NDArray[Any]:
+ """
+ 指定されたスペクトログラムを画像データに変換する
+
+ Args:
+ spectrogram (NDArray[Any]): スペクトログラム
+
+ Returns:
+ NDArray[Any]: 画像データ
+ """
+
+ global __is_matplotlib_imported
+ if not __is_matplotlib_imported:
+ import matplotlib
+
+ matplotlib.use("Agg")
+ __is_matplotlib_imported = True
+ mpl_logger = logging.getLogger("matplotlib")
+ mpl_logger.setLevel(logging.WARNING)
+ import matplotlib.pylab as plt
+ import numpy as np
+
+ fig, ax = plt.subplots(figsize=(10, 2))
+ im = ax.imshow(spectrogram, aspect="auto", origin="lower", interpolation="none")
+ plt.colorbar(im, ax=ax)
+ plt.xlabel("Frames")
+ plt.ylabel("Channels")
+ plt.tight_layout()
+
+ fig.canvas.draw()
+ data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep="") # type: ignore
+ data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
+ plt.close()
+ return data
+
+
+def plot_alignment_to_numpy(
+ alignment: NDArray[Any], info: Optional[str] = None
+) -> NDArray[Any]:
+ """
+ 指定されたアライメントを画像データに変換する
+
+ Args:
+ alignment (NDArray[Any]): アライメント
+ info (Optional[str]): 画像に追加する情報
+
+ Returns:
+ NDArray[Any]: 画像データ
+ """
+
+ global __is_matplotlib_imported
+ if not __is_matplotlib_imported:
+ import matplotlib
+
+ matplotlib.use("Agg")
+ __is_matplotlib_imported = True
+ mpl_logger = logging.getLogger("matplotlib")
+ mpl_logger.setLevel(logging.WARNING)
+ import matplotlib.pylab as plt
+
+ fig, ax = plt.subplots(figsize=(6, 4))
+ im = ax.imshow(
+ alignment.transpose(), aspect="auto", origin="lower", interpolation="none"
+ )
+ fig.colorbar(im, ax=ax)
+ xlabel = "Decoder timestep"
+ if info is not None:
+ xlabel += "\n\n" + info
+ plt.xlabel(xlabel)
+ plt.ylabel("Encoder timestep")
+ plt.tight_layout()
+
+ fig.canvas.draw()
+ data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep="") # type: ignore
+ data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
+ plt.close()
+ return data
+
+
+def load_wav_to_torch(full_path: Union[str, Path]) -> tuple[torch.FloatTensor, int]:
+ """
+ 指定された音声ファイルを読み込み、PyTorch のテンソルに変換して返す
+
+ Args:
+ full_path (Union[str, Path]): 音声ファイルのパス
+
+ Returns:
+ tuple[torch.FloatTensor, int]: 音声データのテンソルとサンプリングレート
+ """
+
+ sampling_rate, data = read(full_path)
+ return torch.FloatTensor(data.astype(np.float32)), sampling_rate
+
+
+def load_filepaths_and_text(
+ filename: Union[str, Path], split: str = "|"
+) -> list[list[str]]:
+ """
+ 指定されたファイルからファイルパスとテキストを読み込む
+
+ Args:
+ filename (Union[str, Path]): ファイルのパス
+ split (str): ファイルの区切り文字 (デフォルト: "|")
+
+ Returns:
+ list[list[str]]: ファイルパスとテキストのリスト
+ """
+
+ with open(filename, "r", encoding="utf-8") as f:
+ filepaths_and_text = [line.strip().split(split) for line in f]
+ return filepaths_and_text
+
+
+def get_logger(
+ model_dir_path: Union[str, Path], filename: str = "train.log"
+) -> logging.Logger:
+ """
+ ロガーを取得する
+
+ Args:
+ model_dir_path (Union[str, Path]): ログを保存するディレクトリのパス
+ filename (str): ログファイルの名前 (デフォルト: "train.log")
+
+ Returns:
+ logging.Logger: ロガー
+ """
+
+ global logger
+ logger = logging.getLogger(os.path.basename(model_dir_path))
+ logger.setLevel(logging.DEBUG)
+
+ formatter = logging.Formatter("%(asctime)s\t%(name)s\t%(levelname)s\t%(message)s")
+ if not os.path.exists(model_dir_path):
+ os.makedirs(model_dir_path)
+ h = logging.FileHandler(os.path.join(model_dir_path, filename))
+ h.setLevel(logging.DEBUG)
+ h.setFormatter(formatter)
+ logger.addHandler(h)
+ return logger
+
+
+def get_steps(model_path: Union[str, Path]) -> Optional[int]:
+ """
+ モデルのパスからイテレーション回数を取得する
+
+ Args:
+ model_path (Union[str, Path]): モデルのパス
+
+ Returns:
+ Optional[int]: イテレーション回数
+ """
+
+ matches = re.findall(r"\d+", model_path) # type: ignore
+ return matches[-1] if matches else None
+
+
+def check_git_hash(model_dir_path: Union[str, Path]) -> None:
+ """
+ モデルのディレクトリに .git ディレクトリが存在する場合、ハッシュ値を比較する
+
+ Args:
+ model_dir_path (Union[str, Path]): モデルのディレクトリのパス
+ """
+
+ source_dir = os.path.dirname(os.path.realpath(__file__))
+ if not os.path.exists(os.path.join(source_dir, ".git")):
+ logger.warning(
+ "{} is not a git repository, therefore hash value comparison will be ignored.".format(
+ source_dir
+ )
+ )
+ return
+
+ cur_hash = subprocess.getoutput("git rev-parse HEAD")
+
+ path = os.path.join(model_dir_path, "githash")
+ if os.path.exists(path):
+ with open(path, "r", encoding="utf-8") as f:
+ saved_hash = f.read()
+ if saved_hash != cur_hash:
+ logger.warning(
+ "git hash values are different. {}(saved) != {}(current)".format(
+ saved_hash[:8], cur_hash[:8]
+ )
+ )
+ else:
+ with open(path, "w", encoding="utf-8") as f:
+ f.write(cur_hash)
diff --git a/style_bert_vits2/models/utils/checkpoints.py b/style_bert_vits2/models/utils/checkpoints.py
new file mode 100644
index 0000000..c601dda
--- /dev/null
+++ b/style_bert_vits2/models/utils/checkpoints.py
@@ -0,0 +1,202 @@
+import glob
+import os
+import re
+from pathlib import Path
+from typing import Any, Optional, Union
+
+import torch
+
+from style_bert_vits2.logging import logger
+
+
+def load_checkpoint(
+ checkpoint_path: Union[str, Path],
+ model: torch.nn.Module,
+ optimizer: Optional[torch.optim.Optimizer] = None,
+ skip_optimizer: bool = False,
+ for_infer: bool = False,
+) -> tuple[torch.nn.Module, Optional[torch.optim.Optimizer], float, int]:
+ """
+ 指定されたパスからチェックポイントを読み込み、モデルとオプティマイザーを更新する。
+
+ Args:
+ checkpoint_path (Union[str, Path]): チェックポイントファイルのパス
+ model (torch.nn.Module): 更新するモデル
+ optimizer (Optional[torch.optim.Optimizer]): 更新するオプティマイザー。None の場合は更新しない
+ skip_optimizer (bool): オプティマイザーの更新をスキップするかどうかのフラグ
+ for_infer (bool): 推論用に読み込むかどうかのフラグ
+
+ Returns:
+ tuple[torch.nn.Module, Optional[torch.optim.Optimizer], float, int]: 更新されたモデルとオプティマイザー、学習率、イテレーション回数
+ """
+
+ assert os.path.isfile(checkpoint_path)
+ checkpoint_dict = torch.load(checkpoint_path, map_location="cpu")
+ iteration = checkpoint_dict["iteration"]
+ learning_rate = checkpoint_dict["learning_rate"]
+ logger.info(
+ f"Loading model and optimizer at iteration {iteration} from {checkpoint_path}"
+ )
+ if (
+ optimizer is not None
+ and not skip_optimizer
+ and checkpoint_dict["optimizer"] is not None
+ ):
+ optimizer.load_state_dict(checkpoint_dict["optimizer"])
+ elif optimizer is None and not skip_optimizer:
+ # else: Disable this line if Infer and resume checkpoint,then enable the line upper
+ new_opt_dict = optimizer.state_dict() # type: ignore
+ new_opt_dict_params = new_opt_dict["param_groups"][0]["params"]
+ new_opt_dict["param_groups"] = checkpoint_dict["optimizer"]["param_groups"]
+ new_opt_dict["param_groups"][0]["params"] = new_opt_dict_params
+ optimizer.load_state_dict(new_opt_dict) # type: ignore
+
+ saved_state_dict = checkpoint_dict["model"]
+ if hasattr(model, "module"):
+ state_dict = model.module.state_dict()
+ else:
+ state_dict = model.state_dict()
+
+ new_state_dict = {}
+ for k, v in state_dict.items():
+ try:
+ # assert "emb_g" not in k
+ new_state_dict[k] = saved_state_dict[k]
+ assert saved_state_dict[k].shape == v.shape, (
+ saved_state_dict[k].shape,
+ v.shape,
+ )
+ except:
+ # For upgrading from the old version
+ if "ja_bert_proj" in k:
+ v = torch.zeros_like(v)
+ logger.warning(
+ f"Seems you are using the old version of the model, the {k} is automatically set to zero for backward compatibility"
+ )
+ elif "enc_q" in k and for_infer:
+ continue
+ else:
+ logger.error(f"{k} is not in the checkpoint {checkpoint_path}")
+
+ new_state_dict[k] = v
+
+ if hasattr(model, "module"):
+ model.module.load_state_dict(new_state_dict, strict=False)
+ else:
+ model.load_state_dict(new_state_dict, strict=False)
+
+ logger.info(f"Loaded '{checkpoint_path}' (iteration {iteration})")
+
+ return model, optimizer, learning_rate, iteration
+
+
+def save_checkpoint(
+ model: torch.nn.Module,
+ optimizer: Union[torch.optim.Optimizer, torch.optim.AdamW],
+ learning_rate: float,
+ iteration: int,
+ checkpoint_path: Union[str, Path],
+) -> None:
+ """
+ モデルとオプティマイザーの状態を指定されたパスに保存する。
+
+ Args:
+ model (torch.nn.Module): 保存するモデル
+ optimizer (Union[torch.optim.Optimizer, torch.optim.AdamW]): 保存するオプティマイザー
+ learning_rate (float): 学習率
+ iteration (int): イテレーション回数
+ checkpoint_path (Union[str, Path]): 保存先のパス
+ """
+ logger.info(
+ f"Saving model and optimizer state at iteration {iteration} to {checkpoint_path}"
+ )
+ if hasattr(model, "module"):
+ state_dict = model.module.state_dict()
+ else:
+ state_dict = model.state_dict()
+ torch.save(
+ {
+ "model": state_dict,
+ "iteration": iteration,
+ "optimizer": optimizer.state_dict(),
+ "learning_rate": learning_rate,
+ },
+ checkpoint_path,
+ )
+
+
+def clean_checkpoints(
+ model_dir_path: Union[str, Path] = "logs/44k/",
+ n_ckpts_to_keep: int = 2,
+ sort_by_time: bool = True,
+) -> None:
+ """
+ 指定されたディレクトリから古いチェックポイントを削除して空き容量を確保する
+
+ Args:
+ model_dir_path (Union[str, Path]): モデルが保存されているディレクトリのパス
+ n_ckpts_to_keep (int): 保持するチェックポイントの数（G_0.pth と D_0.pth を除く）
+ sort_by_time (bool): True の場合、時間順に削除。False の場合、名前順に削除
+ """
+
+ ckpts_files = [
+ f
+ for f in os.listdir(model_dir_path)
+ if os.path.isfile(os.path.join(model_dir_path, f))
+ ]
+
+ def name_key(_f: str) -> int:
+ return int(re.compile("._(\\d+)\\.pth").match(_f).group(1)) # type: ignore
+
+ def time_key(_f: str) -> float:
+ return os.path.getmtime(os.path.join(model_dir_path, _f))
+
+ sort_key = time_key if sort_by_time else name_key
+
+ def x_sorted(_x: str) -> list[str]:
+ return sorted(
+ [f for f in ckpts_files if f.startswith(_x) and not f.endswith("_0.pth")],
+ key=sort_key,
+ )
+
+ to_del = [
+ os.path.join(model_dir_path, fn)
+ for fn in (
+ x_sorted("G_")[:-n_ckpts_to_keep]
+ + x_sorted("D_")[:-n_ckpts_to_keep]
+ + x_sorted("WD_")[:-n_ckpts_to_keep]
+ + x_sorted("DUR_")[:-n_ckpts_to_keep]
+ )
+ ]
+
+ def del_info(fn: str) -> None:
+ return logger.info(f"Free up space by deleting ckpt {fn}")
+
+ def del_routine(x: str) -> list[Any]:
+ return [os.remove(x), del_info(x)]
+
+ [del_routine(fn) for fn in to_del]
+
+
+def get_latest_checkpoint_path(
+ model_dir_path: Union[str, Path], regex: str = "G_*.pth"
+) -> str:
+ """
+ 指定されたディレクトリから最新のチェックポイントのパスを取得する
+
+ Args:
+ model_dir_path (Union[str, Path]): モデルが保存されているディレクトリのパス
+ regex (str): チェックポイントのファイル名の正規表現
+
+ Returns:
+ str: 最新のチェックポイントのパス
+ """
+
+ f_list = glob.glob(os.path.join(str(model_dir_path), regex))
+ f_list.sort(key=lambda f: int("".join(filter(str.isdigit, f))))
+ try:
+ x = f_list[-1]
+ except IndexError:
+ raise ValueError(f"No checkpoint found in {model_dir_path} with regex {regex}")
+
+ return x
diff --git a/style_bert_vits2/models/utils/safetensors.py b/style_bert_vits2/models/utils/safetensors.py
new file mode 100644
index 0000000..52ab115
--- /dev/null
+++ b/style_bert_vits2/models/utils/safetensors.py
@@ -0,0 +1,91 @@
+from pathlib import Path
+from typing import Any, Optional, Union
+
+import torch
+from safetensors import safe_open
+from safetensors.torch import save_file
+
+from style_bert_vits2.logging import logger
+
+
+def load_safetensors(
+ checkpoint_path: Union[str, Path],
+ model: torch.nn.Module,
+ for_infer: bool = False,
+) -> tuple[torch.nn.Module, Optional[int]]:
+ """
+ 指定されたパスから safetensors モデルを読み込み、モデルとイテレーションを返す。
+
+ Args:
+ checkpoint_path (Union[str, Path]): モデルのチェックポイントファイルのパス
+ model (torch.nn.Module): 読み込む対象のモデル
+ for_infer (bool): 推論用に読み込むかどうかのフラグ
+
+ Returns:
+ tuple[torch.nn.Module, Optional[int]]: 読み込まれたモデルとイテレーション回数（存在する場合）
+ """
+
+ tensors: dict[str, Any] = {}
+ iteration: Optional[int] = None
+ with safe_open(str(checkpoint_path), framework="pt", device="cpu") as f: # type: ignore
+ for key in f.keys():
+ if key == "iteration":
+ iteration = f.get_tensor(key).item()
+ tensors[key] = f.get_tensor(key)
+ if hasattr(model, "module"):
+ result = model.module.load_state_dict(tensors, strict=False)
+ else:
+ result = model.load_state_dict(tensors, strict=False)
+ for key in result.missing_keys:
+ if key.startswith("enc_q") and for_infer:
+ continue
+ logger.warning(f"Missing key: {key}")
+ for key in result.unexpected_keys:
+ if key == "iteration":
+ continue
+ logger.warning(f"Unexpected key: {key}")
+ if iteration is None:
+ logger.info(f"Loaded '{checkpoint_path}'")
+ else:
+ logger.info(f"Loaded '{checkpoint_path}' (iteration {iteration})")
+
+ return model, iteration
+
+
+def save_safetensors(
+ model: torch.nn.Module,
+ iteration: int,
+ checkpoint_path: Union[str, Path],
+ is_half: bool = False,
+ for_infer: bool = False,
+) -> None:
+ """
+ モデルを safetensors 形式で保存する。
+
+ Args:
+ model (torch.nn.Module): 保存するモデル
+ iteration (int): イテレーション回数
+ checkpoint_path (Union[str, Path]): 保存先のパス
+ is_half (bool): モデルを半精度で保存するかどうかのフラグ
+ for_infer (bool): 推論用に保存するかどうかのフラグ
+ """
+
+ if hasattr(model, "module"):
+ state_dict = model.module.state_dict()
+ else:
+ state_dict = model.state_dict()
+ keys = []
+ for k in state_dict:
+ if "enc_q" in k and for_infer:
+ continue # noqa: E701
+ keys.append(k)
+
+ new_dict = (
+ {k: state_dict[k].half() for k in keys}
+ if is_half
+ else {k: state_dict[k] for k in keys}
+ )
+ new_dict["iteration"] = torch.LongTensor([iteration])
+ logger.info(f"Saved safetensors to {checkpoint_path}")
+
+ save_file(new_dict, checkpoint_path)
diff --git a/style_bert_vits2/nlp/__init__.py b/style_bert_vits2/nlp/__init__.py
new file mode 100644
index 0000000..b0c908e
--- /dev/null
+++ b/style_bert_vits2/nlp/__init__.py
@@ -0,0 +1,120 @@
+from typing import TYPE_CHECKING, Optional
+
+from style_bert_vits2.constants import Languages
+from style_bert_vits2.nlp.symbols import (
+ LANGUAGE_ID_MAP,
+ LANGUAGE_TONE_START_MAP,
+ SYMBOLS,
+)
+
+
+# __init__.py は配下のモジュールをインポートした時点で実行される
+# PyTorch のインポートは重いので、型チェック時以外はインポートしない
+if TYPE_CHECKING:
+ import torch
+
+
+__symbol_to_id = {s: i for i, s in enumerate(SYMBOLS)}
+
+
+def extract_bert_feature(
+ text: str,
+ word2ph: list[int],
+ language: Languages,
+ device: str,
+ assist_text: Optional[str] = None,
+ assist_text_weight: float = 0.7,
+) -> "torch.Tensor":
+ """
+ テキストから BERT の特徴量を抽出する
+
+ Args:
+ text (str): テキスト
+ word2ph (list[int]): 元のテキストの各文字に音素が何個割り当てられるかを表すリスト
+ language (Languages): テキストの言語
+ device (str): 推論に利用するデバイス
+ assist_text (Optional[str], optional): 補助テキスト (デフォルト: None)
+ assist_text_weight (float, optional): 補助テキストの重み (デフォルト: 0.7)
+
+ Returns:
+ torch.Tensor: BERT の特徴量
+ """
+
+ if language == Languages.JP:
+ from style_bert_vits2.nlp.japanese.bert_feature import extract_bert_feature
+ elif language == Languages.EN:
+ from style_bert_vits2.nlp.english.bert_feature import extract_bert_feature
+ elif language == Languages.ZH:
+ from style_bert_vits2.nlp.chinese.bert_feature import extract_bert_feature
+ else:
+ raise ValueError(f"Language {language} not supported")
+
+ return extract_bert_feature(text, word2ph, device, assist_text, assist_text_weight)
+
+
+def clean_text(
+ text: str,
+ language: Languages,
+ use_jp_extra: bool = True,
+ raise_yomi_error: bool = False,
+) -> tuple[str, list[str], list[int], list[int]]:
+ """
+ テキストをクリーニングし、音素に変換する
+
+ Args:
+ text (str): クリーニングするテキスト
+ language (Languages): テキストの言語
+ use_jp_extra (bool, optional): テキストが日本語の場合に JP-Extra モデルを利用するかどうか。Defaults to True.
+ raise_yomi_error (bool, optional): False の場合、読めない文字が消えたような扱いとして処理される。Defaults to False.
+
+ Returns:
+ tuple[str, list[str], list[int], list[int]]: クリーニングされたテキストと、音素・アクセント・元のテキストの各文字に音素が何個割り当てられるかのリスト
+ """
+
+ # Changed to import inside if condition to avoid unnecessary import
+ if language == Languages.JP:
+ from style_bert_vits2.nlp.japanese.g2p import g2p
+ from style_bert_vits2.nlp.japanese.normalizer import normalize_text
+
+ norm_text = normalize_text(text)
+ phones, tones, word2ph = g2p(norm_text, use_jp_extra, raise_yomi_error)
+ elif language == Languages.EN:
+ from style_bert_vits2.nlp.english.g2p import g2p
+ from style_bert_vits2.nlp.english.normalizer import normalize_text
+
+ norm_text = normalize_text(text)
+ phones, tones, word2ph = g2p(norm_text)
+ elif language == Languages.ZH:
+ from style_bert_vits2.nlp.chinese.g2p import g2p
+ from style_bert_vits2.nlp.chinese.normalizer import normalize_text
+
+ norm_text = normalize_text(text)
+ phones, tones, word2ph = g2p(norm_text)
+ else:
+ raise ValueError(f"Language {language} not supported")
+
+ return norm_text, phones, tones, word2ph
+
+
+def cleaned_text_to_sequence(
+ cleaned_phones: list[str], tones: list[int], language: Languages
+) -> tuple[list[int], list[int], list[int]]:
+ """
+ 音素リスト・アクセントリスト・言語を、テキスト内の対応する ID に変換する
+
+ Args:
+ cleaned_phones (list[str]): clean_text() でクリーニングされた音素のリスト
+ tones (list[int]): 各音素のアクセント
+ language (Languages): テキストの言語
+
+ Returns:
+ tuple[list[int], list[int], list[int]]: List of integers corresponding to the symbols in the text
+ """
+
+ phones = [__symbol_to_id[symbol] for symbol in cleaned_phones]
+ tone_start = LANGUAGE_TONE_START_MAP[language]
+ tones = [i + tone_start for i in tones]
+ lang_id = LANGUAGE_ID_MAP[language]
+ lang_ids = [lang_id for i in phones]
+
+ return phones, tones, lang_ids
diff --git a/style_bert_vits2/nlp/bert_models.py b/style_bert_vits2/nlp/bert_models.py
new file mode 100644
index 0000000..1166846
--- /dev/null
+++ b/style_bert_vits2/nlp/bert_models.py
@@ -0,0 +1,210 @@
+"""
+Style-Bert-VITS2 の学習・推論に必要な各言語ごとの BERT モデルをロード/取得するためのモジュール。
+
+オリジナルの Bert-VITS2 では各言語ごとの BERT モデルが初回インポート時にハードコードされたパスから「暗黙的に」ロードされているが、
+場合によっては多重にロードされて非効率なほか、BERT モデルのロード元のパスがハードコードされているためライブラリ化ができない。
+
+そこで、ライブラリの利用前に、音声合成に利用する言語の BERT モデルだけを「明示的に」ロードできるようにした。
+一度 load_model/tokenizer() で当該言語の BERT モデルがロードされていれば、ライブラリ内部のどこからでもロード済みのモデル/トークナイザーを取得できる。
+"""
+
+import gc
+from typing import Optional, Union, cast
+
+import torch
+from transformers import (
+ AutoModelForMaskedLM,
+ AutoTokenizer,
+ DebertaV2Model,
+ DebertaV2Tokenizer,
+ PreTrainedModel,
+ PreTrainedTokenizer,
+ PreTrainedTokenizerFast,
+)
+
+from style_bert_vits2.constants import DEFAULT_BERT_TOKENIZER_PATHS, Languages
+from style_bert_vits2.logging import logger
+
+
+# 各言語ごとのロード済みの BERT モデルを格納する辞書
+__loaded_models: dict[Languages, Union[PreTrainedModel, DebertaV2Model]] = {}
+
+# 各言語ごとのロード済みの BERT トークナイザーを格納する辞書
+__loaded_tokenizers: dict[
+ Languages, Union[PreTrainedTokenizer, PreTrainedTokenizerFast, DebertaV2Tokenizer]
+] = {}
+
+
+def load_model(
+ language: Languages,
+ pretrained_model_name_or_path: Optional[str] = None,
+ cache_dir: Optional[str] = None,
+ revision: str = "main",
+) -> Union[PreTrainedModel, DebertaV2Model]:
+ """
+ 指定された言語の BERT モデルをロードし、ロード済みの BERT モデルを返す。
+ 一度ロードされていれば、ロード済みの BERT モデルを即座に返す。
+ ライブラリ利用時は常に必ず pretrain_model_name_or_path (Hugging Face のリポジトリ名 or ローカルのファイルパス) を指定する必要がある。
+ ロードにはそれなりに時間がかかるため、ライブラリ利用前に明示的に pretrained_model_name_or_path を指定してロードしておくべき。
+ cache_dir と revision は pretrain_model_name_or_path がリポジトリ名の場合のみ有効。
+
+ Style-Bert-VITS2 では、BERT モデルに下記の 3 つが利用されている。
+ これ以外の BERT モデルを指定した場合は正常に動作しない可能性が高い。
+ - 日本語: ku-nlp/deberta-v2-large-japanese-char-wwm
+ - 英語: microsoft/deberta-v3-large
+ - 中国語: hfl/chinese-roberta-wwm-ext-large
+
+ Args:
+ language (Languages): ロードする学習済みモデルの対象言語
+ pretrained_model_name_or_path (Optional[str]): ロードする学習済みモデルの名前またはパス。指定しない場合はデフォルトのパスが利用される (デフォルト: None)
+ cache_dir (Optional[str]): モデルのキャッシュディレクトリ。指定しない場合はデフォルトのキャッシュディレクトリが利用される (デフォルト: None)
+ revision (str): モデルの Hugging Face 上の Git リビジョン。指定しない場合は最新の main ブランチの内容が利用される (デフォルト: None)
+
+ Returns:
+ Union[PreTrainedModel, DebertaV2Model]: ロード済みの BERT モデル
+ """
+
+ # すでにロード済みの場合はそのまま返す
+ if language in __loaded_models:
+ return __loaded_models[language]
+
+ # pretrained_model_name_or_path が指定されていない場合はデフォルトのパスを利用
+ if pretrained_model_name_or_path is None:
+ assert DEFAULT_BERT_TOKENIZER_PATHS[
+ language
+ ].exists(), f"The default {language} BERT model does not exist on the file system. Please specify the path to the pre-trained model."
+ pretrained_model_name_or_path = str(DEFAULT_BERT_TOKENIZER_PATHS[language])
+
+ # BERT モデルをロードし、辞書に格納して返す
+ ## 英語のみ DebertaV2Model でロードする必要がある
+ if language == Languages.EN:
+ model = cast(
+ DebertaV2Model,
+ DebertaV2Model.from_pretrained(
+ pretrained_model_name_or_path, cache_dir=cache_dir, revision=revision
+ ),
+ )
+ else:
+ model = AutoModelForMaskedLM.from_pretrained(
+ pretrained_model_name_or_path, cache_dir=cache_dir, revision=revision
+ )
+ __loaded_models[language] = model
+ logger.info(
+ f"Loaded the {language} BERT model from {pretrained_model_name_or_path}"
+ )
+
+ return model
+
+
+def load_tokenizer(
+ language: Languages,
+ pretrained_model_name_or_path: Optional[str] = None,
+ cache_dir: Optional[str] = None,
+ revision: str = "main",
+) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast, DebertaV2Tokenizer]:
+ """
+ 指定された言語の BERT モデルをロードし、ロード済みの BERT トークナイザーを返す。
+ 一度ロードされていれば、ロード済みの BERT トークナイザーを即座に返す。
+ ライブラリ利用時は常に必ず pretrain_model_name_or_path (Hugging Face のリポジトリ名 or ローカルのファイルパス) を指定する必要がある。
+ ロードにはそれなりに時間がかかるため、ライブラリ利用前に明示的に pretrained_model_name_or_path を指定してロードしておくべき。
+ cache_dir と revision は pretrain_model_name_or_path がリポジトリ名の場合のみ有効。
+
+ Style-Bert-VITS2 では、BERT モデルに下記の 3 つが利用されている。
+ これ以外の BERT モデルを指定した場合は正常に動作しない可能性が高い。
+ - 日本語: ku-nlp/deberta-v2-large-japanese-char-wwm
+ - 英語: microsoft/deberta-v3-large
+ - 中国語: hfl/chinese-roberta-wwm-ext-large
+
+ Args:
+ language (Languages): ロードする学習済みモデルの対象言語
+ pretrained_model_name_or_path (Optional[str]): ロードする学習済みモデルの名前またはパス。指定しない場合はデフォルトのパスが利用される (デフォルト: None)
+ cache_dir (Optional[str]): モデルのキャッシュディレクトリ。指定しない場合はデフォルトのキャッシュディレクトリが利用される (デフォルト: None)
+ revision (str): モデルの Hugging Face 上の Git リビジョン。指定しない場合は最新の main ブランチの内容が利用される (デフォルト: None)
+
+ Returns:
+ Union[PreTrainedTokenizer, PreTrainedTokenizerFast, DebertaV2Tokenizer]: ロード済みの BERT トークナイザー
+ """
+
+ # すでにロード済みの場合はそのまま返す
+ if language in __loaded_tokenizers:
+ return __loaded_tokenizers[language]
+
+ # pretrained_model_name_or_path が指定されていない場合はデフォルトのパスを利用
+ if pretrained_model_name_or_path is None:
+ assert DEFAULT_BERT_TOKENIZER_PATHS[
+ language
+ ].exists(), f"The default {language} BERT tokenizer does not exist on the file system. Please specify the path to the pre-trained model."
+ pretrained_model_name_or_path = str(DEFAULT_BERT_TOKENIZER_PATHS[language])
+
+ # BERT トークナイザーをロードし、辞書に格納して返す
+ ## 英語のみ DebertaV2Tokenizer でロードする必要がある
+ if language == Languages.EN:
+ tokenizer = DebertaV2Tokenizer.from_pretrained(
+ pretrained_model_name_or_path,
+ cache_dir=cache_dir,
+ revision=revision,
+ )
+ else:
+ tokenizer = AutoTokenizer.from_pretrained(
+ pretrained_model_name_or_path,
+ cache_dir=cache_dir,
+ revision=revision,
+ )
+ __loaded_tokenizers[language] = tokenizer
+ logger.info(
+ f"Loaded the {language} BERT tokenizer from {pretrained_model_name_or_path}"
+ )
+
+ return tokenizer
+
+
+def unload_model(language: Languages) -> None:
+ """
+ 指定された言語の BERT モデルをアンロードする。
+
+ Args:
+ language (Languages): アンロードする BERT モデルの言語
+ """
+
+ if language in __loaded_models:
+ del __loaded_models[language]
+ gc.collect()
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+ logger.info(f"Unloaded the {language} BERT model")
+
+
+def unload_tokenizer(language: Languages) -> None:
+ """
+ 指定された言語の BERT トークナイザーをアンロードする。
+
+ Args:
+ language (Languages): アンロードする BERT トークナイザーの言語
+ """
+
+ if language in __loaded_tokenizers:
+ del __loaded_tokenizers[language]
+ gc.collect()
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+ logger.info(f"Unloaded the {language} BERT tokenizer")
+
+
+def unload_all_models() -> None:
+ """
+ すべての BERT モデルをアンロードする。
+ """
+
+ for language in list(__loaded_models.keys()):
+ unload_model(language)
+ logger.info("Unloaded all BERT models")
+
+
+def unload_all_tokenizers() -> None:
+ """
+ すべての BERT トークナイザーをアンロードする。
+ """
+
+ for language in list(__loaded_tokenizers.keys()):
+ unload_tokenizer(language)
+ logger.info("Unloaded all BERT tokenizers")
diff --git a/style_bert_vits2/nlp/chinese/__init__.py b/style_bert_vits2/nlp/chinese/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/text/chinese_bert.py b/style_bert_vits2/nlp/chinese/bert_feature.py
similarity index 62%
rename from text/chinese_bert.py
rename to style_bert_vits2/nlp/chinese/bert_feature.py
index 94e1408..adc0788 100644
--- a/text/chinese_bert.py
+++ b/style_bert_vits2/nlp/chinese/bert_feature.py
@@ -1,54 +1,58 @@
-import sys
+from typing import Optional
import torch
-from transformers import AutoModelForMaskedLM, AutoTokenizer
-from config import config
-
-LOCAL_PATH = "./bert/chinese-roberta-wwm-ext-large"
-
-tokenizer = AutoTokenizer.from_pretrained(LOCAL_PATH)
-
-models = dict()
+from style_bert_vits2.constants import Languages
+from style_bert_vits2.nlp import bert_models
-def get_bert_feature(
- text,
- word2ph,
- device=config.bert_gen_config.device,
- assist_text=None,
- assist_text_weight=0.7,
-):
- if (
- sys.platform == "darwin"
- and torch.backends.mps.is_available()
- and device == "cpu"
- ):
- device = "mps"
- if not device:
- device = "cuda"
+def extract_bert_feature(
+ text: str,
+ word2ph: list[int],
+ device: str,
+ assist_text: Optional[str] = None,
+ assist_text_weight: float = 0.7,
+) -> torch.Tensor:
+ """
+ 中国語のテキストから BERT の特徴量を抽出する
+
+ Args:
+ text (str): 中国語のテキスト
+ word2ph (list[int]): 元のテキストの各文字に音素が何個割り当てられるかを表すリスト
+ device (str): 推論に利用するデバイス
+ assist_text (Optional[str], optional): 補助テキスト (デフォルト: None)
+ assist_text_weight (float, optional): 補助テキストの重み (デフォルト: 0.7)
+
+ Returns:
+ torch.Tensor: BERT の特徴量
+ """
+
if device == "cuda" and not torch.cuda.is_available():
device = "cpu"
- if device not in models.keys():
- models[device] = AutoModelForMaskedLM.from_pretrained(LOCAL_PATH).to(device)
+ model = bert_models.load_model(Languages.ZH).to(device) # type: ignore
+
+ style_res_mean = None
with torch.no_grad():
+ tokenizer = bert_models.load_tokenizer(Languages.ZH)
inputs = tokenizer(text, return_tensors="pt")
for i in inputs:
- inputs[i] = inputs[i].to(device)
- res = models[device](**inputs, output_hidden_states=True)
+ inputs[i] = inputs[i].to(device) # type: ignore
+ res = model(**inputs, output_hidden_states=True)
res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
if assist_text:
style_inputs = tokenizer(assist_text, return_tensors="pt")
for i in style_inputs:
- style_inputs[i] = style_inputs[i].to(device)
- style_res = models[device](**style_inputs, output_hidden_states=True)
+ style_inputs[i] = style_inputs[i].to(device) # type: ignore
+ style_res = model(**style_inputs, output_hidden_states=True)
style_res = torch.cat(style_res["hidden_states"][-3:-2], -1)[0].cpu()
style_res_mean = style_res.mean(0)
+
assert len(word2ph) == len(text) + 2
word2phone = word2ph
phone_level_feature = []
for i in range(len(word2phone)):
if assist_text:
+ assert style_res_mean is not None
repeat_feature = (
res[i].repeat(word2phone[i], 1) * (1 - assist_text_weight)
+ style_res_mean.repeat(word2phone[i], 1) * assist_text_weight
diff --git a/text/chinese.py b/style_bert_vits2/nlp/chinese/g2p.py
similarity index 59%
rename from text/chinese.py
rename to style_bert_vits2/nlp/chinese/g2p.py
index d9174ee..f38e09f 100644
--- a/text/chinese.py
+++ b/style_bert_vits2/nlp/chinese/g2p.py
@@ -1,75 +1,23 @@
-import os
import re
-
-import cn2an
-from pypinyin import lazy_pinyin, Style
-
-from text.symbols import punctuation
-from text.tone_sandhi import ToneSandhi
-
-current_file_path = os.path.dirname(__file__)
-pinyin_to_symbol_map = {
- line.split("\t")[0]: line.strip().split("\t")[1]
- for line in open(os.path.join(current_file_path, "opencpop-strict.txt")).readlines()
-}
+from pathlib import Path
import jieba.posseg as psg
+from pypinyin import Style, lazy_pinyin
+
+from style_bert_vits2.nlp.chinese.tone_sandhi import ToneSandhi
+from style_bert_vits2.nlp.symbols import PUNCTUATIONS
-rep_map = {
- "：": ",",
- "；": ",",
- "，": ",",
- "。": ".",
- "！": "!",
- "？": "?",
- "\n": ".",
- "·": ",",
- "、": ",",
- "...": "…",
- "$": ".",
- "“": "'",
- "”": "'",
- '"': "'",
- "‘": "'",
- "’": "'",
- "（": "'",
- "）": "'",
- "(": "'",
- ")": "'",
- "《": "'",
- "》": "'",
- "【": "'",
- "】": "'",
- "[": "'",
- "]": "'",
- "—": "-",
- "～": "-",
- "~": "-",
- "「": "'",
- "」": "'",
-}
-
-tone_modifier = ToneSandhi()
+with open(Path(__file__).parent / "opencpop-strict.txt", "r", encoding="utf-8") as f:
+ __PINYIN_TO_SYMBOL_MAP = {
+ line.split("\t")[0]: line.strip().split("\t")[1] for line in f.readlines()
+ }
-def replace_punctuation(text):
- text = text.replace("嗯", "恩").replace("呣", "母")
- pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys()))
-
- replaced_text = pattern.sub(lambda x: rep_map[x.group()], text)
-
- replaced_text = re.sub(
- r"[^\u4e00-\u9fa5" + "".join(punctuation) + r"]+", "", replaced_text
- )
-
- return replaced_text
-
-
-def g2p(text):
- pattern = r"(?<=[{0}])\s*".format("".join(punctuation))
+def g2p(text: str) -> tuple[list[str], list[int], list[int]]:
+ pattern = r"(?<=[{0}])\s*".format("".join(PUNCTUATIONS))
sentences = [i for i in re.split(pattern, text) if i.strip() != ""]
- phones, tones, word2ph = _g2p(sentences)
+ phones, tones, word2ph = __g2p(sentences)
assert sum(word2ph) == len(phones)
assert len(word2ph) == len(text) # Sometimes it will crash,you can add a try-catch.
phones = ["_"] + phones + ["_"]
@@ -78,34 +26,22 @@ def g2p(text):
return phones, tones, word2ph
-def _get_initials_finals(word):
- initials = []
- finals = []
- orig_initials = lazy_pinyin(word, neutral_tone_with_five=True, style=Style.INITIALS)
- orig_finals = lazy_pinyin(
- word, neutral_tone_with_five=True, style=Style.FINALS_TONE3
- )
- for c, v in zip(orig_initials, orig_finals):
- initials.append(c)
- finals.append(v)
- return initials, finals
-
-
-def _g2p(segments):
+def __g2p(segments: list[str]) -> tuple[list[str], list[int], list[int]]:
phones_list = []
tones_list = []
word2ph = []
+ tone_modifier = ToneSandhi()
for seg in segments:
# Replace all English words in the sentence
seg = re.sub("[a-zA-Z]+", "", seg)
seg_cut = psg.lcut(seg)
initials = []
finals = []
- seg_cut = tone_modifier.pre_merge_for_modify(seg_cut)
+ seg_cut = tone_modifier.pre_merge_for_modify(seg_cut) # type: ignore
for word, pos in seg_cut:
if pos == "eng":
continue
- sub_initials, sub_finals = _get_initials_finals(word)
+ sub_initials, sub_finals = __get_initials_finals(word)
sub_finals = tone_modifier.modified_tone(word, pos, sub_finals)
initials.append(sub_initials)
finals.append(sub_finals)
@@ -119,7 +55,7 @@ def _g2p(segments):
# NOTE: post process for pypinyin outputs
# we discriminate i, ii and iii
if c == v:
- assert c in punctuation
+ assert c in PUNCTUATIONS
phone = [c]
tone = "0"
word2ph.append(1)
@@ -159,8 +95,12 @@ def _g2p(segments):
if pinyin[0] in single_rep_map.keys():
pinyin = single_rep_map[pinyin[0]] + pinyin[1:]
- assert pinyin in pinyin_to_symbol_map.keys(), (pinyin, seg, raw_pinyin)
- phone = pinyin_to_symbol_map[pinyin].split(" ")
+ assert pinyin in __PINYIN_TO_SYMBOL_MAP.keys(), (
+ pinyin,
+ seg,
+ raw_pinyin,
+ )
+ phone = __PINYIN_TO_SYMBOL_MAP[pinyin].split(" ")
word2ph.append(len(phone))
phones_list += phone
@@ -168,32 +108,32 @@ def _g2p(segments):
return phones_list, tones_list, word2ph
-def text_normalize(text):
- numbers = re.findall(r"\d+(?:\.?\d+)?", text)
- for number in numbers:
- text = text.replace(number, cn2an.an2cn(number), 1)
- text = replace_punctuation(text)
- return text
-
-
-def get_bert_feature(text, word2ph):
- from text import chinese_bert
-
- return chinese_bert.get_bert_feature(text, word2ph)
+def __get_initials_finals(word: str) -> tuple[list[str], list[str]]:
+ initials = []
+ finals = []
+ orig_initials = lazy_pinyin(word, neutral_tone_with_five=True, style=Style.INITIALS)
+ orig_finals = lazy_pinyin(
+ word, neutral_tone_with_five=True, style=Style.FINALS_TONE3
+ )
+ for c, v in zip(orig_initials, orig_finals):
+ initials.append(c)
+ finals.append(v)
+ return initials, finals
if __name__ == "__main__":
- from text.chinese_bert import get_bert_feature
+ from style_bert_vits2.nlp.chinese.bert_feature import extract_bert_feature
+ from style_bert_vits2.nlp.chinese.normalizer import normalize_text
- text = "啊！但是《原神》是由,米哈\游自主， [研发]的一款全.新开放世界.冒险游戏"
- text = text_normalize(text)
+ text = "啊！但是《原神》是由,米哈游自主， [研发]的一款全.新开放世界.冒险游戏"
+ text = normalize_text(text)
print(text)
phones, tones, word2ph = g2p(text)
- bert = get_bert_feature(text, word2ph)
+ bert = extract_bert_feature(text, word2ph, "cuda")
print(phones, tones, word2ph, bert.shape)
-# # 示例用法
+# 示例用法
# text = "这是一个示例文本：,你好！这是一个测试...."
# print(g2p_paddle(text)) # 输出: 这是一个示例文本你好这是一个测试
diff --git a/style_bert_vits2/nlp/chinese/normalizer.py b/style_bert_vits2/nlp/chinese/normalizer.py
new file mode 100644
index 0000000..c56c636
--- /dev/null
+++ b/style_bert_vits2/nlp/chinese/normalizer.py
@@ -0,0 +1,61 @@
+import re
+
+import cn2an
+
+from style_bert_vits2.nlp.symbols import PUNCTUATIONS
+
+
+def normalize_text(text: str) -> str:
+ numbers = re.findall(r"\d+(?:\.?\d+)?", text)
+ for number in numbers:
+ text = text.replace(number, cn2an.an2cn(number), 1)
+ text = replace_punctuation(text)
+ return text
+
+
+def replace_punctuation(text: str) -> str:
+
+ REPLACE_MAP = {
+ "：": ",",
+ "；": ",",
+ "，": ",",
+ "。": ".",
+ "！": "!",
+ "？": "?",
+ "\n": ".",
+ "·": ",",
+ "、": ",",
+ "...": "…",
+ "$": ".",
+ "“": "'",
+ "”": "'",
+ '"': "'",
+ "‘": "'",
+ "’": "'",
+ "（": "'",
+ "）": "'",
+ "(": "'",
+ ")": "'",
+ "《": "'",
+ "》": "'",
+ "【": "'",
+ "】": "'",
+ "[": "'",
+ "]": "'",
+ "—": "-",
+ "～": "-",
+ "~": "-",
+ "「": "'",
+ "」": "'",
+ }
+
+ text = text.replace("嗯", "恩").replace("呣", "母")
+ pattern = re.compile("|".join(re.escape(p) for p in REPLACE_MAP.keys()))
+
+ replaced_text = pattern.sub(lambda x: REPLACE_MAP[x.group()], text)
+
+ replaced_text = re.sub(
+ r"[^\u4e00-\u9fa5" + "".join(PUNCTUATIONS) + r"]+", "", replaced_text
+ )
+
+ return replaced_text
diff --git a/text/opencpop-strict.txt b/style_bert_vits2/nlp/chinese/opencpop-strict.txt
similarity index 100%
rename from text/opencpop-strict.txt
rename to style_bert_vits2/nlp/chinese/opencpop-strict.txt
diff --git a/text/tone_sandhi.py b/style_bert_vits2/nlp/chinese/tone_sandhi.py
similarity index 95%
rename from text/tone_sandhi.py
rename to style_bert_vits2/nlp/chinese/tone_sandhi.py
index 38f3137..552cb0d 100644
--- a/text/tone_sandhi.py
+++ b/style_bert_vits2/nlp/chinese/tone_sandhi.py
@@ -11,12 +11,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from typing import List
-from typing import Tuple
import jieba
-from pypinyin import lazy_pinyin
-from pypinyin import Style
+from pypinyin import Style, lazy_pinyin
class ToneSandhi:
@@ -463,7 +460,7 @@ class ToneSandhi:
# word: "家里"
# pos: "s"
# finals: ['ia1', 'i3']
- def _neural_sandhi(self, word: str, pos: str, finals: List[str]) -> List[str]:
+ def _neural_sandhi(self, word: str, pos: str, finals: list[str]) -> list[str]:
# reduplication words for n. and v. e.g. 奶奶, 试试, 旺旺
for j, item in enumerate(word):
if (
@@ -522,7 +519,7 @@ class ToneSandhi:
finals = sum(finals_list, [])
return finals
- def _bu_sandhi(self, word: str, finals: List[str]) -> List[str]:
+ def _bu_sandhi(self, word: str, finals: list[str]) -> list[str]:
# e.g. 看不懂
if len(word) == 3 and word[1] == "不":
finals[1] = finals[1][:-1] + "5"
@@ -533,7 +530,7 @@ class ToneSandhi:
finals[i] = finals[i][:-1] + "2"
return finals
- def _yi_sandhi(self, word: str, finals: List[str]) -> List[str]:
+ def _yi_sandhi(self, word: str, finals: list[str]) -> list[str]:
# "一" in number sequences, e.g. 一零零, 二一零
if word.find("一") != -1 and all(
[item.isnumeric() for item in word if item != "一"]
@@ -558,9 +555,9 @@ class ToneSandhi:
finals[i] = finals[i][:-1] + "4"
return finals
- def _split_word(self, word: str) -> List[str]:
+ def _split_word(self, word: str) -> list[str]:
word_list = jieba.cut_for_search(word)
- word_list = sorted(word_list, key=lambda i: len(i), reverse=False)
+ word_list = sorted(word_list, key=lambda i: len(i), reverse=False) # type: ignore
first_subword = word_list[0]
first_begin_idx = word.find(first_subword)
if first_begin_idx == 0:
@@ -571,7 +568,7 @@ class ToneSandhi:
new_word_list = [second_subword, first_subword]
return new_word_list
- def _three_sandhi(self, word: str, finals: List[str]) -> List[str]:
+ def _three_sandhi(self, word: str, finals: list[str]) -> list[str]:
if len(word) == 2 and self._all_tone_three(finals):
finals[0] = finals[0][:-1] + "2"
elif len(word) == 3:
@@ -611,12 +608,12 @@ class ToneSandhi:
return finals
- def _all_tone_three(self, finals: List[str]) -> bool:
+ def _all_tone_three(self, finals: list[str]) -> bool:
return all(x[-1] == "3" for x in finals)
# merge "不" and the word behind it
# if don't merge, "不" sometimes appears alone according to jieba, which may occur sandhi error
- def _merge_bu(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
+ def _merge_bu(self, seg: list[tuple[str, str]]) -> list[tuple[str, str]]:
new_seg = []
last_word = ""
for word, pos in seg:
@@ -636,7 +633,7 @@ class ToneSandhi:
# e.g.
# input seg: [('听', 'v'), ('一', 'm'), ('听', 'v')]
# output seg: [['听一听', 'v']]
- def _merge_yi(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
+ def _merge_yi(self, seg: list[tuple[str, str]]) -> list[tuple[str, str]]:
new_seg = [] * len(seg)
# function 1
i = 0
@@ -674,8 +671,8 @@ class ToneSandhi:
# the first and the second words are all_tone_three
def _merge_continuous_three_tones(
- self, seg: List[Tuple[str, str]]
- ) -> List[Tuple[str, str]]:
+ self, seg: list[tuple[str, str]]
+ ) -> list[tuple[str, str]]:
new_seg = []
sub_finals_list = [
lazy_pinyin(word, neutral_tone_with_five=True, style=Style.FINALS_TONE3)
@@ -709,8 +706,8 @@ class ToneSandhi:
# the last char of first word and the first char of second word is tone_three
def _merge_continuous_three_tones_2(
- self, seg: List[Tuple[str, str]]
- ) -> List[Tuple[str, str]]:
+ self, seg: list[tuple[str, str]]
+ ) -> list[tuple[str, str]]:
new_seg = []
sub_finals_list = [
lazy_pinyin(word, neutral_tone_with_five=True, style=Style.FINALS_TONE3)
@@ -738,7 +735,7 @@ class ToneSandhi:
new_seg.append([word, pos])
return new_seg
- def _merge_er(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
+ def _merge_er(self, seg: list[tuple[str, str]]) -> list[tuple[str, str]]:
new_seg = []
for i, (word, pos) in enumerate(seg):
if i - 1 >= 0 and word == "儿" and seg[i - 1][0] != "#":
@@ -747,7 +744,7 @@ class ToneSandhi:
new_seg.append([word, pos])
return new_seg
- def _merge_reduplication(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
+ def _merge_reduplication(self, seg: list[tuple[str, str]]) -> list[tuple[str, str]]:
new_seg = []
for i, (word, pos) in enumerate(seg):
if new_seg and word == new_seg[-1][0]:
@@ -756,7 +753,7 @@ class ToneSandhi:
new_seg.append([word, pos])
return new_seg
- def pre_merge_for_modify(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
+ def pre_merge_for_modify(self, seg: list[tuple[str, str]]) -> list[tuple[str, str]]:
seg = self._merge_bu(seg)
try:
seg = self._merge_yi(seg)
@@ -768,7 +765,7 @@ class ToneSandhi:
seg = self._merge_er(seg)
return seg
- def modified_tone(self, word: str, pos: str, finals: List[str]) -> List[str]:
+ def modified_tone(self, word: str, pos: str, finals: list[str]) -> list[str]:
finals = self._bu_sandhi(word, finals)
finals = self._yi_sandhi(word, finals)
finals = self._neural_sandhi(word, pos, finals)
diff --git a/style_bert_vits2/nlp/english/__init__.py b/style_bert_vits2/nlp/english/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/style_bert_vits2/nlp/english/bert_feature.py b/style_bert_vits2/nlp/english/bert_feature.py
new file mode 100644
index 0000000..7692c9e
--- /dev/null
+++ b/style_bert_vits2/nlp/english/bert_feature.py
@@ -0,0 +1,66 @@
+from typing import Optional
+
+import torch
+
+from style_bert_vits2.constants import Languages
+from style_bert_vits2.nlp import bert_models
+
+
+def extract_bert_feature(
+ text: str,
+ word2ph: list[int],
+ device: str,
+ assist_text: Optional[str] = None,
+ assist_text_weight: float = 0.7,
+) -> torch.Tensor:
+ """
+ 英語のテキストから BERT の特徴量を抽出する
+
+ Args:
+ text (str): 英語のテキスト
+ word2ph (list[int]): 元のテキストの各文字に音素が何個割り当てられるかを表すリスト
+ device (str): 推論に利用するデバイス
+ assist_text (Optional[str], optional): 補助テキスト (デフォルト: None)
+ assist_text_weight (float, optional): 補助テキストの重み (デフォルト: 0.7)
+
+ Returns:
+ torch.Tensor: BERT の特徴量
+ """
+
+ if device == "cuda" and not torch.cuda.is_available():
+ device = "cpu"
+ model = bert_models.load_model(Languages.EN).to(device) # type: ignore
+
+ style_res_mean = None
+ with torch.no_grad():
+ tokenizer = bert_models.load_tokenizer(Languages.EN)
+ inputs = tokenizer(text, return_tensors="pt")
+ for i in inputs:
+ inputs[i] = inputs[i].to(device) # type: ignore
+ res = model(**inputs, output_hidden_states=True)
+ res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
+ if assist_text:
+ style_inputs = tokenizer(assist_text, return_tensors="pt")
+ for i in style_inputs:
+ style_inputs[i] = style_inputs[i].to(device) # type: ignore
+ style_res = model(**style_inputs, output_hidden_states=True)
+ style_res = torch.cat(style_res["hidden_states"][-3:-2], -1)[0].cpu()
+ style_res_mean = style_res.mean(0)
+
+ assert len(word2ph) == res.shape[0], (text, res.shape[0], len(word2ph))
+ word2phone = word2ph
+ phone_level_feature = []
+ for i in range(len(word2phone)):
+ if assist_text:
+ assert style_res_mean is not None
+ repeat_feature = (
+ res[i].repeat(word2phone[i], 1) * (1 - assist_text_weight)
+ + style_res_mean.repeat(word2phone[i], 1) * assist_text_weight
+ )
+ else:
+ repeat_feature = res[i].repeat(word2phone[i], 1)
+ phone_level_feature.append(repeat_feature)
+
+ phone_level_feature = torch.cat(phone_level_feature, dim=0)
+
+ return phone_level_feature.T
diff --git a/style_bert_vits2/nlp/english/cmudict.py b/style_bert_vits2/nlp/english/cmudict.py
new file mode 100644
index 0000000..7772e77
--- /dev/null
+++ b/style_bert_vits2/nlp/english/cmudict.py
@@ -0,0 +1,46 @@
+import pickle
+from pathlib import Path
+
+
+CMU_DICT_PATH = Path(__file__).parent / "cmudict.rep"
+CACHE_PATH = Path(__file__).parent / "cmudict_cache.pickle"
+
+
+def get_dict() -> dict[str, list[list[str]]]:
+ if CACHE_PATH.exists():
+ with open(CACHE_PATH, "rb") as pickle_file:
+ g2p_dict = pickle.load(pickle_file)
+ else:
+ g2p_dict = read_dict()
+ cache_dict(g2p_dict, CACHE_PATH)
+
+ return g2p_dict
+
+
+def read_dict() -> dict[str, list[list[str]]]:
+ g2p_dict = {}
+ start_line = 49
+ with open(CMU_DICT_PATH, "r", encoding="utf-8") as f:
+ line = f.readline()
+ line_index = 1
+ while line:
+ if line_index >= start_line:
+ line = line.strip()
+ word_split = line.split(" ")
+ word = word_split[0]
+
+ syllable_split = word_split[1].split(" - ")
+ g2p_dict[word] = []
+ for syllable in syllable_split:
+ phone_split = syllable.split(" ")
+ g2p_dict[word].append(phone_split)
+
+ line_index = line_index + 1
+ line = f.readline()
+
+ return g2p_dict
+
+
+def cache_dict(g2p_dict: dict[str, list[list[str]]], file_path: Path) -> None:
+ with open(file_path, "wb") as pickle_file:
+ pickle.dump(g2p_dict, pickle_file)
diff --git a/text/cmudict.rep b/style_bert_vits2/nlp/english/cmudict.rep
similarity index 100%
rename from text/cmudict.rep
rename to style_bert_vits2/nlp/english/cmudict.rep
diff --git a/text/cmudict_cache.pickle b/style_bert_vits2/nlp/english/cmudict_cache.pickle
similarity index 100%
rename from text/cmudict_cache.pickle
rename to style_bert_vits2/nlp/english/cmudict_cache.pickle
diff --git a/style_bert_vits2/nlp/english/g2p.py b/style_bert_vits2/nlp/english/g2p.py
new file mode 100644
index 0000000..db2a87f
--- /dev/null
+++ b/style_bert_vits2/nlp/english/g2p.py
@@ -0,0 +1,240 @@
+import re
+
+from g2p_en import G2p
+
+from style_bert_vits2.constants import Languages
+from style_bert_vits2.nlp import bert_models
+from style_bert_vits2.nlp.english.cmudict import get_dict
+from style_bert_vits2.nlp.symbols import PUNCTUATIONS, SYMBOLS
+
+
+def g2p(text: str) -> tuple[list[str], list[int], list[int]]:
+
+ ARPA = {
+ "AH0",
+ "S",
+ "AH1",
+ "EY2",
+ "AE2",
+ "EH0",
+ "OW2",
+ "UH0",
+ "NG",
+ "B",
+ "G",
+ "AY0",
+ "M",
+ "AA0",
+ "F",
+ "AO0",
+ "ER2",
+ "UH1",
+ "IY1",
+ "AH2",
+ "DH",
+ "IY0",
+ "EY1",
+ "IH0",
+ "K",
+ "N",
+ "W",
+ "IY2",
+ "T",
+ "AA1",
+ "ER1",
+ "EH2",
+ "OY0",
+ "UH2",
+ "UW1",
+ "Z",
+ "AW2",
+ "AW1",
+ "V",
+ "UW2",
+ "AA2",
+ "ER",
+ "AW0",
+ "UW0",
+ "R",
+ "OW1",
+ "EH1",
+ "ZH",
+ "AE0",
+ "IH2",
+ "IH",
+ "Y",
+ "JH",
+ "P",
+ "AY1",
+ "EY0",
+ "OY2",
+ "TH",
+ "HH",
+ "D",
+ "ER0",
+ "CH",
+ "AO1",
+ "AE1",
+ "AO2",
+ "OY1",
+ "AY2",
+ "IH1",
+ "OW0",
+ "L",
+ "SH",
+ }
+
+ _g2p = G2p()
+
+ phones = []
+ tones = []
+ phone_len = []
+ # tokens = [tokenizer.tokenize(i) for i in words]
+ words = __text_to_words(text)
+ eng_dict = get_dict()
+
+ for word in words:
+ temp_phones, temp_tones = [], []
+ if len(word) > 1:
+ if "'" in word:
+ word = ["".join(word)]
+ for w in word:
+ if w in PUNCTUATIONS:
+ temp_phones.append(w)
+ temp_tones.append(0)
+ continue
+ if w.upper() in eng_dict:
+ phns, tns = __refine_syllables(eng_dict[w.upper()])
+ temp_phones += [__post_replace_ph(i) for i in phns]
+ temp_tones += tns
+ # w2ph.append(len(phns))
+ else:
+ phone_list = list(filter(lambda p: p != " ", _g2p(w))) # type: ignore
+ phns = []
+ tns = []
+ for ph in phone_list:
+ if ph in ARPA:
+ ph, tn = __refine_ph(ph)
+ phns.append(ph)
+ tns.append(tn)
+ else:
+ phns.append(ph)
+ tns.append(0)
+ temp_phones += [__post_replace_ph(i) for i in phns]
+ temp_tones += tns
+ phones += temp_phones
+ tones += temp_tones
+ phone_len.append(len(temp_phones))
+ # phones = [post_replace_ph(i) for i in phones]
+
+ word2ph = []
+ for token, pl in zip(words, phone_len):
+ word_len = len(token)
+
+ aaa = __distribute_phone(pl, word_len)
+ word2ph += aaa
+
+ phones = ["_"] + phones + ["_"]
+ tones = [0] + tones + [0]
+ word2ph = [1] + word2ph + [1]
+ assert len(phones) == len(tones), text
+ assert len(phones) == sum(word2ph), text
+
+ return phones, tones, word2ph
+
+
+def __post_replace_ph(ph: str) -> str:
+ REPLACE_MAP = {
+ "：": ",",
+ "；": ",",
+ "，": ",",
+ "。": ".",
+ "！": "!",
+ "？": "?",
+ "\n": ".",
+ "·": ",",
+ "、": ",",
+ "…": "...",
+ "···": "...",
+ "・・・": "...",
+ "v": "V",
+ }
+ if ph in REPLACE_MAP.keys():
+ ph = REPLACE_MAP[ph]
+ if ph in SYMBOLS:
+ return ph
+ if ph not in SYMBOLS:
+ ph = "UNK"
+ return ph
+
+
+def __refine_ph(phn: str) -> tuple[str, int]:
+ tone = 0
+ if re.search(r"\d$", phn):
+ tone = int(phn[-1]) + 1
+ phn = phn[:-1]
+ else:
+ tone = 3
+ return phn.lower(), tone
+
+
+def __refine_syllables(syllables: list[list[str]]) -> tuple[list[str], list[int]]:
+ tones = []
+ phonemes = []
+ for phn_list in syllables:
+ for i in range(len(phn_list)):
+ phn = phn_list[i]
+ phn, tone = __refine_ph(phn)
+ phonemes.append(phn)
+ tones.append(tone)
+ return phonemes, tones
+
+
+def __distribute_phone(n_phone: int, n_word: int) -> list[int]:
+ phones_per_word = [0] * n_word
+ for task in range(n_phone):
+ min_tasks = min(phones_per_word)
+ min_index = phones_per_word.index(min_tasks)
+ phones_per_word[min_index] += 1
+ return phones_per_word
+
+
+def __text_to_words(text: str) -> list[list[str]]:
+ tokenizer = bert_models.load_tokenizer(Languages.EN)
+ tokens = tokenizer.tokenize(text)
+ words = []
+ for idx, t in enumerate(tokens):
+ if t.startswith("▁"):
+ words.append([t[1:]])
+ else:
+ if t in PUNCTUATIONS:
+ if idx == len(tokens) - 1:
+ words.append([f"{t}"])
+ else:
+ if (
+ not tokens[idx + 1].startswith("▁")
+ and tokens[idx + 1] not in PUNCTUATIONS
+ ):
+ if idx == 0:
+ words.append([])
+ words[-1].append(f"{t}")
+ else:
+ words.append([f"{t}"])
+ else:
+ if idx == 0:
+ words.append([])
+ words[-1].append(f"{t}")
+ return words
+
+
+if __name__ == "__main__":
+ # print(get_dict())
+ # print(eng_word_to_phoneme("hello"))
+ print(g2p("In this paper, we propose 1 DSPGAN, a GAN-based universal vocoder."))
+ # all_phones = set()
+ # eng_dict = get_dict()
+ # for k, syllables in eng_dict.items():
+ # for group in syllables:
+ # for ph in group:
+ # all_phones.add(ph)
+ # print(all_phones)
diff --git a/style_bert_vits2/nlp/english/normalizer.py b/style_bert_vits2/nlp/english/normalizer.py
new file mode 100644
index 0000000..f6ddc90
--- /dev/null
+++ b/style_bert_vits2/nlp/english/normalizer.py
@@ -0,0 +1,132 @@
+import re
+
+import inflect
+
+
+__INFLECT = inflect.engine()
+__COMMA_NUMBER_PATTERN = re.compile(r"([0-9][0-9\,]+[0-9])")
+__DECIMAL_NUMBER_PATTERN = re.compile(r"([0-9]+\.[0-9]+)")
+__POUNDS_PATTERN = re.compile(r"£([0-9\,]*[0-9]+)")
+__DOLLARS_PATTERN = re.compile(r"\$([0-9\.\,]*[0-9]+)")
+__ORDINAL_PATTERN = re.compile(r"[0-9]+(st|nd|rd|th)")
+__NUMBER_PATTERN = re.compile(r"[0-9]+")
+
+
+def normalize_text(text: str) -> str:
+ text = __normalize_numbers(text)
+ text = replace_punctuation(text)
+ text = re.sub(r"([,;.\?\!])([\w])", r"\1 \2", text)
+ return text
+
+
+def replace_punctuation(text: str) -> str:
+ REPLACE_MAP = {
+ "：": ",",
+ "；": ",",
+ "，": ",",
+ "。": ".",
+ "！": "!",
+ "？": "?",
+ "\n": ".",
+ "．": ".",
+ "…": "...",
+ "···": "...",
+ "・・・": "...",
+ "·": ",",
+ "・": ",",
+ "、": ",",
+ "$": ".",
+ "“": "'",
+ "”": "'",
+ '"': "'",
+ "‘": "'",
+ "’": "'",
+ "（": "'",
+ "）": "'",
+ "(": "'",
+ ")": "'",
+ "《": "'",
+ "》": "'",
+ "【": "'",
+ "】": "'",
+ "[": "'",
+ "]": "'",
+ "—": "-",
+ "−": "-",
+ "～": "-",
+ "~": "-",
+ "「": "'",
+ "」": "'",
+ }
+ pattern = re.compile("|".join(re.escape(p) for p in REPLACE_MAP.keys()))
+ replaced_text = pattern.sub(lambda x: REPLACE_MAP[x.group()], text)
+ # replaced_text = re.sub(
+ # r"[^\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\u3400-\u4DBF\u3005"
+ # + "".join(punctuation)
+ # + r"]+",
+ # "",
+ # replaced_text,
+ # )
+ return replaced_text
+
+
+def __normalize_numbers(text: str) -> str:
+ text = re.sub(__COMMA_NUMBER_PATTERN, __remove_commas, text)
+ text = re.sub(__POUNDS_PATTERN, r"\1 pounds", text)
+ text = re.sub(__DOLLARS_PATTERN, __expand_dollars, text)
+ text = re.sub(__DECIMAL_NUMBER_PATTERN, __expand_decimal_point, text)
+ text = re.sub(__ORDINAL_PATTERN, __expand_ordinal, text)
+ text = re.sub(__NUMBER_PATTERN, __expand_number, text)
+ return text
+
+
+def __expand_dollars(m: re.Match[str]) -> str:
+ match = m.group(1)
+ parts = match.split(".")
+ if len(parts) > 2:
+ return match + " dollars" # Unexpected format
+ dollars = int(parts[0]) if parts[0] else 0
+ cents = int(parts[1]) if len(parts) > 1 and parts[1] else 0
+ if dollars and cents:
+ dollar_unit = "dollar" if dollars == 1 else "dollars"
+ cent_unit = "cent" if cents == 1 else "cents"
+ return "%s %s, %s %s" % (dollars, dollar_unit, cents, cent_unit)
+ elif dollars:
+ dollar_unit = "dollar" if dollars == 1 else "dollars"
+ return "%s %s" % (dollars, dollar_unit)
+ elif cents:
+ cent_unit = "cent" if cents == 1 else "cents"
+ return "%s %s" % (cents, cent_unit)
+ else:
+ return "zero dollars"
+
+
+def __remove_commas(m: re.Match[str]) -> str:
+ return m.group(1).replace(",", "")
+
+
+def __expand_ordinal(m: re.Match[str]) -> str:
+ return __INFLECT.number_to_words(m.group(0)) # type: ignore
+
+
+def __expand_number(m: re.Match[str]) -> str:
+ num = int(m.group(0))
+ if num > 1000 and num < 3000:
+ if num == 2000:
+ return "two thousand"
+ elif num > 2000 and num < 2010:
+ return "two thousand " + __INFLECT.number_to_words(num % 100) # type: ignore
+ elif num % 100 == 0:
+ return __INFLECT.number_to_words(num // 100) + " hundred" # type: ignore
+ else:
+ return __INFLECT.number_to_words(
+ num, andword="", zero="oh", group=2 # type: ignore
+ ).replace(
+ ", ", " "
+ ) # type: ignore
+ else:
+ return __INFLECT.number_to_words(num, andword="") # type: ignore
+
+
+def __expand_decimal_point(m: re.Match[str]) -> str:
+ return m.group(1).replace(".", " point ")
diff --git a/style_bert_vits2/nlp/japanese/__init__.py b/style_bert_vits2/nlp/japanese/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/style_bert_vits2/nlp/japanese/bert_feature.py b/style_bert_vits2/nlp/japanese/bert_feature.py
new file mode 100644
index 0000000..d6e214b
--- /dev/null
+++ b/style_bert_vits2/nlp/japanese/bert_feature.py
@@ -0,0 +1,73 @@
+from typing import Optional
+
+import torch
+
+from style_bert_vits2.constants import Languages
+from style_bert_vits2.nlp import bert_models
+from style_bert_vits2.nlp.japanese.g2p import text_to_sep_kata
+
+
+def extract_bert_feature(
+ text: str,
+ word2ph: list[int],
+ device: str,
+ assist_text: Optional[str] = None,
+ assist_text_weight: float = 0.7,
+) -> torch.Tensor:
+ """
+ 日本語のテキストから BERT の特徴量を抽出する
+
+ Args:
+ text (str): 日本語のテキスト
+ word2ph (list[int]): 元のテキストの各文字に音素が何個割り当てられるかを表すリスト
+ device (str): 推論に利用するデバイス
+ assist_text (Optional[str], optional): 補助テキスト (デフォルト: None)
+ assist_text_weight (float, optional): 補助テキストの重み (デフォルト: 0.7)
+
+ Returns:
+ torch.Tensor: BERT の特徴量
+ """
+
+ # 各単語が何文字かを作る `word2ph` を使う必要があるので、読めない文字は必ず無視する
+ # でないと `word2ph` の結果とテキストの文字数結果が整合性が取れない
+ text = "".join(text_to_sep_kata(text, raise_yomi_error=False)[0])
+ if assist_text:
+ assist_text = "".join(text_to_sep_kata(assist_text, raise_yomi_error=False)[0])
+
+ if device == "cuda" and not torch.cuda.is_available():
+ device = "cpu"
+ model = bert_models.load_model(Languages.JP).to(device) # type: ignore
+
+ style_res_mean = None
+ with torch.no_grad():
+ tokenizer = bert_models.load_tokenizer(Languages.JP)
+ inputs = tokenizer(text, return_tensors="pt")
+ for i in inputs:
+ inputs[i] = inputs[i].to(device) # type: ignore
+ res = model(**inputs, output_hidden_states=True)
+ res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
+ if assist_text:
+ style_inputs = tokenizer(assist_text, return_tensors="pt")
+ for i in style_inputs:
+ style_inputs[i] = style_inputs[i].to(device) # type: ignore
+ style_res = model(**style_inputs, output_hidden_states=True)
+ style_res = torch.cat(style_res["hidden_states"][-3:-2], -1)[0].cpu()
+ style_res_mean = style_res.mean(0)
+
+ assert len(word2ph) == len(text) + 2, text
+ word2phone = word2ph
+ phone_level_feature = []
+ for i in range(len(word2phone)):
+ if assist_text:
+ assert style_res_mean is not None
+ repeat_feature = (
+ res[i].repeat(word2phone[i], 1) * (1 - assist_text_weight)
+ + style_res_mean.repeat(word2phone[i], 1) * assist_text_weight
+ )
+ else:
+ repeat_feature = res[i].repeat(word2phone[i], 1)
+ phone_level_feature.append(repeat_feature)
+
+ phone_level_feature = torch.cat(phone_level_feature, dim=0)
+
+ return phone_level_feature.T
diff --git a/style_bert_vits2/nlp/japanese/g2p.py b/style_bert_vits2/nlp/japanese/g2p.py
new file mode 100644
index 0000000..1f6b450
--- /dev/null
+++ b/style_bert_vits2/nlp/japanese/g2p.py
@@ -0,0 +1,489 @@
+import re
+
+from style_bert_vits2.constants import Languages
+from style_bert_vits2.logging import logger
+from style_bert_vits2.nlp import bert_models
+from style_bert_vits2.nlp.japanese import pyopenjtalk_worker as pyopenjtalk
+from style_bert_vits2.nlp.japanese.mora_list import MORA_KATA_TO_MORA_PHONEMES
+from style_bert_vits2.nlp.japanese.normalizer import replace_punctuation
+from style_bert_vits2.nlp.symbols import PUNCTUATIONS
+
+
+def g2p(
+ norm_text: str, use_jp_extra: bool = True, raise_yomi_error: bool = False
+) -> tuple[list[str], list[int], list[int]]:
+ """
+ 他で使われるメインの関数。`normalize_text()` で正規化された `norm_text` を受け取り、
+ - phones: 音素のリスト（ただし `!` や `,` や `.` など punctuation が含まれうる）
+ - tones: アクセントのリスト、0（低）と1（高）からなり、phones と同じ長さ
+ - word2ph: 元のテキストの各文字に音素が何個割り当てられるかを表すリスト
+ のタプルを返す。
+ ただし `phones` と `tones` の最初と終わりに `_` が入り、応じて `word2ph` の最初と最後に 1 が追加される。
+
+ Args:
+ norm_text (str): 正規化されたテキスト
+ use_jp_extra (bool, optional): False の場合、「ん」の音素を「N」ではなく「n」とする。Defaults to True.
+ raise_yomi_error (bool, optional): False の場合、読めない文字が消えたような扱いとして処理される。Defaults to False.
+
+ Returns:
+ tuple[list[str], list[int], list[int]]: 音素のリスト、アクセントのリスト、word2ph のリスト
+ """
+
+ # pyopenjtalk のフルコンテキストラベルを使ってアクセントを取り出すと、punctuation の位置が消えてしまい情報が失われてしまう：
+ # 「こんにちは、世界。」と「こんにちは！世界。」と「こんにちは！！！？？？世界……。」は全て同じになる。
+ # よって、まず punctuation 無しの音素とアクセントのリストを作り、
+ # それとは別に pyopenjtalk.run_frontend() で得られる音素リスト（こちらは punctuation が保持される）を使い、
+ # アクセント割当をしなおすことによって punctuation を含めた音素とアクセントのリストを作る。
+
+ # punctuation がすべて消えた、音素とアクセントのタプルのリスト（「ん」は「N」）
+ phone_tone_list_wo_punct = __g2phone_tone_wo_punct(norm_text)
+
+ # sep_text: 単語単位の単語のリスト、読めない文字があったら raise_yomi_error なら例外、そうでないなら読めない文字が消えて返ってくる
+ # sep_kata: 単語単位の単語のカタカナ読みのリスト
+ sep_text, sep_kata = text_to_sep_kata(norm_text, raise_yomi_error=raise_yomi_error)
+
+ # sep_phonemes: 各単語ごとの音素のリストのリスト
+ sep_phonemes = __handle_long([__kata_to_phoneme_list(i) for i in sep_kata])
+
+ # phone_w_punct: sep_phonemes を結合した、punctuation を元のまま保持した音素列
+ phone_w_punct: list[str] = []
+ for i in sep_phonemes:
+ phone_w_punct += i
+
+ # punctuation 無しのアクセント情報を使って、punctuation を含めたアクセント情報を作る
+ phone_tone_list = __align_tones(phone_w_punct, phone_tone_list_wo_punct)
+ # logger.debug(f"phone_tone_list:\n{phone_tone_list}")
+
+ # word2ph は厳密な解答は不可能なので（「今日」「眼鏡」等の熟字訓が存在）、
+ # Bert-VITS2 では、単語単位の分割を使って、単語の文字ごとにだいたい均等に音素を分配する
+
+ # sep_text から、各単語を1文字1文字分割して、文字のリスト（のリスト）を作る
+ sep_tokenized: list[list[str]] = []
+ for i in sep_text:
+ if i not in PUNCTUATIONS:
+ sep_tokenized.append(
+ bert_models.load_tokenizer(Languages.JP).tokenize(i)
+ ) # ここでおそらく`i`が文字単位に分割される
+ else:
+ sep_tokenized.append([i])
+
+ # 各単語について、音素の数と文字の数を比較して、均等っぽく分配する
+ word2ph = []
+ for token, phoneme in zip(sep_tokenized, sep_phonemes):
+ phone_len = len(phoneme)
+ word_len = len(token)
+ word2ph += __distribute_phone(phone_len, word_len)
+
+ # 最初と最後に `_` 記号を追加、アクセントは 0（低）、word2ph もそれに合わせて追加
+ phone_tone_list = [("_", 0)] + phone_tone_list + [("_", 0)]
+ word2ph = [1] + word2ph + [1]
+
+ phones = [phone for phone, _ in phone_tone_list]
+ tones = [tone for _, tone in phone_tone_list]
+
+ assert len(phones) == sum(word2ph), f"{len(phones)} != {sum(word2ph)}"
+
+ # use_jp_extra でない場合は「N」を「n」に変換
+ if not use_jp_extra:
+ phones = [phone if phone != "N" else "n" for phone in phones]
+
+ return phones, tones, word2ph
+
+
+def text_to_sep_kata(
+ norm_text: str, raise_yomi_error: bool = False
+) -> tuple[list[str], list[str]]:
+ """
+ `normalize_text` で正規化済みの `norm_text` を受け取り、それを単語分割し、
+ 分割された単語リストとその読み（カタカナ or 記号1文字）のリストのタプルを返す。
+ 単語分割結果は、`g2p()` の `word2ph` で1文字あたりに割り振る音素記号の数を決めるために使う。
+ 例:
+ `私はそう思う!って感じ?` →
+ ["私", "は", "そう", "思う", "!", "って", "感じ", "?"], ["ワタシ", "ワ", "ソー", "オモウ", "!", "ッテ", "カンジ", "?"]
+
+ Args:
+ norm_text (str): 正規化されたテキスト
+ raise_yomi_error (bool, optional): False の場合、読めない文字が消えたような扱いとして処理される。Defaults to False.
+
+ Returns:
+ tuple[list[str], list[str]]: 分割された単語リストと、その読み（カタカナ or 記号1文字）のリスト
+ """
+
+ # parsed: OpenJTalkの解析結果
+ parsed = pyopenjtalk.run_frontend(norm_text)
+ sep_text: list[str] = []
+ sep_kata: list[str] = []
+
+ for parts in parsed:
+ # word: 実際の単語の文字列
+ # yomi: その読み、但し無声化サインの`’`は除去
+ word, yomi = replace_punctuation(parts["string"]), parts["pron"].replace(
+ "’", ""
+ )
+ """
+ ここで `yomi` の取りうる値は以下の通りのはず。
+ - `word` が通常単語 → 通常の読み（カタカナ）
+ （カタカナからなり、長音記号も含みうる、`アー` 等）
+ - `word` が `ー` から始まる → `ーラー` や `ーーー` など
+ - `word` が句読点や空白等 → `、`
+ - `word` が punctuation の繰り返し → 全角にしたもの
+ 基本的に punctuation は1文字ずつ分かれるが、何故かある程度連続すると1つにまとまる。
+ 他にも `word` が読めないキリル文字アラビア文字等が来ると `、` になるが、正規化でこの場合は起きないはず。
+ また元のコードでは `yomi` が空白の場合の処理があったが、これは起きないはず。
+ 処理すべきは `yomi` が `、` の場合のみのはず。
+ """
+ assert yomi != "", f"Empty yomi: {word}"
+ if yomi == "、":
+ # word は正規化されているので、`.`, `,`, `!`, `'`, `-`, `--` のいずれか
+ if not set(word).issubset(set(PUNCTUATIONS)): # 記号繰り返しか判定
+ # ここは pyopenjtalk が読めない文字等のときに起こる
+ if raise_yomi_error:
+ raise YomiError(f"Cannot read: {word} in:\n{norm_text}")
+ logger.warning(f"Ignoring unknown: {word} in:\n{norm_text}")
+ continue
+ # yomi は元の記号のままに変更
+ yomi = word
+ elif yomi == "？":
+ assert word == "?", f"yomi `？` comes from: {word}"
+ yomi = "?"
+ sep_text.append(word)
+ sep_kata.append(yomi)
+
+ return sep_text, sep_kata
+
+
+def __g2phone_tone_wo_punct(text: str) -> list[tuple[str, int]]:
+ """
+ テキストに対して、音素とアクセント（0か1）のペアのリストを返す。
+ ただし「!」「.」「?」等の非音素記号 (punctuation) は全て消える（ポーズ記号も残さない）。
+ 非音素記号を含める処理は `align_tones()` で行われる。
+ また「っ」は「q」に、「ん」は「N」に変換される。
+ 例: "こんにちは、世界ー。。元気？！" →
+ [('k', 0), ('o', 0), ('N', 1), ('n', 1), ('i', 1), ('ch', 1), ('i', 1), ('w', 1), ('a', 1), ('s', 1), ('e', 1), ('k', 0), ('a', 0), ('i', 0), ('i', 0), ('g', 1), ('e', 1), ('N', 0), ('k', 0), ('i', 0)]
+
+ Args:
+ text (str): テキスト
+
+ Returns:
+ list[tuple[str, int]]: 音素とアクセントのペアのリスト
+ """
+
+ prosodies = __pyopenjtalk_g2p_prosody(text, drop_unvoiced_vowels=True)
+ # logger.debug(f"prosodies: {prosodies}")
+ result: list[tuple[str, int]] = []
+ current_phrase: list[tuple[str, int]] = []
+ current_tone = 0
+
+ for i, letter in enumerate(prosodies):
+ # 特殊記号の処理
+
+ # 文頭記号、無視する
+ if letter == "^":
+ assert i == 0, "Unexpected ^"
+ # アクセント句の終わりに来る記号
+ elif letter in ("$", "?", "_", "#"):
+ # 保持しているフレーズを、アクセント数値を 0-1 に修正し結果に追加
+ result.extend(__fix_phone_tone(current_phrase))
+ # 末尾に来る終了記号、無視（文中の疑問文は `_` になる）
+ if letter in ("$", "?"):
+ assert i == len(prosodies) - 1, f"Unexpected {letter}"
+ # あとは "_"（ポーズ）と "#"（アクセント句の境界）のみ
+ # これらは残さず、次のアクセント句に備える。
+ current_phrase = []
+ # 0 を基準点にしてそこから上昇・下降する（負の場合は上の `fix_phone_tone` で直る）
+ current_tone = 0
+ # アクセント上昇記号
+ elif letter == "[":
+ current_tone = current_tone + 1
+ # アクセント下降記号
+ elif letter == "]":
+ current_tone = current_tone - 1
+ # それ以外は通常の音素
+ else:
+ if letter == "cl": # 「っ」の処理
+ letter = "q"
+ # elif letter == "N": # 「ん」の処理
+ # letter = "n"
+ current_phrase.append((letter, current_tone))
+
+ return result
+
+
+def __pyopenjtalk_g2p_prosody(
+ text: str, drop_unvoiced_vowels: bool = True
+) -> list[str]:
+ """
+ ESPnet の実装から引用、変更点無し。「ん」は「N」なことに注意。
+ ref: https://github.com/espnet/espnet/blob/master/espnet2/text/phoneme_tokenizer.py
+ ------------------------------------------------------------------------------------------
+
+ Extract phoneme + prosoody symbol sequence from input full-context labels.
+
+ The algorithm is based on `Prosodic features control by symbols as input of
+ sequence-to-sequence acoustic modeling for neural TTS`_ with some r9y9's tweaks.
+
+ Args:
+ text (str): Input text.
+ drop_unvoiced_vowels (bool): whether to drop unvoiced vowels.
+
+ Returns:
+ List[str]: List of phoneme + prosody symbols.
+
+ Examples:
+ >>> from espnet2.text.phoneme_tokenizer import pyopenjtalk_g2p_prosody
+ >>> pyopenjtalk_g2p_prosody("こんにちは。")
+ ['^', 'k', 'o', '[', 'N', 'n', 'i', 'ch', 'i', 'w', 'a', '$']
+
+ .. _`Prosodic features control by symbols as input of sequence-to-sequence acoustic
+ modeling for neural TTS`: https://doi.org/10.1587/transinf.2020EDP7104
+ """
+
+ def _numeric_feature_by_regex(regex: str, s: str) -> int:
+ match = re.search(regex, s)
+ if match is None:
+ return -50
+ return int(match.group(1))
+
+ labels = pyopenjtalk.make_label(pyopenjtalk.run_frontend(text))
+ N = len(labels)
+
+ phones = []
+ for n in range(N):
+ lab_curr = labels[n]
+
+ # current phoneme
+ p3 = re.search(r"\-(.*?)\+", lab_curr).group(1) # type: ignore
+ # deal unvoiced vowels as normal vowels
+ if drop_unvoiced_vowels and p3 in "AEIOU":
+ p3 = p3.lower()
+
+ # deal with sil at the beginning and the end of text
+ if p3 == "sil":
+ assert n == 0 or n == N - 1
+ if n == 0:
+ phones.append("^")
+ elif n == N - 1:
+ # check question form or not
+ e3 = _numeric_feature_by_regex(r"!(\d+)_", lab_curr)
+ if e3 == 0:
+ phones.append("$")
+ elif e3 == 1:
+ phones.append("?")
+ continue
+ elif p3 == "pau":
+ phones.append("_")
+ continue
+ else:
+ phones.append(p3)
+
+ # accent type and position info (forward or backward)
+ a1 = _numeric_feature_by_regex(r"/A:([0-9\-]+)\+", lab_curr)
+ a2 = _numeric_feature_by_regex(r"\+(\d+)\+", lab_curr)
+ a3 = _numeric_feature_by_regex(r"\+(\d+)/", lab_curr)
+
+ # number of mora in accent phrase
+ f1 = _numeric_feature_by_regex(r"/F:(\d+)_", lab_curr)
+
+ a2_next = _numeric_feature_by_regex(r"\+(\d+)\+", labels[n + 1])
+ # accent phrase border
+ if a3 == 1 and a2_next == 1 and p3 in "aeiouAEIOUNcl":
+ phones.append("#")
+ # pitch falling
+ elif a1 == 0 and a2_next == a2 + 1 and a2 != f1:
+ phones.append("]")
+ # pitch rising
+ elif a2 == 1 and a2_next == 2:
+ phones.append("[")
+
+ return phones
+
+
+def __fix_phone_tone(phone_tone_list: list[tuple[str, int]]) -> list[tuple[str, int]]:
+ """
+ `phone_tone_list` の tone（アクセントの値）を 0 か 1 の範囲に修正する。
+ 例: [(a, 0), (i, -1), (u, -1)] → [(a, 1), (i, 0), (u, 0)]
+
+ Args:
+ phone_tone_list (list[tuple[str, int]]): 音素とアクセントのペアのリスト
+
+ Returns:
+ list[tuple[str, int]]: 修正された音素とアクセントのペアのリスト
+ """
+
+ tone_values = set(tone for _, tone in phone_tone_list)
+ if len(tone_values) == 1:
+ assert tone_values == {0}, tone_values
+ return phone_tone_list
+ elif len(tone_values) == 2:
+ if tone_values == {0, 1}:
+ return phone_tone_list
+ elif tone_values == {-1, 0}:
+ return [
+ (letter, 0 if tone == -1 else 1) for letter, tone in phone_tone_list
+ ]
+ else:
+ raise ValueError(f"Unexpected tone values: {tone_values}")
+ else:
+ raise ValueError(f"Unexpected tone values: {tone_values}")
+
+
+def __handle_long(sep_phonemes: list[list[str]]) -> list[list[str]]:
+ """
+ フレーズごとに分かれた音素（長音記号がそのまま）のリストのリスト `sep_phonemes` を受け取り、
+ その長音記号を処理して、音素のリストのリストを返す。
+ 基本的には直前の音素を伸ばすが、直前の音素が母音でない場合もしくは冒頭の場合は、
+ おそらく長音記号とダッシュを勘違いしていると思われるので、ダッシュに対応する音素 `-` に変換する。
+
+ Args:
+ sep_phonemes (list[list[str]]): フレーズごとに分かれた音素のリストのリスト
+
+ Returns:
+ list[list[str]]: 長音記号を処理した音素のリストのリスト
+ """
+
+ # 母音の集合 (便宜上「ん」を含める)
+ VOWELS = {"a", "i", "u", "e", "o", "N"}
+
+ for i in range(len(sep_phonemes)):
+ if len(sep_phonemes[i]) == 0:
+ # 空白文字等でリストが空の場合
+ continue
+ if sep_phonemes[i][0] == "ー":
+ if i != 0:
+ prev_phoneme = sep_phonemes[i - 1][-1]
+ if prev_phoneme in VOWELS:
+ # 母音と「ん」のあとの伸ばし棒なので、その母音に変換
+ sep_phonemes[i][0] = sep_phonemes[i - 1][-1]
+ else:
+ # 「。ーー」等おそらく予期しない長音記号
+ # ダッシュの勘違いだと思われる
+ sep_phonemes[i][0] = "-"
+ else:
+ # 冒頭に長音記号が来ていおり、これはダッシュの勘違いと思われる
+ sep_phonemes[i][0] = "-"
+ if "ー" in sep_phonemes[i]:
+ for j in range(len(sep_phonemes[i])):
+ if sep_phonemes[i][j] == "ー":
+ sep_phonemes[i][j] = sep_phonemes[i][j - 1][-1]
+
+ return sep_phonemes
+
+
+def __kata_to_phoneme_list(text: str) -> list[str]:
+ """
+ 原則カタカナの `text` を受け取り、それをそのままいじらずに音素記号のリストに変換。
+ 注意点：
+ - punctuation かその繰り返しが来た場合、punctuation たちをそのままリストにして返す。
+ - 冒頭に続く「ー」はそのまま「ー」のままにする（`handle_long()` で処理される）
+ - 文中の「ー」は前の音素記号の最後の音素記号に変換される。
+ 例：
+ `ーーソーナノカーー` → ["ー", "ー", "s", "o", "o", "n", "a", "n", "o", "k", "a", "a", "a"]
+ `?` → ["?"]
+ `!?!?!?!?!` → ["!", "?", "!", "?", "!", "?", "!", "?", "!"]
+
+ Args:
+ text (str): カタカナのテキスト
+
+ Returns:
+ list[str]: 音素記号のリスト
+ """
+
+ if set(text).issubset(set(PUNCTUATIONS)):
+ return list(text)
+ # `text` がカタカナ（`ー`含む）のみからなるかどうかをチェック
+ if re.fullmatch(r"[\u30A0-\u30FF]+", text) is None:
+ raise ValueError(f"Input must be katakana only: {text}")
+ sorted_keys = sorted(MORA_KATA_TO_MORA_PHONEMES.keys(), key=len, reverse=True)
+ pattern = "|".join(map(re.escape, sorted_keys))
+
+ def mora2phonemes(mora: str) -> str:
+ cosonant, vowel = MORA_KATA_TO_MORA_PHONEMES[mora]
+ if cosonant is None:
+ return f" {vowel}"
+ return f" {cosonant} {vowel}"
+
+ spaced_phonemes = re.sub(pattern, lambda m: mora2phonemes(m.group()), text)
+
+ # 長音記号「ー」の処理
+ long_pattern = r"(\w)(ー*)"
+ long_replacement = lambda m: m.group(1) + (" " + m.group(1)) * len(m.group(2)) # type: ignore
+ spaced_phonemes = re.sub(long_pattern, long_replacement, spaced_phonemes)
+
+ return spaced_phonemes.strip().split(" ")
+
+
+def __align_tones(
+ phones_with_punct: list[str], phone_tone_list: list[tuple[str, int]]
+) -> list[tuple[str, int]]:
+ """
+ 例: …私は、、そう思う。
+ phones_with_punct:
+ [".", ".", ".", "w", "a", "t", "a", "sh", "i", "w", "a", ",", ",", "s", "o", "o", "o", "m", "o", "u", "."]
+ phone_tone_list:
+ [("w", 0), ("a", 0), ("t", 1), ("a", 1), ("sh", 1), ("i", 1), ("w", 1), ("a", 1), ("_", 0), ("s", 0), ("o", 0), ("o", 1), ("o", 1), ("m", 1), ("o", 1), ("u", 0))]
+ Return:
+ [(".", 0), (".", 0), (".", 0), ("w", 0), ("a", 0), ("t", 1), ("a", 1), ("sh", 1), ("i", 1), ("w", 1), ("a", 1), (",", 0), (",", 0), ("s", 0), ("o", 0), ("o", 1), ("o", 1), ("m", 1), ("o", 1), ("u", 0), (".", 0)]
+
+ Args:
+ phones_with_punct (list[str]): punctuation を含む音素のリスト
+ phone_tone_list (list[tuple[str, int]]): punctuation を含まない音素とアクセントのペアのリスト
+
+ Returns:
+ list[tuple[str, int]]: punctuation を含む音素とアクセントのペアのリスト
+ """
+
+ result: list[tuple[str, int]] = []
+ tone_index = 0
+ for phone in phones_with_punct:
+ if tone_index >= len(phone_tone_list):
+ # 余った punctuation がある場合 → (punctuation, 0) を追加
+ result.append((phone, 0))
+ elif phone == phone_tone_list[tone_index][0]:
+ # phone_tone_list の現在の音素と一致する場合 → tone をそこから取得、(phone, tone) を追加
+ result.append((phone, phone_tone_list[tone_index][1]))
+ # 探す index を1つ進める
+ tone_index += 1
+ elif phone in PUNCTUATIONS:
+ # phone が punctuation の場合 → (phone, 0) を追加
+ result.append((phone, 0))
+ else:
+ logger.debug(f"phones: {phones_with_punct}")
+ logger.debug(f"phone_tone_list: {phone_tone_list}")
+ logger.debug(f"result: {result}")
+ logger.debug(f"tone_index: {tone_index}")
+ logger.debug(f"phone: {phone}")
+ raise ValueError(f"Unexpected phone: {phone}")
+
+ return result
+
+
+def __distribute_phone(n_phone: int, n_word: int) -> list[int]:
+ """
+ 左から右に 1 ずつ振り分け、次にまた左から右に1ずつ増やし、というふうに、
+ 音素の数 `n_phone` を単語の数 `n_word` に分配する。
+
+ Args:
+ n_phone (int): 音素の数
+ n_word (int): 単語の数
+
+ Returns:
+ list[int]: 単語ごとの音素の数のリスト
+ """
+
+ phones_per_word = [0] * n_word
+ for _ in range(n_phone):
+ min_tasks = min(phones_per_word)
+ min_index = phones_per_word.index(min_tasks)
+ phones_per_word[min_index] += 1
+
+ return phones_per_word
+
+
+class YomiError(Exception):
+ """
+ OpenJTalk で、読みが正しく取得できない箇所があるときに発生する例外。
+ 基本的に「学習の前処理のテキスト処理時」には発生させ、そうでない場合は、
+ ignore_yomi_error=True にしておいて、この例外を発生させないようにする。
+ """
+
+ pass
diff --git a/style_bert_vits2/nlp/japanese/g2p_utils.py b/style_bert_vits2/nlp/japanese/g2p_utils.py
new file mode 100644
index 0000000..511793f
--- /dev/null
+++ b/style_bert_vits2/nlp/japanese/g2p_utils.py
@@ -0,0 +1,92 @@
+from style_bert_vits2.nlp.japanese.g2p import g2p
+from style_bert_vits2.nlp.japanese.mora_list import (
+ MORA_KATA_TO_MORA_PHONEMES,
+ MORA_PHONEMES_TO_MORA_KATA,
+)
+from style_bert_vits2.nlp.symbols import PUNCTUATIONS
+
+
+def g2kata_tone(norm_text: str) -> list[tuple[str, int]]:
+ """
+ テキストからカタカナとアクセントのペアのリストを返す。
+ 推論時のみに使われる関数のため、常に `raise_yomi_error=False` を指定して g2p() を呼ぶ仕様になっている。
+
+ Args:
+ norm_text: 正規化されたテキスト。
+
+ Returns:
+ カタカナと音高のリスト。
+ """
+
+ phones, tones, _ = g2p(norm_text, use_jp_extra=True, raise_yomi_error=False)
+ return phone_tone2kata_tone(list(zip(phones, tones)))
+
+
+def phone_tone2kata_tone(phone_tone: list[tuple[str, int]]) -> list[tuple[str, int]]:
+ """
+ phone_tone の phone 部分をカタカナに変換する。ただし最初と最後の ("_", 0) は無視する。
+
+ Args:
+ phone_tone: 音素と音高のリスト。
+
+ Returns:
+ カタカナと音高のリスト。
+ """
+
+ # 子音の集合
+ CONSONANTS = set(
+ [
+ consonant
+ for consonant, _ in MORA_KATA_TO_MORA_PHONEMES.values()
+ if consonant is not None
+ ]
+ )
+
+ phone_tone = phone_tone[1:] # 最初の("_", 0)を無視
+ phones = [phone for phone, _ in phone_tone]
+ tones = [tone for _, tone in phone_tone]
+ result: list[tuple[str, int]] = []
+ current_mora = ""
+ for phone, next_phone, tone, next_tone in zip(phones, phones[1:], tones, tones[1:]):
+ # zip の関係で最後の ("_", 0) は無視されている
+ if phone in PUNCTUATIONS:
+ result.append((phone, tone))
+ continue
+ if phone in CONSONANTS: # n以外の子音の場合
+ assert current_mora == "", f"Unexpected {phone} after {current_mora}"
+ assert tone == next_tone, f"Unexpected {phone} tone {tone} != {next_tone}"
+ current_mora = phone
+ else:
+ # phoneが母音もしくは「N」
+ current_mora += phone
+ result.append((MORA_PHONEMES_TO_MORA_KATA[current_mora], tone))
+ current_mora = ""
+
+ return result
+
+
+def kata_tone2phone_tone(kata_tone: list[tuple[str, int]]) -> list[tuple[str, int]]:
+ """
+ `phone_tone2kata_tone()` の逆の変換を行う。
+
+ Args:
+ kata_tone: カタカナと音高のリスト。
+
+ Returns:
+ 音素と音高のリスト。
+ """
+
+ result: list[tuple[str, int]] = [("_", 0)]
+ for mora, tone in kata_tone:
+ if mora in PUNCTUATIONS:
+ result.append((mora, tone))
+ else:
+ consonant, vowel = MORA_KATA_TO_MORA_PHONEMES[mora]
+ if consonant is None:
+ result.append((vowel, tone))
+ else:
+ result.append((consonant, tone))
+ result.append((vowel, tone))
+ result.append(("_", 0))
+
+ return result
diff --git a/text/japanese_mora_list.py b/style_bert_vits2/nlp/japanese/mora_list.py
similarity index 89%
rename from text/japanese_mora_list.py
rename to style_bert_vits2/nlp/japanese/mora_list.py
index b43e54d..a0dab2f 100644
--- a/text/japanese_mora_list.py
+++ b/style_bert_vits2/nlp/japanese/mora_list.py
@@ -1,10 +1,10 @@
"""
-VOICEVOXのソースコードからお借りして最低限に改造したコード。
+以下のコードは VOICEVOX のソースコードからお借りし最低限の改造を行ったもの。
https://github.com/VOICEVOX/voicevox_engine/blob/master/voicevox_engine/tts_pipeline/mora_list.py
"""
"""
-以下のモーラ対応表はOpenJTalkのソースコードから取得し、
+以下のモーラ対応表は OpenJTalk のソースコードから取得し、
カタカナ表記とモーラが一対一対応するように改造した。
ライセンス表記：
-----------------------------------------------------------------
@@ -46,13 +46,15 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
"""
+
from typing import Optional
-# (カタカナ, 子音, 母音)の順。子音がない場合はNoneを入れる。
+
+# (カタカナ, 子音, 母音)の順。子音がない場合は None を入れる。
# 但し「ン」と「ッ」は母音のみという扱いで、「ン」は「N」、「ッ」は「q」とする。
# （元々「ッ」は「cl」）
-# また「デェ = dy e」はpyopenjtalkの出力（de e）と合わないため削除
-_mora_list_minimum: list[tuple[str, Optional[str], str]] = [
+# また「デェ = dy e」は pyopenjtalk の出力（de e）と合わないため削除
+__MORA_LIST_MINIMUM: list[tuple[str, Optional[str], str]] = [
("ヴォ", "v", "o"),
("ヴェ", "v", "e"),
("ヴィ", "v", "i"),
@@ -199,7 +201,7 @@ _mora_list_minimum: list[tuple[str, Optional[str], str]] = [
("イ", None, "i"),
("ア", None, "a"),
]
-_mora_list_additional: list[tuple[str, Optional[str], str]] = [
+__MORA_LIST_ADDITIONAL: list[tuple[str, Optional[str], str]] = [
("ヴョ", "by", "o"),
("ヴュ", "by", "u"),
("ヴャ", "by", "a"),
@@ -220,13 +222,15 @@ _mora_list_additional: list[tuple[str, Optional[str], str]] = [
("ァ", None, "a"),
]
+# モーラの音素表記とカタカナの対応表
# 例: "vo" -> "ヴォ", "a" -> "ア"
-mora_phonemes_to_mora_kata: dict[str, str] = {
- (consonant or "") + vowel: kana for [kana, consonant, vowel] in _mora_list_minimum
+MORA_PHONEMES_TO_MORA_KATA: dict[str, str] = {
+ (consonant or "") + vowel: kana for [kana, consonant, vowel] in __MORA_LIST_MINIMUM
}
+# モーラのカタカナ表記と音素の対応表
# 例: "ヴォ" -> ("v", "o"), "ア" -> (None, "a")
-mora_kata_to_mora_phonemes: dict[str, tuple[Optional[str], str]] = {
+MORA_KATA_TO_MORA_PHONEMES: dict[str, tuple[Optional[str], str]] = {
kana: (consonant, vowel)
- for [kana, consonant, vowel] in _mora_list_minimum + _mora_list_additional
+ for [kana, consonant, vowel] in __MORA_LIST_MINIMUM + __MORA_LIST_ADDITIONAL
}
diff --git a/style_bert_vits2/nlp/japanese/normalizer.py b/style_bert_vits2/nlp/japanese/normalizer.py
new file mode 100644
index 0000000..07b742c
--- /dev/null
+++ b/style_bert_vits2/nlp/japanese/normalizer.py
@@ -0,0 +1,162 @@
+import re
+import unicodedata
+
+from num2words import num2words
+
+from style_bert_vits2.nlp.symbols import PUNCTUATIONS
+
+
+def normalize_text(text: str) -> str:
+ """
+ 日本語のテキストを正規化する。
+ 結果は、ちょうど次の文字のみからなる：
+ - ひらがな
+ - カタカナ（全角長音記号「ー」が入る！）
+ - 漢字
+ - 半角アルファベット（大文字と小文字）
+ - ギリシャ文字
+ - `.` （句点`。`や`…`の一部や改行等）
+ - `,` （読点`、`や`:`等）
+ - `?` （疑問符`？`）
+ - `!` （感嘆符`！`）
+ - `'` （`「`や`」`等）
+ - `-` （`―`（ダッシュ、長音記号ではない）や`-`等）
+
+ 注意点:
+ - 三点リーダー`…`は`...`に変換される（`なるほど…。` → `なるほど....`）
+ - 数字は漢字に変換される（`1,100円` → `千百円`、`52.34` → `五十二点三四`）
+ - 読点や疑問符等の位置・個数等は保持される（`??あ、、！！！` → `??あ,,!!!`）
+
+ Args:
+ text (str): 正規化するテキスト
+
+ Returns:
+ str: 正規化されたテキスト
+ """
+
+ res = unicodedata.normalize("NFKC", text) # ここでアルファベットは半角になる
+ res = __convert_numbers_to_words(res) # 「100円」→「百円」等
+ # 「～」と「~」も長音記号として扱う
+ res = res.replace("~", "ー")
+ res = res.replace("～", "ー")
+
+ res = replace_punctuation(res) # 句読点等正規化、読めない文字を削除
+
+ # 結合文字の濁点・半濁点を削除
+ # 通常の「ば」等はそのままのこされる、「あ゛」は上で「あ゙」になりここで「あ」になる
+ res = res.replace("\u3099", "") # 結合文字の濁点を削除、る゙ → る
+ res = res.replace("\u309A", "") # 結合文字の半濁点を削除、な゚ → な
+ return res
+
+
+def replace_punctuation(text: str) -> str:
+ """
+ 句読点等を「.」「,」「!」「?」「'」「-」に正規化し、OpenJTalk で読みが取得できるもののみ残す：
+ 漢字・平仮名・カタカナ、アルファベット、ギリシャ文字
+
+ Args:
+ text (str): 正規化するテキスト
+
+ Returns:
+ str: 正規化されたテキスト
+ """
+
+ # 記号類の正規化変換マップ
+ REPLACE_MAP = {
+ "：": ",",
+ "；": ",",
+ "，": ",",
+ "。": ".",
+ "！": "!",
+ "？": "?",
+ "\n": ".",
+ "．": ".",
+ "…": "...",
+ "···": "...",
+ "・・・": "...",
+ "·": ",",
+ "・": ",",
+ "、": ",",
+ "$": ".",
+ "“": "'",
+ "”": "'",
+ '"': "'",
+ "‘": "'",
+ "’": "'",
+ "（": "'",
+ "）": "'",
+ "(": "'",
+ ")": "'",
+ "《": "'",
+ "》": "'",
+ "【": "'",
+ "】": "'",
+ "[": "'",
+ "]": "'",
+ # NFKC 正規化後のハイフン・ダッシュの変種を全て通常半角ハイフン - \u002d に変換
+ "\u02d7": "\u002d", # ˗, Modifier Letter Minus Sign
+ "\u2010": "\u002d", # ‐, Hyphen,
+ # "\u2011": "\u002d", # ‑, Non-Breaking Hyphen, NFKC により \u2010 に変換される
+ "\u2012": "\u002d", # ‒, Figure Dash
+ "\u2013": "\u002d", # –, En Dash
+ "\u2014": "\u002d", # —, Em Dash
+ "\u2015": "\u002d", # ―, Horizontal Bar
+ "\u2043": "\u002d", # ⁃, Hyphen Bullet
+ "\u2212": "\u002d", # −, Minus Sign
+ "\u23af": "\u002d", # ⎯, Horizontal Line Extension
+ "\u23e4": "\u002d", # ⏤, Straightness
+ "\u2500": "\u002d", # ─, Box Drawings Light Horizontal
+ "\u2501": "\u002d", # ━, Box Drawings Heavy Horizontal
+ "\u2e3a": "\u002d", # ⸺, Two-Em Dash
+ "\u2e3b": "\u002d", # ⸻, Three-Em Dash
+ # "～": "-", # これは長音記号「ー」として扱うよう変更
+ # "~": "-", # これも長音記号「ー」として扱うよう変更
+ "「": "'",
+ "」": "'",
+ }
+
+ pattern = re.compile("|".join(re.escape(p) for p in REPLACE_MAP.keys()))
+
+ # 句読点を辞書で置換
+ replaced_text = pattern.sub(lambda x: REPLACE_MAP[x.group()], text)
+
+ replaced_text = re.sub(
+ # ↓ ひらがな、カタカナ、漢字
+ r"[^\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\u3400-\u4DBF\u3005"
+ # ↓ 半角アルファベット（大文字と小文字）
+ + r"\u0041-\u005A\u0061-\u007A"
+ # ↓ 全角アルファベット（大文字と小文字）
+ + r"\uFF21-\uFF3A\uFF41-\uFF5A"
+ # ↓ ギリシャ文字
+ + r"\u0370-\u03FF\u1F00-\u1FFF"
+ # ↓ "!", "?", "…", ",", ".", "'", "-", 但し`…`はすでに`...`に変換されている
+ + "".join(PUNCTUATIONS) + r"]+",
+ # 上述以外の文字を削除
+ "",
+ replaced_text,
+ )
+
+ return replaced_text
+
+
+def __convert_numbers_to_words(text: str) -> str:
+ """
+ 記号や数字を日本語の文字表現に変換する。
+
+ Args:
+ text (str): 変換するテキスト
+
+ Returns:
+ str: 変換されたテキスト
+ """
+
+ NUMBER_WITH_SEPARATOR_PATTERN = re.compile("[0-9]{1,3}(,[0-9]{3})+")
+ CURRENCY_MAP = {"$": "ドル", "¥": "円", "£": "ポンド", "€": "ユーロ"}
+ CURRENCY_PATTERN = re.compile(r"([$¥£€])([0-9.]*[0-9])")
+ NUMBER_PATTERN = re.compile(r"[0-9]+(\.[0-9]+)?")
+
+ res = NUMBER_WITH_SEPARATOR_PATTERN.sub(lambda m: m[0].replace(",", ""), text)
+ res = CURRENCY_PATTERN.sub(lambda m: m[2] + CURRENCY_MAP.get(m[1], m[1]), res)
+ res = NUMBER_PATTERN.sub(lambda m: num2words(m[0], lang="ja"), res)
+
+ return res
diff --git a/style_bert_vits2/nlp/japanese/pyopenjtalk_worker/__init__.py b/style_bert_vits2/nlp/japanese/pyopenjtalk_worker/__init__.py
new file mode 100644
index 0000000..3a146b6
--- /dev/null
+++ b/style_bert_vits2/nlp/japanese/pyopenjtalk_worker/__init__.py
@@ -0,0 +1,160 @@
+"""
+Run the pyopenjtalk worker in a separate process
+to avoid user dictionary access error
+"""
+
+from typing import Any, Optional
+
+from style_bert_vits2.logging import logger
+from style_bert_vits2.nlp.japanese.pyopenjtalk_worker.worker_client import WorkerClient
+from style_bert_vits2.nlp.japanese.pyopenjtalk_worker.worker_common import WORKER_PORT
+
+
+WORKER_CLIENT: Optional[WorkerClient] = None
+
+
+# pyopenjtalk interface
+# g2p(): not used
+
+
+def run_frontend(text: str) -> list[dict[str, Any]]:
+ if WORKER_CLIENT is not None:
+ ret = WORKER_CLIENT.dispatch_pyopenjtalk("run_frontend", text)
+ assert isinstance(ret, list)
+ return ret
+ else:
+ # without worker
+ import pyopenjtalk
+
+ return pyopenjtalk.run_frontend(text)
+
+
+def make_label(njd_features: Any) -> list[str]:
+ if WORKER_CLIENT is not None:
+ ret = WORKER_CLIENT.dispatch_pyopenjtalk("make_label", njd_features)
+ assert isinstance(ret, list)
+ return ret
+ else:
+ # without worker
+ import pyopenjtalk
+
+ return pyopenjtalk.make_label(njd_features)
+
+
+def mecab_dict_index(path: str, out_path: str, dn_mecab: Optional[str] = None) -> None:
+ if WORKER_CLIENT is not None:
+ WORKER_CLIENT.dispatch_pyopenjtalk("mecab_dict_index", path, out_path, dn_mecab)
+ else:
+ # without worker
+ import pyopenjtalk
+
+ pyopenjtalk.mecab_dict_index(path, out_path, dn_mecab)
+
+
+def update_global_jtalk_with_user_dict(path: str) -> None:
+ if WORKER_CLIENT is not None:
+ WORKER_CLIENT.dispatch_pyopenjtalk("update_global_jtalk_with_user_dict", path)
+ else:
+ # without worker
+ import pyopenjtalk
+
+ pyopenjtalk.update_global_jtalk_with_user_dict(path)
+
+
+def unset_user_dict() -> None:
+ if WORKER_CLIENT is not None:
+ WORKER_CLIENT.dispatch_pyopenjtalk("unset_user_dict")
+ else:
+ # without worker
+ import pyopenjtalk
+
+ pyopenjtalk.unset_user_dict()
+
+
+# initialize module when imported
+
+
+def initialize_worker(port: int = WORKER_PORT) -> None:
+ import atexit
+ import signal
+ import socket
+ import sys
+ import time
+
+ global WORKER_CLIENT
+ if WORKER_CLIENT:
+ return
+
+ client = None
+ try:
+ client = WorkerClient(port)
+ except (socket.timeout, socket.error):
+ logger.debug("try starting pyopenjtalk worker server")
+ import os
+ import subprocess
+
+ worker_pkg_path = os.path.relpath(
+ os.path.dirname(__file__), os.getcwd()
+ ).replace(os.sep, ".")
+ args = [sys.executable, "-m", worker_pkg_path, "--port", str(port)]
+ # new session, new process group
+ if sys.platform.startswith("win"):
+ cf = subprocess.CREATE_NEW_CONSOLE | subprocess.CREATE_NEW_PROCESS_GROUP # type: ignore
+ si = subprocess.STARTUPINFO() # type: ignore
+ si.dwFlags |= subprocess.STARTF_USESHOWWINDOW # type: ignore
+ si.wShowWindow = subprocess.SW_HIDE # type: ignore
+ subprocess.Popen(args, creationflags=cf, startupinfo=si)
+ else:
+ # align with Windows behavior
+ # start_new_session is same as specifying setsid in preexec_fn
+ subprocess.Popen(
+ args,
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ start_new_session=True,
+ )
+
+ # wait until server listening
+ count = 0
+ while True:
+ try:
+ client = WorkerClient(port)
+ break
+ except socket.error:
+ time.sleep(0.5)
+ count += 1
+ # 20: max number of retries
+ if count == 20:
+ raise TimeoutError("サーバーに接続できませんでした")
+
+ logger.debug("pyopenjtalk worker server started")
+ WORKER_CLIENT = client
+ atexit.register(terminate_worker)
+
+ # when the process is killed
+ def signal_handler(signum: int, frame: Any):
+ terminate_worker()
+
+ try:
+ signal.signal(signal.SIGTERM, signal_handler)
+ except ValueError:
+ # signal only works in main thread
+ pass
+
+
+# top-level declaration
+def terminate_worker() -> None:
+ logger.debug("pyopenjtalk worker server terminated")
+ global WORKER_CLIENT
+ if not WORKER_CLIENT:
+ return
+
+ # prepare for unexpected errors
+ try:
+ if WORKER_CLIENT.status() == 1:
+ WORKER_CLIENT.quit_server()
+ except Exception as e:
+ logger.error(e)
+
+ WORKER_CLIENT.close()
+ WORKER_CLIENT = None
diff --git a/text/pyopenjtalk_worker/__main__.py b/style_bert_vits2/nlp/japanese/pyopenjtalk_worker/__main__.py
similarity index 57%
rename from text/pyopenjtalk_worker/__main__.py
rename to style_bert_vits2/nlp/japanese/pyopenjtalk_worker/__main__.py
index 3bb6b53..2452a16 100644
--- a/text/pyopenjtalk_worker/__main__.py
+++ b/style_bert_vits2/nlp/japanese/pyopenjtalk_worker/__main__.py
@@ -1,16 +1,16 @@
-import argparse
-
-from .worker_server import WorkerServer
-from .worker_common import WORKER_PORT
-
-
-def main():
- parser = argparse.ArgumentParser()
- parser.add_argument("--port", type=int, default=WORKER_PORT)
- args = parser.parse_args()
- server = WorkerServer()
- server.start_server(port=args.port)
-
-
-if __name__ == "__main__":
- main()
+import argparse
+
+from style_bert_vits2.nlp.japanese.pyopenjtalk_worker.worker_common import WORKER_PORT
+from style_bert_vits2.nlp.japanese.pyopenjtalk_worker.worker_server import WorkerServer
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--port", type=int, default=WORKER_PORT)
+ args = parser.parse_args()
+ server = WorkerServer()
+ server.start_server(port=args.port)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/text/pyopenjtalk_worker/worker_client.py b/style_bert_vits2/nlp/japanese/pyopenjtalk_worker/worker_client.py
similarity index 69%
rename from text/pyopenjtalk_worker/worker_client.py
rename to style_bert_vits2/nlp/japanese/pyopenjtalk_worker/worker_client.py
index 710243d..c4c5606 100644
--- a/text/pyopenjtalk_worker/worker_client.py
+++ b/style_bert_vits2/nlp/japanese/pyopenjtalk_worker/worker_client.py
@@ -1,55 +1,60 @@
-from typing import Any
-import socket
-
-from .worker_common import RequestType, receive_data, send_data
-
-from common.log import logger
-
-
-class WorkerClient:
- def __init__(self, port: int) -> None:
- sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- # 60: timeout
- sock.settimeout(60)
- sock.connect((socket.gethostname(), port))
- self.sock = sock
-
- def __enter__(self):
- return self
-
- def __exit__(self, exc_type, exc_value, traceback):
- self.close()
-
- def close(self):
- self.sock.close()
-
- def dispatch_pyopenjtalk(self, func: str, *args, **kwargs):
- data = {
- "request-type": RequestType.PYOPENJTALK,
- "func": func,
- "args": args,
- "kwargs": kwargs,
- }
- logger.trace(f"client sends request: {data}")
- send_data(self.sock, data)
- logger.trace("client sent request successfully")
- response = receive_data(self.sock)
- logger.trace(f"client received response: {response}")
- return response.get("return")
-
- def status(self):
- data = {"request-type": RequestType.STATUS}
- logger.trace(f"client sends request: {data}")
- send_data(self.sock, data)
- logger.trace("client sent request successfully")
- response = receive_data(self.sock)
- logger.trace(f"client received response: {response}")
- return response.get("client-count")
-
- def quit_server(self):
- data = {"request-type": RequestType.QUIT_SERVER}
- logger.trace(f"client sends request: {data}")
- send_data(self.sock, data)
- logger.trace("client sent request successfully")
- response = receive_data(self.sock)
- logger.trace(f"client received response: {response}")
+import socket
+from typing import Any, cast
+
+from style_bert_vits2.logging import logger
+from style_bert_vits2.nlp.japanese.pyopenjtalk_worker.worker_common import (
+ RequestType,
+ receive_data,
+ send_data,
+)
+
+
+class WorkerClient:
+ """pyopenjtalk worker client"""
+
+ def __init__(self, port: int) -> None:
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ # timeout: 60 seconds
+ sock.settimeout(60)
+ sock.connect((socket.gethostname(), port))
+ self.sock = sock
+
+ def __enter__(self) -> "WorkerClient":
+ return self
+
+ def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+ self.close()
+
+ def close(self) -> None:
+ self.sock.close()
+
+ def dispatch_pyopenjtalk(self, func: str, *args: Any, **kwargs: Any) -> Any:
+ data = {
+ "request-type": RequestType.PYOPENJTALK,
+ "func": func,
+ "args": args,
+ "kwargs": kwargs,
+ }
+ logger.trace(f"client sends request: {data}")
+ send_data(self.sock, data)
+ logger.trace("client sent request successfully")
+ response = receive_data(self.sock)
+ logger.trace(f"client received response: {response}")
+ return response.get("return")
+
+ def status(self) -> int:
+ data = {"request-type": RequestType.STATUS}
+ logger.trace(f"client sends request: {data}")
+ send_data(self.sock, data)
+ logger.trace("client sent request successfully")
+ response = receive_data(self.sock)
+ logger.trace(f"client received response: {response}")
+ return cast(int, response.get("client-count"))
+
+ def quit_server(self) -> None:
+ data = {"request-type": RequestType.QUIT_SERVER}
+ logger.trace(f"client sends request: {data}")
+ send_data(self.sock, data)
+ logger.trace("client sent request successfully")
+ response = receive_data(self.sock)
+ logger.trace(f"client received response: {response}")
diff --git a/text/pyopenjtalk_worker/worker_common.py b/style_bert_vits2/nlp/japanese/pyopenjtalk_worker/worker_common.py
similarity index 79%
rename from text/pyopenjtalk_worker/worker_common.py
rename to style_bert_vits2/nlp/japanese/pyopenjtalk_worker/worker_common.py
index 606d0c3..7d79a5a 100644
--- a/text/pyopenjtalk_worker/worker_common.py
+++ b/style_bert_vits2/nlp/japanese/pyopenjtalk_worker/worker_common.py
@@ -1,44 +1,45 @@
-from typing import Any, Optional, Final
-from enum import IntEnum, auto
-import socket
-import json
-
-WORKER_PORT: Final[int] = 7861
-HEADER_SIZE: Final[int] = 4
-
-
-class RequestType(IntEnum):
- STATUS = auto()
- QUIT_SERVER = auto()
- PYOPENJTALK = auto()
-
-
-class ConnectionClosedException(Exception):
- pass
-
-
-# socket communication
-
-
-def send_data(sock: socket.socket, data: dict[str, Any]):
- json_data = json.dumps(data).encode()
- header = len(json_data).to_bytes(HEADER_SIZE, byteorder="big")
- sock.sendall(header + json_data)
-
-
-def _receive_until(sock: socket.socket, size: int):
- data = b""
- while len(data) < size:
- part = sock.recv(size - len(data))
- if part == b"":
- raise ConnectionClosedException("接続が閉じられました")
- data += part
-
- return data
-
-
-def receive_data(sock: socket.socket) -> dict[str, Any]:
- header = _receive_until(sock, HEADER_SIZE)
- data_length = int.from_bytes(header, byteorder="big")
- body = _receive_until(sock, data_length)
- return json.loads(body.decode())
+import json
+import socket
+from enum import IntEnum, auto
+from typing import Any, Final
+
+
+WORKER_PORT: Final[int] = 7861
+HEADER_SIZE: Final[int] = 4
+
+
+class RequestType(IntEnum):
+ STATUS = auto()
+ QUIT_SERVER = auto()
+ PYOPENJTALK = auto()
+
+
+class ConnectionClosedException(Exception):
+ pass
+
+
+# socket communication
+
+
+def send_data(sock: socket.socket, data: dict[str, Any]):
+ json_data = json.dumps(data).encode()
+ header = len(json_data).to_bytes(HEADER_SIZE, byteorder="big")
+ sock.sendall(header + json_data)
+
+
+def __receive_until(sock: socket.socket, size: int):
+ data = b""
+ while len(data) < size:
+ part = sock.recv(size - len(data))
+ if part == b"":
+ raise ConnectionClosedException("接続が閉じられました")
+ data += part
+
+ return data
+
+
+def receive_data(sock: socket.socket) -> dict[str, Any]:
+ header = __receive_until(sock, HEADER_SIZE)
+ data_length = int.from_bytes(header, byteorder="big")
+ body = __receive_until(sock, data_length)
+ return json.loads(body.decode())
diff --git a/text/pyopenjtalk_worker/worker_server.py b/style_bert_vits2/nlp/japanese/pyopenjtalk_worker/worker_server.py
similarity index 89%
rename from text/pyopenjtalk_worker/worker_server.py
rename to style_bert_vits2/nlp/japanese/pyopenjtalk_worker/worker_server.py
index 6babd91..ed0d4e7 100644
--- a/text/pyopenjtalk_worker/worker_server.py
+++ b/style_bert_vits2/nlp/japanese/pyopenjtalk_worker/worker_server.py
@@ -1,118 +1,123 @@
-import pyopenjtalk
-import socket
-import select
-import time
-
-from .worker_common import (
- ConnectionClosedException,
- RequestType,
- receive_data,
- send_data,
-)
-
-from common.log import logger
-
-# To make it as fast as possible
-# Probably faster than calling getattr every time
-_PYOPENJTALK_FUNC_DICT = {
- "run_frontend": pyopenjtalk.run_frontend,
- "make_label": pyopenjtalk.make_label,
- "mecab_dict_index": pyopenjtalk.mecab_dict_index,
- "update_global_jtalk_with_user_dict": pyopenjtalk.update_global_jtalk_with_user_dict,
- "unset_user_dict": pyopenjtalk.unset_user_dict,
-}
-
-
-class WorkerServer:
- def __init__(self) -> None:
- self.client_count: int = 0
- self.quit: bool = False
-
- def handle_request(self, request):
- request_type = None
- try:
- request_type = RequestType(request.get("request-type"))
- except Exception:
- return {
- "success": False,
- "reason": "request-type is invalid",
- }
-
- if request_type:
- if request_type == RequestType.STATUS:
- response = {
- "success": True,
- "client-count": self.client_count,
- }
- elif request_type == RequestType.QUIT_SERVER:
- self.quit = True
- response = {"success": True}
- elif request_type == RequestType.PYOPENJTALK:
- func_name = request.get("func")
- assert isinstance(func_name, str)
- func = _PYOPENJTALK_FUNC_DICT[func_name]
- args = request.get("args")
- kwargs = request.get("kwargs")
- assert isinstance(args, list)
- assert isinstance(kwargs, dict)
- ret = func(*args, **kwargs)
- response = {"success": True, "return": ret}
- else:
- # NOT REACHED
- response = request
-
- return response
-
- def start_server(self, port: int, no_client_timeout: int = 30):
- logger.info("start pyopenjtalk worker server")
- with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server_socket:
- server_socket.bind((socket.gethostname(), port))
- server_socket.listen()
- sockets = [server_socket]
- no_client_since = time.time()
- while True:
- if self.client_count == 0:
- if no_client_since is None:
- no_client_since = time.time()
- elif (time.time() - no_client_since) > no_client_timeout:
- logger.info("quit because there is no client")
- return
- else:
- no_client_since = None
-
- ready_sockets, _, _ = select.select(sockets, [], [], 0.1)
- for sock in ready_sockets:
- if sock is server_socket:
- logger.info("new client connected")
- client_socket, _ = server_socket.accept()
- sockets.append(client_socket)
- self.client_count += 1
- else:
- # client
- try:
- request = receive_data(sock)
- except Exception as e:
- sock.close()
- sockets.remove(sock)
- self.client_count -= 1
- # unexpected disconnections
- if not isinstance(e, ConnectionClosedException):
- logger.error(e)
-
- logger.info("close connection")
- continue
-
- logger.trace(f"server received request: {request}")
-
- response = self.handle_request(request)
- logger.trace(f"server sends response: {response}")
- try:
- send_data(sock, response)
- logger.trace("server sent response successfully")
- except Exception:
- logger.warning(
- "an exception occurred during sending responce"
- )
- if self.quit:
- logger.info("quit pyopenjtalk worker server")
- return
+import select
+import socket
+import time
+from typing import Any, cast
+
+import pyopenjtalk
+
+from style_bert_vits2.logging import logger
+from style_bert_vits2.nlp.japanese.pyopenjtalk_worker.worker_common import (
+ ConnectionClosedException,
+ RequestType,
+ receive_data,
+ send_data,
+)
+
+
+# To make it as fast as possible
+# Probably faster than calling getattr every time
+PYOPENJTALK_FUNC_DICT = {
+ "run_frontend": pyopenjtalk.run_frontend,
+ "make_label": pyopenjtalk.make_label,
+ "mecab_dict_index": pyopenjtalk.mecab_dict_index,
+ "update_global_jtalk_with_user_dict": pyopenjtalk.update_global_jtalk_with_user_dict,
+ "unset_user_dict": pyopenjtalk.unset_user_dict,
+}
+
+
+class WorkerServer:
+ """pyopenjtalk worker server"""
+
+ def __init__(self) -> None:
+ self.client_count: int = 0
+ self.quit: bool = False
+
+ def handle_request(self, request: dict[str, Any]) -> dict[str, Any]:
+ request_type = None
+ try:
+ request_type = RequestType(cast(int, request.get("request-type")))
+ except Exception:
+ return {
+ "success": False,
+ "reason": "request-type is invalid",
+ }
+
+ response: dict[str, Any] = {}
+ if request_type:
+ if request_type == RequestType.STATUS:
+ response = {
+ "success": True,
+ "client-count": self.client_count,
+ }
+ elif request_type == RequestType.QUIT_SERVER:
+ self.quit = True
+ response = {"success": True}
+ elif request_type == RequestType.PYOPENJTALK:
+ func_name = request.get("func")
+ assert isinstance(func_name, str)
+ func = PYOPENJTALK_FUNC_DICT[func_name]
+ args = request.get("args")
+ kwargs = request.get("kwargs")
+ assert isinstance(args, list)
+ assert isinstance(kwargs, dict)
+ ret = func(*args, **kwargs)
+ response = {"success": True, "return": ret}
+ else:
+ # NOT REACHED
+ response = request
+
+ return response
+
+ def start_server(self, port: int, no_client_timeout: int = 30) -> None:
+ logger.info("start pyopenjtalk worker server")
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server_socket:
+ server_socket.bind((socket.gethostname(), port))
+ server_socket.listen()
+ sockets = [server_socket]
+ no_client_since = time.time()
+ while True:
+ if self.client_count == 0:
+ if no_client_since is None:
+ no_client_since = time.time()
+ elif (time.time() - no_client_since) > no_client_timeout:
+ logger.info("quit because there is no client")
+ return
+ else:
+ no_client_since = None
+
+ ready_sockets, _, _ = select.select(sockets, [], [], 0.1)
+ for sock in ready_sockets:
+ if sock is server_socket:
+ logger.info("new client connected")
+ client_socket, _ = server_socket.accept()
+ sockets.append(client_socket)
+ self.client_count += 1
+ else:
+ # client
+ try:
+ request = receive_data(sock)
+ except Exception as e:
+ sock.close()
+ sockets.remove(sock)
+ self.client_count -= 1
+ # unexpected disconnections
+ if not isinstance(e, ConnectionClosedException):
+ logger.error(e)
+
+ logger.info("close connection")
+ continue
+
+ logger.trace(f"server received request: {request}")
+
+ response = self.handle_request(request)
+ logger.trace(f"server sends response: {response}")
+ try:
+ send_data(sock, response)
+ logger.trace("server sent response successfully")
+ except Exception:
+ logger.warning(
+ "an exception occurred during sending responce"
+ )
+ if self.quit:
+ logger.info("quit pyopenjtalk worker server")
+ return
diff --git a/style_bert_vits2/nlp/japanese/user_dict/README.md b/style_bert_vits2/nlp/japanese/user_dict/README.md
new file mode 100644
index 0000000..3e7a0c0
--- /dev/null
+++ b/style_bert_vits2/nlp/japanese/user_dict/README.md
@@ -0,0 +1,27 @@
+
+## ユーザー辞書関連のコードについて
+
+このフォルダに含まれるユーザー辞書関連のコードは、[VOICEVOX ENGINE](https://github.com/VOICEVOX/voicevox_engine) プロジェクトのコードを改変したものを使用しています。
+VOICEVOX プロジェクトのチームに深く感謝し、その貢献を尊重します。
+
+### 元のコード
+
+- [voicevox_engine/user_dict/](https://github.com/VOICEVOX/voicevox_engine/tree/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/user_dict)
+- [voicevox_engine/model.py](https://github.com/VOICEVOX/voicevox_engine/blob/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/model.py#L207)
+
+### 改変の詳細
+
+- ファイル名の書き換えおよびそれに伴う import 文の書き換え。
+- VOICEVOX 固有の部分をコメントアウト。
+- mutex を使用している部分をコメントアウト。
+- 参照している pyopenjtalk の違いによるメソッド名の書き換え。
+- UserDictWord の mora_count のデフォルト値を None に指定。
+- `model.py` のうち、必要な Pydantic モデルのみを抽出。
+
+### ライセンス
+
+元の VOICEVOX ENGINE のリポジトリのコードは、LGPL v3 と、ソースコードの公開が不要な別ライセンスのデュアルライセンスの下で使用されています。
+当プロジェクトにおけるこのモジュールも LGPL ライセンスの下にあります。
+
+詳細については、プロジェクトのルートディレクトリにある [LGPL_LICENSE](/LGPL_LICENSE) ファイルをご参照ください。
+また、元の VOICEVOX ENGINE プロジェクトのライセンスについては、[こちら](https://github.com/VOICEVOX/voicevox_engine/blob/master/LICENSE) をご覧ください。
diff --git a/text/user_dict/__init__.py b/style_bert_vits2/nlp/japanese/user_dict/__init__.py
similarity index 92%
rename from text/user_dict/__init__.py
rename to style_bert_vits2/nlp/japanese/user_dict/__init__.py
index 95515cb..53ea63f 100644
--- a/text/user_dict/__init__.py
+++ b/style_bert_vits2/nlp/japanese/user_dict/__init__.py
@@ -1,41 +1,43 @@
-# このファイルは、VOICEVOXプロジェクトのVOICEVOX engineからお借りしています。
-# 引用元:
-# https://github.com/VOICEVOX/voicevox_engine/blob/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/user_dict/user_dict.py
-# ライセンス: LGPL-3.0
-# 詳しくは、このファイルと同じフォルダにあるREADME.mdを参照してください。
+"""
+このファイルは、VOICEVOX プロジェクトの VOICEVOX ENGINE からお借りしています。
+引用元: https://github.com/VOICEVOX/voicevox_engine/blob/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/user_dict/user_dict.py
+ライセンス: LGPL-3.0
+詳しくは、このファイルと同じフォルダにある README.md を参照してください。
+"""
+
import json
import sys
-import threading
import traceback
from pathlib import Path
from typing import Dict, List, Optional
from uuid import UUID, uuid4
import numpy as np
-from .. import pyopenjtalk_worker as pyopenjtalk
-
-pyopenjtalk.initialize()
from fastapi import HTTPException
-from .word_model import UserDictWord, WordTypes
+from style_bert_vits2.constants import DEFAULT_USER_DICT_DIR
+from style_bert_vits2.nlp.japanese import pyopenjtalk_worker as pyopenjtalk
+from style_bert_vits2.nlp.japanese.user_dict.part_of_speech_data import (
+ MAX_PRIORITY,
+ MIN_PRIORITY,
+ part_of_speech_data,
+)
+from style_bert_vits2.nlp.japanese.user_dict.word_model import UserDictWord, WordTypes
-# from ..utility.mutex_utility import mutex_wrapper
-# from ..utility.path_utility import engine_root, get_save_dir
-from .part_of_speech_data import MAX_PRIORITY, MIN_PRIORITY, part_of_speech_data
-from common.constants import USER_DICT_DIR
# root_dir = engine_root()
# save_dir = get_save_dir()
-root_dir = Path(USER_DICT_DIR)
-save_dir = Path(USER_DICT_DIR)
+# if not save_dir.is_dir():
+# save_dir.mkdir(parents=True)
-if not save_dir.is_dir():
- save_dir.mkdir(parents=True)
-
-default_dict_path = root_dir / "default.csv" # VOICEVOXデフォルト辞書ファイルのパス
-user_dict_path = save_dir / "user_dict.json" # ユーザー辞書ファイルのパス
-compiled_dict_path = save_dir / "user.dic" # コンパイル済み辞書ファイルのパス
+default_dict_path = (
+ DEFAULT_USER_DICT_DIR / "default.csv"
+) # VOICEVOXデフォルト辞書ファイルのパス
+user_dict_path = DEFAULT_USER_DICT_DIR / "user_dict.json" # ユーザー辞書ファイルのパス
+compiled_dict_path = (
+ DEFAULT_USER_DICT_DIR / "user.dic"
+) # コンパイル済み辞書ファイルのパス
# # 同時書き込みの制御
@@ -56,7 +58,7 @@ def _write_to_json(user_dict: Dict[str, UserDictWord], user_dict_path: Path) ->
"""
converted_user_dict = {}
for word_uuid, word in user_dict.items():
- word_dict = word.dict()
+ word_dict = word.model_dump()
word_dict["cost"] = _priority2cost(
word_dict["context_id"], word_dict["priority"]
)
@@ -86,6 +88,7 @@ def update_dict(
compiled_dict_path : Path
コンパイル済み辞書ファイルのパス
"""
+
random_string = uuid4()
tmp_csv_path = compiled_dict_path.with_suffix(
f".dict_csv-{random_string}.tmp"
diff --git a/text/user_dict/part_of_speech_data.py b/style_bert_vits2/nlp/japanese/user_dict/part_of_speech_data.py
similarity index 87%
rename from text/user_dict/part_of_speech_data.py
rename to style_bert_vits2/nlp/japanese/user_dict/part_of_speech_data.py
index 7e22699..a48f0c5 100644
--- a/text/user_dict/part_of_speech_data.py
+++ b/style_bert_vits2/nlp/japanese/user_dict/part_of_speech_data.py
@@ -1,18 +1,20 @@
-# このファイルは、VOICEVOXプロジェクトのVOICEVOX engineからお借りしています。
-# 引用元:
-# https://github.com/VOICEVOX/voicevox_engine/blob/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/user_dict/part_of_speech_data.py
-# ライセンス: LGPL-3.0
-# 詳しくは、このファイルと同じフォルダにあるREADME.mdを参照してください。
+"""
+このファイルは、VOICEVOX プロジェクトの VOICEVOX ENGINE からお借りしています。
+引用元: https://github.com/VOICEVOX/voicevox_engine/blob/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/user_dict/part_of_speech_data.py
+ライセンス: LGPL-3.0
+詳しくは、このファイルと同じフォルダにある README.md を参照してください。
+"""
from typing import Dict
-from .word_model import (
+from style_bert_vits2.nlp.japanese.user_dict.word_model import (
USER_DICT_MAX_PRIORITY,
USER_DICT_MIN_PRIORITY,
PartOfSpeechDetail,
WordTypes,
)
+
MIN_PRIORITY = USER_DICT_MIN_PRIORITY
MAX_PRIORITY = USER_DICT_MAX_PRIORITY
diff --git a/text/user_dict/word_model.py b/style_bert_vits2/nlp/japanese/user_dict/word_model.py
similarity index 93%
rename from text/user_dict/word_model.py
rename to style_bert_vits2/nlp/japanese/user_dict/word_model.py
index f05d8dc..c85a5b9 100644
--- a/text/user_dict/word_model.py
+++ b/style_bert_vits2/nlp/japanese/user_dict/word_model.py
@@ -1,14 +1,17 @@
-# このファイルは、VOICEVOXプロジェクトのVOICEVOX engineからお借りしています。
-# 引用元:
-# https://github.com/VOICEVOX/voicevox_engine/blob/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/model.py#L207
-# ライセンス: LGPL-3.0
-# 詳しくは、このファイルと同じフォルダにあるREADME.mdを参照してください。
+"""
+このファイルは、VOICEVOX プロジェクトの VOICEVOX ENGINE からお借りしています。
+引用元: https://github.com/VOICEVOX/voicevox_engine/blob/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/model.py#L207
+ライセンス: LGPL-3.0
+詳しくは、このファイルと同じフォルダにある README.md を参照してください。
+"""
+
from enum import Enum
from re import findall, fullmatch
from typing import List, Optional
from pydantic import BaseModel, Field, validator
+
USER_DICT_MIN_PRIORITY = 0
USER_DICT_MAX_PRIORITY = 10
diff --git a/text/symbols.py b/style_bert_vits2/nlp/symbols.py
similarity index 63%
rename from text/symbols.py
rename to style_bert_vits2/nlp/symbols.py
index 846de64..e3a650f 100644
--- a/text/symbols.py
+++ b/style_bert_vits2/nlp/symbols.py
@@ -1,9 +1,14 @@
-punctuation = ["!", "?", "…", ",", ".", "'", "-"]
-pu_symbols = punctuation + ["SP", "UNK"]
-pad = "_"
+# Punctuations
+PUNCTUATIONS = ["!", "?", "…", ",", ".", "'", "-"]
-# chinese
-zh_symbols = [
+# Punctuations and special tokens
+PUNCTUATION_SYMBOLS = PUNCTUATIONS + ["SP", "UNK"]
+
+# Padding
+PAD = "_"
+
+# Chinese symbols
+ZH_SYMBOLS = [
"E",
"En",
"a",
@@ -70,10 +75,10 @@ zh_symbols = [
"EE",
"OO",
]
-num_zh_tones = 6
+NUM_ZH_TONES = 6
-# japanese
-ja_symbols = [
+# Japanese
+JP_SYMBOLS = [
"N",
"a",
"a:",
@@ -117,10 +122,10 @@ ja_symbols = [
"z",
"zy",
]
-num_ja_tones = 2
+NUM_JP_TONES = 2
# English
-en_symbols = [
+EN_SYMBOLS = [
"aa",
"ae",
"ah",
@@ -161,27 +166,29 @@ en_symbols = [
"z",
"zh",
]
-num_en_tones = 4
+NUM_EN_TONES = 4
-# combine all symbols
-normal_symbols = sorted(set(zh_symbols + ja_symbols + en_symbols))
-symbols = [pad] + normal_symbols + pu_symbols
-sil_phonemes_ids = [symbols.index(i) for i in pu_symbols]
+# Combine all symbols
+NORMAL_SYMBOLS = sorted(set(ZH_SYMBOLS + JP_SYMBOLS + EN_SYMBOLS))
+SYMBOLS = [PAD] + NORMAL_SYMBOLS + PUNCTUATION_SYMBOLS
+SIL_PHONEMES_IDS = [SYMBOLS.index(i) for i in PUNCTUATION_SYMBOLS]
-# combine all tones
-num_tones = num_zh_tones + num_ja_tones + num_en_tones
+# Combine all tones
+NUM_TONES = NUM_ZH_TONES + NUM_JP_TONES + NUM_EN_TONES
-# language maps
-language_id_map = {"ZH": 0, "JP": 1, "EN": 2}
-num_languages = len(language_id_map.keys())
+# Language maps
+LANGUAGE_ID_MAP = {"ZH": 0, "JP": 1, "EN": 2}
+NUM_LANGUAGES = len(LANGUAGE_ID_MAP.keys())
-language_tone_start_map = {
+# Language tone start map
+LANGUAGE_TONE_START_MAP = {
"ZH": 0,
- "JP": num_zh_tones,
- "EN": num_zh_tones + num_ja_tones,
+ "JP": NUM_ZH_TONES,
+ "EN": NUM_ZH_TONES + NUM_JP_TONES,
}
+
if __name__ == "__main__":
- a = set(zh_symbols)
- b = set(en_symbols)
+ a = set(ZH_SYMBOLS)
+ b = set(EN_SYMBOLS)
print(sorted(a & b))
diff --git a/style_bert_vits2/tts_model.py b/style_bert_vits2/tts_model.py
new file mode 100644
index 0000000..da30f48
--- /dev/null
+++ b/style_bert_vits2/tts_model.py
@@ -0,0 +1,435 @@
+import warnings
+from pathlib import Path
+from typing import Any, Optional, Union
+
+import gradio as gr
+import numpy as np
+import pyannote.audio
+import torch
+from gradio.processing_utils import convert_to_16_bit_wav
+from numpy.typing import NDArray
+from pydantic import BaseModel
+
+from style_bert_vits2.constants import (
+ DEFAULT_ASSIST_TEXT_WEIGHT,
+ DEFAULT_LENGTH,
+ DEFAULT_LINE_SPLIT,
+ DEFAULT_NOISE,
+ DEFAULT_NOISEW,
+ DEFAULT_SDP_RATIO,
+ DEFAULT_SPLIT_INTERVAL,
+ DEFAULT_STYLE,
+ DEFAULT_STYLE_WEIGHT,
+ Languages,
+)
+from style_bert_vits2.logging import logger
+from style_bert_vits2.models.hyper_parameters import HyperParameters
+from style_bert_vits2.models.infer import get_net_g, infer
+from style_bert_vits2.models.models import SynthesizerTrn
+from style_bert_vits2.models.models_jp_extra import (
+ SynthesizerTrn as SynthesizerTrnJPExtra,
+)
+from style_bert_vits2.nlp import bert_models
+from style_bert_vits2.voice import adjust_voice
+
+
+class TTSModel:
+ """
+ Style-Bert-Vits2 の音声合成モデルを操作するクラス。
+ モデル/ハイパーパラメータ/スタイルベクトルのパスとデバイスを指定して初期化し、model.infer() メソッドを呼び出すと音声合成を行える。
+ """
+
+ def __init__(
+ self, model_path: Path, config_path: Path, style_vec_path: Path, device: str
+ ) -> None:
+ """
+ Style-Bert-Vits2 の音声合成モデルを初期化する。
+ この時点ではモデルはロードされていない (明示的にロードしたい場合は model.load() を呼び出す)。
+
+ Args:
+ model_path (Path): モデル (.safetensors) のパス
+ config_path (Path): ハイパーパラメータ (config.json) のパス
+ style_vec_path (Path): スタイルベクトル (style_vectors.npy) のパス
+ device (str): 音声合成時に利用するデバイス (cpu, cuda, mps など)
+ """
+
+ self.model_path: Path = model_path
+ self.config_path: Path = config_path
+ self.style_vec_path: Path = style_vec_path
+ self.device: str = device
+ self.hyper_parameters: HyperParameters = HyperParameters.load_from_json(
+ self.config_path
+ )
+ self.spk2id: dict[str, int] = self.hyper_parameters.data.spk2id
+ self.id2spk: dict[int, str] = {v: k for k, v in self.spk2id.items()}
+
+ num_styles: int = self.hyper_parameters.data.num_styles
+ if hasattr(self.hyper_parameters.data, "style2id"):
+ self.style2id: dict[str, int] = self.hyper_parameters.data.style2id
+ else:
+ self.style2id: dict[str, int] = {str(i): i for i in range(num_styles)}
+ if len(self.style2id) != num_styles:
+ raise ValueError(
+ f"Number of styles ({num_styles}) does not match the number of style2id ({len(self.style2id)})"
+ )
+
+ self.__style_vector_inference: Optional[pyannote.audio.Inference] = None
+ self.__style_vectors: NDArray[Any] = np.load(self.style_vec_path)
+ if self.__style_vectors.shape[0] != num_styles:
+ raise ValueError(
+ f"The number of styles ({num_styles}) does not match the number of style vectors ({self.__style_vectors.shape[0]})"
+ )
+
+ self.__net_g: Union[SynthesizerTrn, SynthesizerTrnJPExtra, None] = None
+
+ def load(self) -> None:
+ """
+ 音声合成モデルをデバイスにロードする。
+ """
+ self.__net_g = get_net_g(
+ model_path=str(self.model_path),
+ version=self.hyper_parameters.version,
+ device=self.device,
+ hps=self.hyper_parameters,
+ )
+
+ def __get_style_vector(self, style_id: int, weight: float = 1.0) -> NDArray[Any]:
+ """
+ スタイルベクトルを取得する。
+
+ Args:
+ style_id (int): スタイル ID (0 から始まるインデックス)
+ weight (float, optional): スタイルベクトルの重み. Defaults to 1.0.
+
+ Returns:
+ NDArray[Any]: スタイルベクトル
+ """
+ mean = self.__style_vectors[0]
+ style_vec = self.__style_vectors[style_id]
+ style_vec = mean + (style_vec - mean) * weight
+ return style_vec
+
+ def __get_style_vector_from_audio(
+ self, audio_path: str, weight: float = 1.0
+ ) -> NDArray[Any]:
+ """
+ 音声からスタイルベクトルを推論する。
+
+ Args:
+ audio_path (str): 音声ファイルのパス
+ weight (float, optional): スタイルベクトルの重み. Defaults to 1.0.
+ Returns:
+ NDArray[Any]: スタイルベクトル
+ """
+
+ # スタイルベクトルを取得するための推論モデルを初期化
+ if self.__style_vector_inference is None:
+ self.__style_vector_inference = pyannote.audio.Inference(
+ model=pyannote.audio.Model.from_pretrained(
+ "pyannote/wespeaker-voxceleb-resnet34-LM"
+ ),
+ window="whole",
+ )
+ self.__style_vector_inference.to(torch.device(self.device))
+
+ # 音声からスタイルベクトルを推論
+ xvec = self.__style_vector_inference(audio_path)
+ mean = self.__style_vectors[0]
+ xvec = mean + (xvec - mean) * weight
+ return xvec
+
+ def infer(
+ self,
+ text: str,
+ language: Languages = Languages.JP,
+ speaker_id: int = 0,
+ reference_audio_path: Optional[str] = None,
+ sdp_ratio: float = DEFAULT_SDP_RATIO,
+ noise: float = DEFAULT_NOISE,
+ noise_w: float = DEFAULT_NOISEW,
+ length: float = DEFAULT_LENGTH,
+ line_split: bool = DEFAULT_LINE_SPLIT,
+ split_interval: float = DEFAULT_SPLIT_INTERVAL,
+ assist_text: Optional[str] = None,
+ assist_text_weight: float = DEFAULT_ASSIST_TEXT_WEIGHT,
+ use_assist_text: bool = False,
+ style: str = DEFAULT_STYLE,
+ style_weight: float = DEFAULT_STYLE_WEIGHT,
+ given_tone: Optional[list[int]] = None,
+ pitch_scale: float = 1.0,
+ intonation_scale: float = 1.0,
+ ) -> tuple[int, NDArray[Any]]:
+ """
+ テキストから音声を合成する。
+
+ Args:
+ text (str): 読み上げるテキスト
+ language (Languages, optional): 言語. Defaults to Languages.JP.
+ speaker_id (int, optional): 話者 ID. Defaults to 0.
+ reference_audio_path (Optional[str], optional): 音声スタイルの参照元の音声ファイルのパス. Defaults to None.
+ sdp_ratio (float, optional): DP と SDP の混合比。0 で DP のみ、1で SDP のみを使用 (値を大きくするとテンポに緩急がつく). Defaults to DEFAULT_SDP_RATIO.
+ noise (float, optional): DP に与えられるノイズ. Defaults to DEFAULT_NOISE.
+ noise_w (float, optional): SDP に与えられるノイズ. Defaults to DEFAULT_NOISEW.
+ length (float, optional): 生成音声の長さ（話速）のパラメータ。大きいほど生成音声が長くゆっくり、小さいほど短く早くなる。 Defaults to DEFAULT_LENGTH.
+ line_split (bool, optional): テキストを改行ごとに分割して生成するかどうか. Defaults to DEFAULT_LINE_SPLIT.
+ split_interval (float, optional): 改行ごとに分割する場合の無音 (秒). Defaults to DEFAULT_SPLIT_INTERVAL.
+ assist_text (Optional[str], optional): 感情表現の参照元の補助テキスト. Defaults to None.
+ assist_text_weight (float, optional): 感情表現の補助テキストを適用する強さ. Defaults to DEFAULT_ASSIST_TEXT_WEIGHT.
+ use_assist_text (bool, optional): 音声合成時に感情表現の補助テキストを使用するかどうか. Defaults to False.
+ style (str, optional): 音声スタイル (Neutral, Happy など). Defaults to DEFAULT_STYLE.
+ style_weight (float, optional): 音声スタイルを適用する強さ. Defaults to DEFAULT_STYLE_WEIGHT.
+ given_tone (Optional[list[int]], optional): アクセントのトーンのリスト. Defaults to None.
+ pitch_scale (float, optional): ピッチの高さ (1.0 から変更すると若干音質が低下する). Defaults to 1.0.
+ intonation_scale (float, optional): イントネーションの高さ (1.0 から変更すると若干音質が低下する). Defaults to 1.0.
+
+ Returns:
+ tuple[int, NDArray[Any]]: サンプリングレートと音声データ (16bit PCM)
+ """
+
+ logger.info(f"Start generating audio data from text:\n{text}")
+ if language != "JP" and self.hyper_parameters.version.endswith("JP-Extra"):
+ raise ValueError(
+ "The model is trained with JP-Extra, but the language is not JP"
+ )
+ if reference_audio_path == "":
+ reference_audio_path = None
+ if assist_text == "" or not use_assist_text:
+ assist_text = None
+
+ if self.__net_g is None:
+ self.load()
+ assert self.__net_g is not None
+ if reference_audio_path is None:
+ style_id = self.style2id[style]
+ style_vector = self.__get_style_vector(style_id, style_weight)
+ else:
+ style_vector = self.__get_style_vector_from_audio(
+ reference_audio_path, style_weight
+ )
+ if not line_split:
+ with torch.no_grad():
+ audio = infer(
+ text=text,
+ sdp_ratio=sdp_ratio,
+ noise_scale=noise,
+ noise_scale_w=noise_w,
+ length_scale=length,
+ sid=speaker_id,
+ language=language,
+ hps=self.hyper_parameters,
+ net_g=self.__net_g,
+ device=self.device,
+ assist_text=assist_text,
+ assist_text_weight=assist_text_weight,
+ style_vec=style_vector,
+ given_tone=given_tone,
+ )
+ else:
+ texts = text.split("\n")
+ texts = [t for t in texts if t != ""]
+ audios = []
+ with torch.no_grad():
+ for i, t in enumerate(texts):
+ audios.append(
+ infer(
+ text=t,
+ sdp_ratio=sdp_ratio,
+ noise_scale=noise,
+ noise_scale_w=noise_w,
+ length_scale=length,
+ sid=speaker_id,
+ language=language,
+ hps=self.hyper_parameters,
+ net_g=self.__net_g,
+ device=self.device,
+ assist_text=assist_text,
+ assist_text_weight=assist_text_weight,
+ style_vec=style_vector,
+ )
+ )
+ if i != len(texts) - 1:
+ audios.append(np.zeros(int(44100 * split_interval)))
+ audio = np.concatenate(audios)
+ logger.info("Audio data generated successfully")
+ if not (pitch_scale == 1.0 and intonation_scale == 1.0):
+ _, audio = adjust_voice(
+ fs=self.hyper_parameters.data.sampling_rate,
+ wave=audio,
+ pitch_scale=pitch_scale,
+ intonation_scale=intonation_scale,
+ )
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ audio = convert_to_16_bit_wav(audio)
+ return (self.hyper_parameters.data.sampling_rate, audio)
+
+
+class TTSModelInfo(BaseModel):
+ name: str
+ files: list[str]
+ styles: list[str]
+ speakers: list[str]
+
+
+class TTSModelHolder:
+ """
+ Style-Bert-Vits2 の音声合成モデルを管理するクラス。
+ model_holder.models_info から指定されたディレクトリ内にある音声合成モデルの一覧を取得できる。
+ """
+
+ def __init__(self, model_root_dir: Path, device: str) -> None:
+ """
+ Style-Bert-Vits2 の音声合成モデルを管理するクラスを初期化する。
+ 音声合成モデルは下記のように配置されていることを前提とする (.safetensors のファイル名は自由) 。
+ ```
+ model_root_dir
+ ├── model-name-1
+ │ ├── config.json
+ │ ├── model-name-1_e160_s14000.safetensors
+ │ └── style_vectors.npy
+ ├── model-name-2
+ │ ├── config.json
+ │ ├── model-name-2_e160_s14000.safetensors
+ │ └── style_vectors.npy
+ └── ...
+ ```
+
+ Args:
+ model_root_dir (Path): 音声合成モデルが配置されているディレクトリのパス
+ device (str): 音声合成時に利用するデバイス (cpu, cuda, mps など)
+ """
+
+ self.root_dir: Path = model_root_dir
+ self.device: str = device
+ self.model_files_dict: dict[str, list[Path]] = {}
+ self.current_model: Optional[TTSModel] = None
+ self.model_names: list[str] = []
+ self.models_info: list[TTSModelInfo] = []
+ self.refresh()
+
+ def refresh(self) -> None:
+ """
+ 音声合成モデルの一覧を更新する。
+ """
+
+ self.model_files_dict = {}
+ self.model_names = []
+ self.current_model = None
+ self.models_info = []
+
+ model_dirs = [d for d in self.root_dir.iterdir() if d.is_dir()]
+ for model_dir in model_dirs:
+ model_files = [
+ f
+ for f in model_dir.iterdir()
+ if f.suffix in [".pth", ".pt", ".safetensors"]
+ ]
+ if len(model_files) == 0:
+ logger.warning(f"No model files found in {model_dir}, so skip it")
+ continue
+ config_path = model_dir / "config.json"
+ if not config_path.exists():
+ logger.warning(
+ f"Config file {config_path} not found, so skip {model_dir}"
+ )
+ continue
+ self.model_files_dict[model_dir.name] = model_files
+ self.model_names.append(model_dir.name)
+ hyper_parameters = HyperParameters.load_from_json(config_path)
+ style2id: dict[str, int] = hyper_parameters.data.style2id
+ styles = list(style2id.keys())
+ spk2id: dict[str, int] = hyper_parameters.data.spk2id
+ speakers = list(spk2id.keys())
+ self.models_info.append(
+ TTSModelInfo(
+ name=model_dir.name,
+ files=[str(f) for f in model_files],
+ styles=styles,
+ speakers=speakers,
+ )
+ )
+
+ def get_model(self, model_name: str, model_path_str: str) -> TTSModel:
+ """
+ 指定された音声合成モデルのインスタンスを取得する。
+ この時点ではモデルはロードされていない (明示的にロードしたい場合は model.load() を呼び出す)。
+
+ Args:
+ model_name (str): 音声合成モデルの名前
+ model_path_str (str): 音声合成モデルのファイルパス (.safetensors)
+
+ Returns:
+ TTSModel: 音声合成モデルのインスタンス
+ """
+
+ model_path = Path(model_path_str)
+ if model_name not in self.model_files_dict:
+ raise ValueError(f"Model `{model_name}` is not found")
+ if model_path not in self.model_files_dict[model_name]:
+ raise ValueError(f"Model file `{model_path}` is not found")
+ if self.current_model is None or self.current_model.model_path != model_path:
+ self.current_model = TTSModel(
+ model_path=model_path,
+ config_path=self.root_dir / model_name / "config.json",
+ style_vec_path=self.root_dir / model_name / "style_vectors.npy",
+ device=self.device,
+ )
+
+ return self.current_model
+
+ def get_model_for_gradio(
+ self, model_name: str, model_path_str: str
+ ) -> tuple[gr.Dropdown, gr.Button, gr.Dropdown]:
+ bert_models.load_model(Languages.JP)
+ bert_models.load_tokenizer(Languages.JP)
+ bert_models.load_model(Languages.EN)
+ bert_models.load_tokenizer(Languages.EN)
+ bert_models.load_model(Languages.ZH)
+ bert_models.load_tokenizer(Languages.ZH)
+
+ model_path = Path(model_path_str)
+ if model_name not in self.model_files_dict:
+ raise ValueError(f"Model `{model_name}` is not found")
+ if model_path not in self.model_files_dict[model_name]:
+ raise ValueError(f"Model file `{model_path}` is not found")
+ if (
+ self.current_model is not None
+ and self.current_model.model_path == model_path
+ ):
+ # Already loaded
+ speakers = list(self.current_model.spk2id.keys())
+ styles = list(self.current_model.style2id.keys())
+ return (
+ gr.Dropdown(choices=styles, value=styles[0]), # type: ignore
+ gr.Button(interactive=True, value="音声合成"),
+ gr.Dropdown(choices=speakers, value=speakers[0]), # type: ignore
+ )
+ self.current_model = TTSModel(
+ model_path=model_path,
+ config_path=self.root_dir / model_name / "config.json",
+ style_vec_path=self.root_dir / model_name / "style_vectors.npy",
+ device=self.device,
+ )
+ speakers = list(self.current_model.spk2id.keys())
+ styles = list(self.current_model.style2id.keys())
+ return (
+ gr.Dropdown(choices=styles, value=styles[0]), # type: ignore
+ gr.Button(interactive=True, value="音声合成"),
+ gr.Dropdown(choices=speakers, value=speakers[0]), # type: ignore
+ )
+
+ def update_model_files_for_gradio(self, model_name: str) -> gr.Dropdown:
+ model_files = self.model_files_dict[model_name]
+ return gr.Dropdown(choices=model_files, value=model_files[0]) # type: ignore
+
+ def update_model_names_for_gradio(
+ self,
+ ) -> tuple[gr.Dropdown, gr.Dropdown, gr.Button]:
+ self.refresh()
+ initial_model_name = self.model_names[0]
+ initial_model_files = self.model_files_dict[initial_model_name]
+ return (
+ gr.Dropdown(choices=self.model_names, value=initial_model_name), # type: ignore
+ gr.Dropdown(choices=initial_model_files, value=initial_model_files[0]), # type: ignore
+ gr.Button(interactive=False), # For tts_button
+ )
diff --git a/style_bert_vits2/utils/__init__.py b/style_bert_vits2/utils/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/common/stdout_wrapper.py b/style_bert_vits2/utils/stdout_wrapper.py
similarity index 50%
rename from common/stdout_wrapper.py
rename to style_bert_vits2/utils/stdout_wrapper.py
index 192f908..174d1fd 100644
--- a/common/stdout_wrapper.py
+++ b/style_bert_vits2/utils/stdout_wrapper.py
@@ -1,39 +1,41 @@
-"""
-`sys.stdout` wrapper for both Google Colab and local environment.
-"""
-
import sys
import tempfile
+from typing import TextIO
-class StdoutWrapper:
- def __init__(self):
+class StdoutWrapper(TextIO):
+ """
+ `sys.stdout` wrapper for both Google Colab and local environment.
+ """
+
+ def __init__(self) -> None:
self.temp_file = tempfile.NamedTemporaryFile(
mode="w+", delete=False, encoding="utf-8"
)
self.original_stdout = sys.stdout
- def write(self, message: str):
- self.temp_file.write(message)
+ def write(self, message: str) -> int:
+ result = self.temp_file.write(message)
self.temp_file.flush()
print(message, end="", file=self.original_stdout)
+ return result
- def flush(self):
+ def flush(self) -> None:
self.temp_file.flush()
- def read(self):
+ def read(self, n: int = -1) -> str:
self.temp_file.seek(0)
- return self.temp_file.read()
+ return self.temp_file.read(n)
- def close(self):
+ def close(self) -> None:
self.temp_file.close()
- def fileno(self):
+ def fileno(self) -> int:
return self.temp_file.fileno()
try:
- import google.colab
+ import google.colab # type: ignore
SAFE_STDOUT = StdoutWrapper()
except ImportError:
diff --git a/style_bert_vits2/utils/strenum.py b/style_bert_vits2/utils/strenum.py
new file mode 100644
index 0000000..b2e2e3c
--- /dev/null
+++ b/style_bert_vits2/utils/strenum.py
@@ -0,0 +1,37 @@
+import enum
+
+
+class StrEnum(str, enum.Enum):
+ """
+ Enum where members are also (and must be) strings (backported from Python 3.11).
+ """
+
+ def __new__(cls, *values: str) -> "StrEnum":
+ "values must already be of type `str`"
+ if len(values) > 3:
+ raise TypeError("too many arguments for str(): %r" % (values,))
+ if len(values) == 1:
+ # it must be a string
+ if not isinstance(values[0], str): # type: ignore
+ raise TypeError("%r is not a string" % (values[0],))
+ if len(values) >= 2:
+ # check that encoding argument is a string
+ if not isinstance(values[1], str): # type: ignore
+ raise TypeError("encoding must be a string, not %r" % (values[1],))
+ if len(values) == 3:
+ # check that errors argument is a string
+ if not isinstance(values[2], str): # type: ignore
+ raise TypeError("errors must be a string, not %r" % (values[2]))
+ value = str(*values)
+ member = str.__new__(cls, value)
+ member._value_ = value
+ return member
+
+ @staticmethod
+ def _generate_next_value_(
+ name: str, start: int, count: int, last_values: list[str]
+ ) -> str:
+ """
+ Return the lower-cased version of the member name.
+ """
+ return name.lower()
diff --git a/style_bert_vits2/utils/subprocess.py b/style_bert_vits2/utils/subprocess.py
new file mode 100644
index 0000000..f8e8e94
--- /dev/null
+++ b/style_bert_vits2/utils/subprocess.py
@@ -0,0 +1,58 @@
+import subprocess
+import sys
+from typing import Any, Callable
+
+from style_bert_vits2.logging import logger
+from style_bert_vits2.utils.stdout_wrapper import SAFE_STDOUT
+
+
+def run_script_with_log(
+ cmd: list[str], ignore_warning: bool = False
+) -> tuple[bool, str]:
+ """
+ 指定されたコマンドを実行し、そのログを記録する。
+
+ Args:
+ cmd: 実行するコマンドのリスト
+ ignore_warning: 警告を無視するかどうかのフラグ
+
+ Returns:
+ tuple[bool, str]: 実行が成功したかどうかのブール値と、エラーまたは警告のメッセージ（ある場合）
+ """
+
+ logger.info(f"Running: {' '.join(cmd)}")
+ result = subprocess.run(
+ [sys.executable] + cmd,
+ stdout=SAFE_STDOUT,
+ stderr=subprocess.PIPE,
+ text=True,
+ encoding="utf-8",
+ )
+ if result.returncode != 0:
+ logger.error(f"Error: {' '.join(cmd)}\n{result.stderr}")
+ return False, result.stderr
+ elif result.stderr and not ignore_warning:
+ logger.warning(f"Warning: {' '.join(cmd)}\n{result.stderr}")
+ return True, result.stderr
+ logger.success(f"Success: {' '.join(cmd)}")
+
+ return True, ""
+
+
+def second_elem_of(
+ original_function: Callable[..., tuple[Any, Any]]
+) -> Callable[..., Any]:
+ """
+ 与えられた関数をラップし、その戻り値の 2 番目の要素のみを返す関数を生成する。
+
+ Args:
+ original_function (Callable[..., tuple[Any, Any]])): ラップする元の関数
+
+ Returns:
+ Callable[..., Any]: 元の関数の戻り値の 2 番目の要素のみを返す関数
+ """
+
+ def inner_function(*args, **kwargs) -> Any: # type: ignore
+ return original_function(*args, **kwargs)[1]
+
+ return inner_function
diff --git a/style_bert_vits2/voice.py b/style_bert_vits2/voice.py
new file mode 100644
index 0000000..75f7d51
--- /dev/null
+++ b/style_bert_vits2/voice.py
@@ -0,0 +1,52 @@
+from typing import Any
+
+import numpy as np
+import pyworld
+from numpy.typing import NDArray
+
+
+def adjust_voice(
+ fs: int,
+ wave: NDArray[Any],
+ pitch_scale: float = 1.0,
+ intonation_scale: float = 1.0,
+) -> tuple[int, NDArray[Any]]:
+ """
+ 音声のピッチとイントネーションを調整する。
+ 変更すると若干音質が劣化するので、どちらも初期値のままならそのまま返す。
+
+ Args:
+ fs (int): 音声のサンプリング周波数
+ wave (NDArray[Any]): 音声データ
+ pitch_scale (float, optional): ピッチの高さ. Defaults to 1.0.
+ intonation_scale (float, optional): イントネーションの平均からの変更比率. Defaults to 1.0.
+
+ Returns:
+ tuple[int, NDArray[Any]]: 調整後の音声データのサンプリング周波数と音声データ
+ """
+
+ if pitch_scale == 1.0 and intonation_scale == 1.0:
+ # 初期値の場合は、音質劣化を避けるためにそのまま返す
+ return fs, wave
+
+ # pyworld で f0 を加工して合成
+ # pyworld よりもよいのがあるかもしれないが……
+
+ wave = wave.astype(np.double)
+
+ # 質が高そうだしとりあえずharvestにしておく
+ f0, t = pyworld.harvest(wave, fs)
+
+ sp = pyworld.cheaptrick(wave, f0, t, fs)
+ ap = pyworld.d4c(wave, f0, t, fs)
+
+ non_zero_f0 = [f for f in f0 if f != 0]
+ f0_mean = sum(non_zero_f0) / len(non_zero_f0)
+
+ for i, f in enumerate(f0):
+ if f == 0:
+ continue
+ f0[i] = pitch_scale * f0_mean + intonation_scale * (f - f0_mean)
+
+ wave = pyworld.synthesize(f0, sp, ap, fs)
+ return fs, wave
diff --git a/style_gen.py b/style_gen.py
index 97a0aee..02af067 100644
--- a/style_gen.py
+++ b/style_gen.py
@@ -1,18 +1,19 @@
import argparse
-from concurrent.futures import ThreadPoolExecutor
import warnings
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any
import numpy as np
import torch
+from numpy.typing import NDArray
+from pyannote.audio import Inference, Model
from tqdm import tqdm
-import utils
-from common.log import logger
-from common.stdout_wrapper import SAFE_STDOUT
from config import config
+from style_bert_vits2.logging import logger
+from style_bert_vits2.models.hyper_parameters import HyperParameters
+from style_bert_vits2.utils.stdout_wrapper import SAFE_STDOUT
-warnings.filterwarnings("ignore", category=UserWarning)
-from pyannote.audio import Inference, Model
model = Model.from_pretrained("pyannote/wespeaker-voxceleb-resnet34-LM")
inference = Inference(model, window="whole")
@@ -27,11 +28,11 @@ class NaNValueError(ValueError):
# 推論時にインポートするために短いが関数を書く
-def get_style_vector(wav_path):
- return inference(wav_path)
+def get_style_vector(wav_path: str) -> NDArray[Any]:
+ return inference(wav_path) # type: ignore
-def save_style_vector(wav_path):
+def save_style_vector(wav_path: str):
try:
style_vec = get_style_vector(wav_path)
except Exception as e:
@@ -46,20 +47,15 @@ def save_style_vector(wav_path):
np.save(f"{wav_path}.npy", style_vec) # `test.wav` -> `test.wav.npy`
-def process_line(line):
- wavname = line.split("|")[0]
+def process_line(line: str):
+ wav_path = line.split("|")[0]
try:
- save_style_vector(wavname)
+ save_style_vector(wav_path)
return line, None
except NaNValueError:
return line, "nan_error"
-def save_average_style_vector(style_vectors, filename="style_vectors.npy"):
- average_vector = np.mean(style_vectors, axis=0)
- np.save(filename, average_vector)
-
-
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
@@ -69,15 +65,15 @@ if __name__ == "__main__":
"--num_processes", type=int, default=config.style_gen_config.num_processes
)
args, _ = parser.parse_known_args()
- config_path = args.config
- num_processes = args.num_processes
+ config_path: str = args.config
+ num_processes: int = args.num_processes
- hps = utils.get_hparams_from_file(config_path)
+ hps = HyperParameters.load_from_json(config_path)
device = config.style_gen_config.device
- training_lines = []
- with open(hps.data.training_files, encoding="utf-8") as f:
+ training_lines: list[str] = []
+ with open(hps.data.training_files, "r", encoding="utf-8") as f:
training_lines.extend(f.readlines())
with ThreadPoolExecutor(max_workers=num_processes) as executor:
training_results = list(
@@ -97,8 +93,8 @@ if __name__ == "__main__":
f"Found NaN value in {len(nan_training_lines)} files: {nan_files}, so they will be deleted from training data."
)
- val_lines = []
- with open(hps.data.validation_files, encoding="utf-8") as f:
+ val_lines: list[str] = []
+ with open(hps.data.validation_files, "r", encoding="utf-8") as f:
val_lines.extend(f.readlines())
with ThreadPoolExecutor(max_workers=num_processes) as executor:
diff --git a/tests/.gitignore b/tests/.gitignore
new file mode 100644
index 0000000..697e56f
--- /dev/null
+++ b/tests/.gitignore
@@ -0,0 +1 @@
+*.wav
\ No newline at end of file
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/test_main.py b/tests/test_main.py
new file mode 100644
index 0000000..e2e6530
--- /dev/null
+++ b/tests/test_main.py
@@ -0,0 +1,56 @@
+import pytest
+from scipy.io import wavfile
+
+from style_bert_vits2.constants import BASE_DIR, Languages
+from style_bert_vits2.tts_model import TTSModelHolder
+
+
+def synthesize(device: str = "cpu"):
+
+ # 音声合成モデルが配置されていれば、音声合成を実行
+ model_holder = TTSModelHolder(BASE_DIR / "model_assets", device)
+ if len(model_holder.models_info) > 0:
+
+ # jvnv-F2-jp モデルを探す
+ for model_info in model_holder.models_info:
+ if model_info.name == "jvnv-F2-jp":
+ # すべてのスタイルに対して音声合成を実行
+ for style in model_info.styles:
+
+ # 音声合成を実行
+ model = model_holder.get_model(model_info.name, model_info.files[0])
+ model.load()
+ sample_rate, audio_data = model.infer(
+ "あらゆる現実を、すべて自分のほうへねじ曲げたのだ。",
+ # 言語 (JP, EN, ZH / JP-Extra モデルの場合は JP のみ)
+ language=Languages.JP,
+ # 話者 ID (音声合成モデルに複数の話者が含まれる場合のみ必須、単一話者のみの場合は 0)
+ speaker_id=0,
+ # 感情表現の強さ (0.0 〜 1.0)
+ sdp_ratio=0.4,
+ # スタイル (Neutral, Happy など)
+ style=style,
+ # スタイルの強さ (0.0 〜 100.0)
+ style_weight=6.0,
+ )
+
+ # 音声データを保存
+ (BASE_DIR / "tests/wavs").mkdir(exist_ok=True, parents=True)
+ wav_file_path = BASE_DIR / f"tests/wavs/{style}.wav"
+ with open(wav_file_path, "wb") as f:
+ wavfile.write(f, sample_rate, audio_data)
+
+ # 音声データが保存されたことを確認
+ assert wav_file_path.exists()
+ # wav_file_path.unlink()
+ else:
+ pytest.skip("音声合成モデルが見つかりませんでした。")
+
+
+def test_synthesize_cpu():
+ synthesize(device="cpu")
+
+
+# Windows環境ではtorchのcudaが簡単に入らないため、テストをスキップ
+# def test_synthesize_cuda():
+# synthesize(device="cuda")
diff --git a/text/__init__.py b/text/__init__.py
deleted file mode 100644
index 7ba6e20..0000000
--- a/text/__init__.py
+++ /dev/null
@@ -1,18 +0,0 @@
-from text.symbols import *
-
-_symbol_to_id = {s: i for i, s in enumerate(symbols)}
-
-
-def cleaned_text_to_sequence(cleaned_text, tones, language):
- """Converts a string of text to a sequence of IDs corresponding to the symbols in the text.
- Args:
- text: string to convert to a sequence
- Returns:
- List of integers corresponding to the symbols in the text
- """
- phones = [_symbol_to_id[symbol] for symbol in cleaned_text]
- tone_start = language_tone_start_map[language]
- tones = [i + tone_start for i in tones]
- lang_id = language_id_map[language]
- lang_ids = [lang_id for i in phones]
- return phones, tones, lang_ids
diff --git a/text/cleaner.py b/text/cleaner.py
deleted file mode 100644
index d805b51..0000000
--- a/text/cleaner.py
+++ /dev/null
@@ -1,26 +0,0 @@
-def clean_text(text, language, use_jp_extra=True, raise_yomi_error=False):
- # Changed to import inside if condition to avoid unnecessary import
- if language == "ZH":
- from . import chinese as language_module
-
- norm_text = language_module.text_normalize(text)
- phones, tones, word2ph = language_module.g2p(norm_text)
- elif language == "EN":
- from . import english as language_module
-
- norm_text = language_module.text_normalize(text)
- phones, tones, word2ph = language_module.g2p(norm_text)
- elif language == "JP":
- from . import japanese as language_module
-
- norm_text = language_module.text_normalize(text)
- phones, tones, word2ph = language_module.g2p(
- norm_text, use_jp_extra, raise_yomi_error=raise_yomi_error
- )
- else:
- raise ValueError(f"Language {language} not supported")
- return norm_text, phones, tones, word2ph
-
-
-if __name__ == "__main__":
- pass
diff --git a/text/english.py b/text/english.py
deleted file mode 100644
index 4a2af95..0000000
--- a/text/english.py
+++ /dev/null
@@ -1,495 +0,0 @@
-import pickle
-import os
-import re
-from g2p_en import G2p
-from transformers import DebertaV2Tokenizer
-
-from text import symbols
-from text.symbols import punctuation
-
-current_file_path = os.path.dirname(__file__)
-CMU_DICT_PATH = os.path.join(current_file_path, "cmudict.rep")
-CACHE_PATH = os.path.join(current_file_path, "cmudict_cache.pickle")
-_g2p = G2p()
-LOCAL_PATH = "./bert/deberta-v3-large"
-tokenizer = DebertaV2Tokenizer.from_pretrained(LOCAL_PATH)
-
-arpa = {
- "AH0",
- "S",
- "AH1",
- "EY2",
- "AE2",
- "EH0",
- "OW2",
- "UH0",
- "NG",
- "B",
- "G",
- "AY0",
- "M",
- "AA0",
- "F",
- "AO0",
- "ER2",
- "UH1",
- "IY1",
- "AH2",
- "DH",
- "IY0",
- "EY1",
- "IH0",
- "K",
- "N",
- "W",
- "IY2",
- "T",
- "AA1",
- "ER1",
- "EH2",
- "OY0",
- "UH2",
- "UW1",
- "Z",
- "AW2",
- "AW1",
- "V",
- "UW2",
- "AA2",
- "ER",
- "AW0",
- "UW0",
- "R",
- "OW1",
- "EH1",
- "ZH",
- "AE0",
- "IH2",
- "IH",
- "Y",
- "JH",
- "P",
- "AY1",
- "EY0",
- "OY2",
- "TH",
- "HH",
- "D",
- "ER0",
- "CH",
- "AO1",
- "AE1",
- "AO2",
- "OY1",
- "AY2",
- "IH1",
- "OW0",
- "L",
- "SH",
-}
-
-
-def post_replace_ph(ph):
- rep_map = {
- "：": ",",
- "；": ",",
- "，": ",",
- "。": ".",
- "！": "!",
- "？": "?",
- "\n": ".",
- "·": ",",
- "、": ",",
- "…": "...",
- "···": "...",
- "・・・": "...",
- "v": "V",
- }
- if ph in rep_map.keys():
- ph = rep_map[ph]
- if ph in symbols:
- return ph
- if ph not in symbols:
- ph = "UNK"
- return ph
-
-
-rep_map = {
- "：": ",",
- "；": ",",
- "，": ",",
- "。": ".",
- "！": "!",
- "？": "?",
- "\n": ".",
- "．": ".",
- "…": "...",
- "···": "...",
- "・・・": "...",
- "·": ",",
- "・": ",",
- "、": ",",
- "$": ".",
- "“": "'",
- "”": "'",
- '"': "'",
- "‘": "'",
- "’": "'",
- "（": "'",
- "）": "'",
- "(": "'",
- ")": "'",
- "《": "'",
- "》": "'",
- "【": "'",
- "】": "'",
- "[": "'",
- "]": "'",
- "—": "-",
- "−": "-",
- "～": "-",
- "~": "-",
- "「": "'",
- "」": "'",
-}
-
-
-def replace_punctuation(text):
- pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys()))
-
- replaced_text = pattern.sub(lambda x: rep_map[x.group()], text)
-
- # replaced_text = re.sub(
- # r"[^\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\u3400-\u4DBF\u3005"
- # + "".join(punctuation)
- # + r"]+",
- # "",
- # replaced_text,
- # )
-
- return replaced_text
-
-
-def read_dict():
- g2p_dict = {}
- start_line = 49
- with open(CMU_DICT_PATH) as f:
- line = f.readline()
- line_index = 1
- while line:
- if line_index >= start_line:
- line = line.strip()
- word_split = line.split(" ")
- word = word_split[0]
-
- syllable_split = word_split[1].split(" - ")
- g2p_dict[word] = []
- for syllable in syllable_split:
- phone_split = syllable.split(" ")
- g2p_dict[word].append(phone_split)
-
- line_index = line_index + 1
- line = f.readline()
-
- return g2p_dict
-
-
-def cache_dict(g2p_dict, file_path):
- with open(file_path, "wb") as pickle_file:
- pickle.dump(g2p_dict, pickle_file)
-
-
-def get_dict():
- if os.path.exists(CACHE_PATH):
- with open(CACHE_PATH, "rb") as pickle_file:
- g2p_dict = pickle.load(pickle_file)
- else:
- g2p_dict = read_dict()
- cache_dict(g2p_dict, CACHE_PATH)
-
- return g2p_dict
-
-
-eng_dict = get_dict()
-
-
-def refine_ph(phn):
- tone = 0
- if re.search(r"\d$", phn):
- tone = int(phn[-1]) + 1
- phn = phn[:-1]
- else:
- tone = 3
- return phn.lower(), tone
-
-
-def refine_syllables(syllables):
- tones = []
- phonemes = []
- for phn_list in syllables:
- for i in range(len(phn_list)):
- phn = phn_list[i]
- phn, tone = refine_ph(phn)
- phonemes.append(phn)
- tones.append(tone)
- return phonemes, tones
-
-
-import re
-import inflect
-
-_inflect = inflect.engine()
-_comma_number_re = re.compile(r"([0-9][0-9\,]+[0-9])")
-_decimal_number_re = re.compile(r"([0-9]+\.[0-9]+)")
-_pounds_re = re.compile(r"£([0-9\,]*[0-9]+)")
-_dollars_re = re.compile(r"\$([0-9\.\,]*[0-9]+)")
-_ordinal_re = re.compile(r"[0-9]+(st|nd|rd|th)")
-_number_re = re.compile(r"[0-9]+")
-
-# List of (regular expression, replacement) pairs for abbreviations:
-_abbreviations = [
- (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
- for x in [
- ("mrs", "misess"),
- ("mr", "mister"),
- ("dr", "doctor"),
- ("st", "saint"),
- ("co", "company"),
- ("jr", "junior"),
- ("maj", "major"),
- ("gen", "general"),
- ("drs", "doctors"),
- ("rev", "reverend"),
- ("lt", "lieutenant"),
- ("hon", "honorable"),
- ("sgt", "sergeant"),
- ("capt", "captain"),
- ("esq", "esquire"),
- ("ltd", "limited"),
- ("col", "colonel"),
- ("ft", "fort"),
- ]
-]
-
-
-# List of (ipa, lazy ipa) pairs:
-_lazy_ipa = [
- (re.compile("%s" % x[0]), x[1])
- for x in [
- ("r", "ɹ"),
- ("æ", "e"),
- ("ɑ", "a"),
- ("ɔ", "o"),
- ("ð", "z"),
- ("θ", "s"),
- ("ɛ", "e"),
- ("ɪ", "i"),
- ("ʊ", "u"),
- ("ʒ", "ʥ"),
- ("ʤ", "ʥ"),
- ("ˈ", "↓"),
- ]
-]
-
-# List of (ipa, lazy ipa2) pairs:
-_lazy_ipa2 = [
- (re.compile("%s" % x[0]), x[1])
- for x in [
- ("r", "ɹ"),
- ("ð", "z"),
- ("θ", "s"),
- ("ʒ", "ʑ"),
- ("ʤ", "dʑ"),
- ("ˈ", "↓"),
- ]
-]
-
-# List of (ipa, ipa2) pairs
-_ipa_to_ipa2 = [
- (re.compile("%s" % x[0]), x[1]) for x in [("r", "ɹ"), ("ʤ", "dʒ"), ("ʧ", "tʃ")]
-]
-
-
-def _expand_dollars(m):
- match = m.group(1)
- parts = match.split(".")
- if len(parts) > 2:
- return match + " dollars" # Unexpected format
- dollars = int(parts[0]) if parts[0] else 0
- cents = int(parts[1]) if len(parts) > 1 and parts[1] else 0
- if dollars and cents:
- dollar_unit = "dollar" if dollars == 1 else "dollars"
- cent_unit = "cent" if cents == 1 else "cents"
- return "%s %s, %s %s" % (dollars, dollar_unit, cents, cent_unit)
- elif dollars:
- dollar_unit = "dollar" if dollars == 1 else "dollars"
- return "%s %s" % (dollars, dollar_unit)
- elif cents:
- cent_unit = "cent" if cents == 1 else "cents"
- return "%s %s" % (cents, cent_unit)
- else:
- return "zero dollars"
-
-
-def _remove_commas(m):
- return m.group(1).replace(",", "")
-
-
-def _expand_ordinal(m):
- return _inflect.number_to_words(m.group(0))
-
-
-def _expand_number(m):
- num = int(m.group(0))
- if num > 1000 and num < 3000:
- if num == 2000:
- return "two thousand"
- elif num > 2000 and num < 2010:
- return "two thousand " + _inflect.number_to_words(num % 100)
- elif num % 100 == 0:
- return _inflect.number_to_words(num // 100) + " hundred"
- else:
- return _inflect.number_to_words(
- num, andword="", zero="oh", group=2
- ).replace(", ", " ")
- else:
- return _inflect.number_to_words(num, andword="")
-
-
-def _expand_decimal_point(m):
- return m.group(1).replace(".", " point ")
-
-
-def normalize_numbers(text):
- text = re.sub(_comma_number_re, _remove_commas, text)
- text = re.sub(_pounds_re, r"\1 pounds", text)
- text = re.sub(_dollars_re, _expand_dollars, text)
- text = re.sub(_decimal_number_re, _expand_decimal_point, text)
- text = re.sub(_ordinal_re, _expand_ordinal, text)
- text = re.sub(_number_re, _expand_number, text)
- return text
-
-
-def text_normalize(text):
- text = normalize_numbers(text)
- text = replace_punctuation(text)
- text = re.sub(r"([,;.\?\!])([\w])", r"\1 \2", text)
- return text
-
-
-def distribute_phone(n_phone, n_word):
- phones_per_word = [0] * n_word
- for task in range(n_phone):
- min_tasks = min(phones_per_word)
- min_index = phones_per_word.index(min_tasks)
- phones_per_word[min_index] += 1
- return phones_per_word
-
-
-def sep_text(text):
- words = re.split(r"([,;.\?\!\s+])", text)
- words = [word for word in words if word.strip() != ""]
- return words
-
-
-def text_to_words(text):
- tokens = tokenizer.tokenize(text)
- words = []
- for idx, t in enumerate(tokens):
- if t.startswith("▁"):
- words.append([t[1:]])
- else:
- if t in punctuation:
- if idx == len(tokens) - 1:
- words.append([f"{t}"])
- else:
- if (
- not tokens[idx + 1].startswith("▁")
- and tokens[idx + 1] not in punctuation
- ):
- if idx == 0:
- words.append([])
- words[-1].append(f"{t}")
- else:
- words.append([f"{t}"])
- else:
- if idx == 0:
- words.append([])
- words[-1].append(f"{t}")
- return words
-
-
-def g2p(text):
- phones = []
- tones = []
- phone_len = []
- # words = sep_text(text)
- # tokens = [tokenizer.tokenize(i) for i in words]
- words = text_to_words(text)
-
- for word in words:
- temp_phones, temp_tones = [], []
- if len(word) > 1:
- if "'" in word:
- word = ["".join(word)]
- for w in word:
- if w in punctuation:
- temp_phones.append(w)
- temp_tones.append(0)
- continue
- if w.upper() in eng_dict:
- phns, tns = refine_syllables(eng_dict[w.upper()])
- temp_phones += [post_replace_ph(i) for i in phns]
- temp_tones += tns
- # w2ph.append(len(phns))
- else:
- phone_list = list(filter(lambda p: p != " ", _g2p(w)))
- phns = []
- tns = []
- for ph in phone_list:
- if ph in arpa:
- ph, tn = refine_ph(ph)
- phns.append(ph)
- tns.append(tn)
- else:
- phns.append(ph)
- tns.append(0)
- temp_phones += [post_replace_ph(i) for i in phns]
- temp_tones += tns
- phones += temp_phones
- tones += temp_tones
- phone_len.append(len(temp_phones))
- # phones = [post_replace_ph(i) for i in phones]
-
- word2ph = []
- for token, pl in zip(words, phone_len):
- word_len = len(token)
-
- aaa = distribute_phone(pl, word_len)
- word2ph += aaa
-
- phones = ["_"] + phones + ["_"]
- tones = [0] + tones + [0]
- word2ph = [1] + word2ph + [1]
- assert len(phones) == len(tones), text
- assert len(phones) == sum(word2ph), text
-
- return phones, tones, word2ph
-
-
-def get_bert_feature(text, word2ph):
- from text import english_bert_mock
-
- return english_bert_mock.get_bert_feature(text, word2ph)
-
-
-if __name__ == "__main__":
- # print(get_dict())
- # print(eng_word_to_phoneme("hello"))
- print(g2p("In this paper, we propose 1 DSPGAN, a GAN-based universal vocoder."))
- # all_phones = set()
- # for k, syllables in eng_dict.items():
- # for group in syllables:
- # for ph in group:
- # all_phones.add(ph)
- # print(all_phones)
diff --git a/text/english_bert_mock.py b/text/english_bert_mock.py
deleted file mode 100644
index 782b65d..0000000
--- a/text/english_bert_mock.py
+++ /dev/null
@@ -1,63 +0,0 @@
-import sys
-
-import torch
-from transformers import DebertaV2Model, DebertaV2Tokenizer
-
-from config import config
-
-
-LOCAL_PATH = "./bert/deberta-v3-large"
-
-tokenizer = DebertaV2Tokenizer.from_pretrained(LOCAL_PATH)
-
-models = dict()
-
-
-def get_bert_feature(
- text,
- word2ph,
- device=config.bert_gen_config.device,
- assist_text=None,
- assist_text_weight=0.7,
-):
- if (
- sys.platform == "darwin"
- and torch.backends.mps.is_available()
- and device == "cpu"
- ):
- device = "mps"
- if not device:
- device = "cuda"
- if device == "cuda" and not torch.cuda.is_available():
- device = "cpu"
- if device not in models.keys():
- models[device] = DebertaV2Model.from_pretrained(LOCAL_PATH).to(device)
- with torch.no_grad():
- inputs = tokenizer(text, return_tensors="pt")
- for i in inputs:
- inputs[i] = inputs[i].to(device)
- res = models[device](**inputs, output_hidden_states=True)
- res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
- if assist_text:
- style_inputs = tokenizer(assist_text, return_tensors="pt")
- for i in style_inputs:
- style_inputs[i] = style_inputs[i].to(device)
- style_res = models[device](**style_inputs, output_hidden_states=True)
- style_res = torch.cat(style_res["hidden_states"][-3:-2], -1)[0].cpu()
- style_res_mean = style_res.mean(0)
- assert len(word2ph) == res.shape[0], (text, res.shape[0], len(word2ph))
- word2phone = word2ph
- phone_level_feature = []
- for i in range(len(word2phone)):
- if assist_text:
- repeat_feature = (
- res[i].repeat(word2phone[i], 1) * (1 - assist_text_weight)
- + style_res_mean.repeat(word2phone[i], 1) * assist_text_weight
- )
- else:
- repeat_feature = res[i].repeat(word2phone[i], 1)
- phone_level_feature.append(repeat_feature)
-
- phone_level_feature = torch.cat(phone_level_feature, dim=0)
-
- return phone_level_feature.T
diff --git a/text/get_bert.py b/text/get_bert.py
deleted file mode 100644
index 07e4bd7..0000000
--- a/text/get_bert.py
+++ /dev/null
@@ -1,16 +0,0 @@
-from .japanese_bert import get_bert_feature as get_japanese_bert_feature
-
-
-def get_bert(text, word2ph, language, device, assist_text=None, assist_text_weight=0.7):
- if language == "ZH":
- from .chinese_bert import get_bert_feature
- elif language == "EN":
- from .english_bert_mock import get_bert_feature
- elif language == "JP":
- # pyopenjtalkのworkerを1度だけ起動するため、ここでのimportは避ける
- # 他言語のようにimportすると、get_bertが呼ばれるたびにpyopenjtalkのworkerが起動してしまう
- get_bert_feature = get_japanese_bert_feature
- else:
- raise ValueError(f"Language {language} not supported")
-
- return get_bert_feature(text, word2ph, device, assist_text, assist_text_weight)
diff --git a/text/japanese.py b/text/japanese.py
deleted file mode 100644
index f7bfcae..0000000
--- a/text/japanese.py
+++ /dev/null
@@ -1,644 +0,0 @@
-# Convert Japanese text to phonemes which is
-# compatible with Julius https://github.com/julius-speech/segmentation-kit
-import re
-import unicodedata
-from pathlib import Path
-
-from num2words import num2words
-from transformers import AutoTokenizer
-
-from common.log import logger
-from text import punctuation
-from text.japanese_mora_list import (
- mora_kata_to_mora_phonemes,
- mora_phonemes_to_mora_kata,
-)
-from text.user_dict import update_dict
-
-from . import pyopenjtalk_worker as pyopenjtalk
-
-# 最初にpyopenjtalkの辞書を更新
-update_dict()
-
-# 子音の集合
-COSONANTS = set(
- [
- cosonant
- for cosonant, _ in mora_kata_to_mora_phonemes.values()
- if cosonant is not None
- ]
-)
-
-# 母音の集合、便宜上「ん」を含める
-VOWELS = {"a", "i", "u", "e", "o", "N"}
-
-
-class YomiError(Exception):
- """
- OpenJTalkで、読みが正しく取得できない箇所があるときに発生する例外。
- 基本的に「学習の前処理のテキスト処理時」には発生させ、そうでない場合は、
- ignore_yomi_error=Trueにしておいて、この例外を発生させないようにする。
- """
-
- pass
-
-
-# 正規化で記号を変換するための辞書
-rep_map = {
- "：": ",",
- "；": ",",
- "，": ",",
- "。": ".",
- "！": "!",
- "？": "?",
- "\n": ".",
- "．": ".",
- "…": "...",
- "···": "...",
- "・・・": "...",
- "·": ",",
- "・": ",",
- "、": ",",
- "$": ".",
- "“": "'",
- "”": "'",
- '"': "'",
- "‘": "'",
- "’": "'",
- "（": "'",
- "）": "'",
- "(": "'",
- ")": "'",
- "《": "'",
- "》": "'",
- "【": "'",
- "】": "'",
- "[": "'",
- "]": "'",
- # NFKC正規化後のハイフン・ダッシュの変種を全て通常半角ハイフン - \u002d に変換
- "\u02d7": "\u002d", # ˗, Modifier Letter Minus Sign
- "\u2010": "\u002d", # ‐, Hyphen,
- # "\u2011": "\u002d", # ‑, Non-Breaking Hyphen, NFKCにより\u2010に変換される
- "\u2012": "\u002d", # ‒, Figure Dash
- "\u2013": "\u002d", # –, En Dash
- "\u2014": "\u002d", # —, Em Dash
- "\u2015": "\u002d", # ―, Horizontal Bar
- "\u2043": "\u002d", # ⁃, Hyphen Bullet
- "\u2212": "\u002d", # −, Minus Sign
- "\u23af": "\u002d", # ⎯, Horizontal Line Extension
- "\u23e4": "\u002d", # ⏤, Straightness
- "\u2500": "\u002d", # ─, Box Drawings Light Horizontal
- "\u2501": "\u002d", # ━, Box Drawings Heavy Horizontal
- "\u2e3a": "\u002d", # ⸺, Two-Em Dash
- "\u2e3b": "\u002d", # ⸻, Three-Em Dash
- # "～": "-", # これは長音記号「ー」として扱うよう変更
- # "~": "-", # これも長音記号「ー」として扱うよう変更
- "「": "'",
- "」": "'",
-}
-
-
-def text_normalize(text):
- """
- 日本語のテキストを正規化する。
- 結果は、ちょうど次の文字のみからなる：
- - ひらがな
- - カタカナ（全角長音記号「ー」が入る！）
- - 漢字
- - 半角アルファベット（大文字と小文字）
- - ギリシャ文字
- - `.` （句点`。`や`…`の一部や改行等）
- - `,` （読点`、`や`:`等）
- - `?` （疑問符`？`）
- - `!` （感嘆符`！`）
- - `'` （`「`や`」`等）
- - `-` （`―`（ダッシュ、長音記号ではない）や`-`等）
-
- 注意点:
- - 三点リーダー`…`は`...`に変換される（`なるほど…。` → `なるほど....`）
- - 数字は漢字に変換される（`1,100円` → `千百円`、`52.34` → `五十二点三四`）
- - 読点や疑問符等の位置・個数等は保持される（`??あ、、！！！` → `??あ,,!!!`）
- """
- res = unicodedata.normalize("NFKC", text) # ここでアルファベットは半角になる
- res = japanese_convert_numbers_to_words(res) # 「100円」→「百円」等
- # 「～」と「~」も長音記号として扱う
- res = res.replace("~", "ー")
- res = res.replace("～", "ー")
-
- res = replace_punctuation(res) # 句読点等正規化、読めない文字を削除
-
- # 結合文字の濁点・半濁点を削除
- # 通常の「ば」等はそのままのこされる、「あ゛」は上で「あ゙」になりここで「あ」になる
- res = res.replace("\u3099", "") # 結合文字の濁点を削除、る゙ → る
- res = res.replace("\u309A", "") # 結合文字の半濁点を削除、な゚ → な
- return res
-
-
-def replace_punctuation(text: str) -> str:
- """句読点等を「.」「,」「!」「?」「'」「-」に正規化し、OpenJTalkで読みが取得できるもののみ残す：
- 漢字・平仮名・カタカナ、アルファベット、ギリシャ文字
- """
- pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys()))
-
- # 句読点を辞書で置換
- replaced_text = pattern.sub(lambda x: rep_map[x.group()], text)
-
- replaced_text = re.sub(
- # ↓ ひらがな、カタカナ、漢字
- r"[^\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\u3400-\u4DBF\u3005"
- # ↓ 半角アルファベット（大文字と小文字）
- + r"\u0041-\u005A\u0061-\u007A"
- # ↓ 全角アルファベット（大文字と小文字）
- + r"\uFF21-\uFF3A\uFF41-\uFF5A"
- # ↓ ギリシャ文字
- + r"\u0370-\u03FF\u1F00-\u1FFF"
- # ↓ "!", "?", "…", ",", ".", "'", "-", 但し`…`はすでに`...`に変換されている
- + "".join(punctuation) + r"]+",
- # 上述以外の文字を削除
- "",
- replaced_text,
- )
-
- return replaced_text
-
-
-_NUMBER_WITH_SEPARATOR_RX = re.compile("[0-9]{1,3}(,[0-9]{3})+")
-_CURRENCY_MAP = {"$": "ドル", "¥": "円", "£": "ポンド", "€": "ユーロ"}
-_CURRENCY_RX = re.compile(r"([$¥£€])([0-9.]*[0-9])")
-_NUMBER_RX = re.compile(r"[0-9]+(\.[0-9]+)?")
-
-
-def japanese_convert_numbers_to_words(text: str) -> str:
- res = _NUMBER_WITH_SEPARATOR_RX.sub(lambda m: m[0].replace(",", ""), text)
- res = _CURRENCY_RX.sub(lambda m: m[2] + _CURRENCY_MAP.get(m[1], m[1]), res)
- res = _NUMBER_RX.sub(lambda m: num2words(m[0], lang="ja"), res)
- return res
-
-
-def g2p(
- norm_text: str, use_jp_extra: bool = True, raise_yomi_error: bool = False
-) -> tuple[list[str], list[int], list[int]]:
- """
- 他で使われるメインの関数。`text_normalize()`で正規化された`norm_text`を受け取り、
- - phones: 音素のリスト（ただし`!`や`,`や`.`等punctuationが含まれうる）
- - tones: アクセントのリスト、0（低）と1（高）からなり、phonesと同じ長さ
- - word2ph: 元のテキストの各文字に音素が何個割り当てられるかを表すリスト
- のタプルを返す。
- ただし`phones`と`tones`の最初と終わりに`_`が入り、応じて`word2ph`の最初と最後に1が追加される。
-
- use_jp_extra: Falseの場合、「ん」の音素を「N」ではなく「n」とする。
- raise_yomi_error: Trueの場合、読めない文字があるときに例外を発生させる。
- Falseの場合は読めない文字が消えたような扱いとして処理される。
- """
- # pyopenjtalkのフルコンテキストラベルを使ってアクセントを取り出すと、punctuationの位置が消えてしまい情報が失われてしまう：
- # 「こんにちは、世界。」と「こんにちは！世界。」と「こんにちは！！！？？？世界……。」は全て同じになる。
- # よって、まずpunctuation無しの音素とアクセントのリストを作り、
- # それとは別にpyopenjtalk.run_frontend()で得られる音素リスト（こちらはpunctuationが保持される）を使い、
- # アクセント割当をしなおすことによってpunctuationを含めた音素とアクセントのリストを作る。
-
- # punctuationがすべて消えた、音素とアクセントのタプルのリスト（「ん」は「N」）
- phone_tone_list_wo_punct = g2phone_tone_wo_punct(norm_text)
-
- # sep_text: 単語単位の単語のリスト、読めない文字があったらraise_yomi_errorなら例外、そうでないなら読めない文字が消えて返ってくる
- # sep_kata: 単語単位の単語のカタカナ読みのリスト
- sep_text, sep_kata = text2sep_kata(norm_text, raise_yomi_error=raise_yomi_error)
-
- # sep_phonemes: 各単語ごとの音素のリストのリスト
- sep_phonemes = handle_long([kata2phoneme_list(i) for i in sep_kata])
-
- # phone_w_punct: sep_phonemesを結合した、punctuationを元のまま保持した音素列
- phone_w_punct: list[str] = []
- for i in sep_phonemes:
- phone_w_punct += i
-
- # punctuation無しのアクセント情報を使って、punctuationを含めたアクセント情報を作る
- phone_tone_list = align_tones(phone_w_punct, phone_tone_list_wo_punct)
- # logger.debug(f"phone_tone_list:\n{phone_tone_list}")
- # word2phは厳密な解答は不可能なので（「今日」「眼鏡」等の熟字訓が存在）、
- # Bert-VITS2では、単語単位の分割を使って、単語の文字ごとにだいたい均等に音素を分配する
-
- # sep_textから、各単語を1文字1文字分割して、文字のリスト（のリスト）を作る
- sep_tokenized: list[list[str]] = []
- for i in sep_text:
- if i not in punctuation:
- sep_tokenized.append(
- tokenizer.tokenize(i)
- ) # ここでおそらく`i`が文字単位に分割される
- else:
- sep_tokenized.append([i])
-
- # 各単語について、音素の数と文字の数を比較して、均等っぽく分配する
- word2ph = []
- for token, phoneme in zip(sep_tokenized, sep_phonemes):
- phone_len = len(phoneme)
- word_len = len(token)
- word2ph += distribute_phone(phone_len, word_len)
-
- # 最初と最後に`_`記号を追加、アクセントは0（低）、word2phもそれに合わせて追加
- phone_tone_list = [("_", 0)] + phone_tone_list + [("_", 0)]
- word2ph = [1] + word2ph + [1]
-
- phones = [phone for phone, _ in phone_tone_list]
- tones = [tone for _, tone in phone_tone_list]
-
- assert len(phones) == sum(word2ph), f"{len(phones)} != {sum(word2ph)}"
-
- # use_jp_extraでない場合は「N」を「n」に変換
- if not use_jp_extra:
- phones = [phone if phone != "N" else "n" for phone in phones]
-
- return phones, tones, word2ph
-
-
-def g2kata_tone(norm_text: str) -> list[tuple[str, int]]:
- """
- テキストからカタカナとアクセントのペアのリストを返す。
- 推論時のみに使われるので、常に`raise_yomi_error=False`でg2pを呼ぶ。
- """
- phones, tones, _ = g2p(norm_text, use_jp_extra=True, raise_yomi_error=False)
- return phone_tone2kata_tone(list(zip(phones, tones)))
-
-
-def phone_tone2kata_tone(phone_tone: list[tuple[str, int]]) -> list[tuple[str, int]]:
- """phone_toneをのphone部分をカタカナに変換する。ただし最初と最後の("_", 0)は無視"""
- phone_tone = phone_tone[1:] # 最初の("_", 0)を無視
- phones = [phone for phone, _ in phone_tone]
- tones = [tone for _, tone in phone_tone]
- result: list[tuple[str, int]] = []
- current_mora = ""
- for phone, next_phone, tone, next_tone in zip(phones, phones[1:], tones, tones[1:]):
- # zipの関係で最後の("_", 0)は無視されている
- if phone in punctuation:
- result.append((phone, tone))
- continue
- if phone in COSONANTS: # n以外の子音の場合
- assert current_mora == "", f"Unexpected {phone} after {current_mora}"
- assert tone == next_tone, f"Unexpected {phone} tone {tone} != {next_tone}"
- current_mora = phone
- else:
- # phoneが母音もしくは「N」
- current_mora += phone
- result.append((mora_phonemes_to_mora_kata[current_mora], tone))
- current_mora = ""
- return result
-
-
-def kata_tone2phone_tone(kata_tone: list[tuple[str, int]]) -> list[tuple[str, int]]:
- """`phone_tone2kata_tone()`の逆。"""
- result: list[tuple[str, int]] = [("_", 0)]
- for mora, tone in kata_tone:
- if mora in punctuation:
- result.append((mora, tone))
- else:
- cosonant, vowel = mora_kata_to_mora_phonemes[mora]
- if cosonant is None:
- result.append((vowel, tone))
- else:
- result.append((cosonant, tone))
- result.append((vowel, tone))
- result.append(("_", 0))
- return result
-
-
-def g2phone_tone_wo_punct(text: str) -> list[tuple[str, int]]:
- """
- テキストに対して、音素とアクセント（0か1）のペアのリストを返す。
- ただし「!」「.」「?」等の非音素記号(punctuation)は全て消える（ポーズ記号も残さない）。
- 非音素記号を含める処理は`align_tones()`で行われる。
- また「っ」は「q」に、「ん」は「N」に変換される。
- 例: "こんにちは、世界ー。。元気？！" →
- [('k', 0), ('o', 0), ('N', 1), ('n', 1), ('i', 1), ('ch', 1), ('i', 1), ('w', 1), ('a', 1), ('s', 1), ('e', 1), ('k', 0), ('a', 0), ('i', 0), ('i', 0), ('g', 1), ('e', 1), ('N', 0), ('k', 0), ('i', 0)]
- """
- prosodies = pyopenjtalk_g2p_prosody(text, drop_unvoiced_vowels=True)
- # logger.debug(f"prosodies: {prosodies}")
- result: list[tuple[str, int]] = []
- current_phrase: list[tuple[str, int]] = []
- current_tone = 0
- for i, letter in enumerate(prosodies):
- # 特殊記号の処理
-
- # 文頭記号、無視する
- if letter == "^":
- assert i == 0, "Unexpected ^"
- # アクセント句の終わりに来る記号
- elif letter in ("$", "?", "_", "#"):
- # 保持しているフレーズを、アクセント数値を0-1に修正し結果に追加
- result.extend(fix_phone_tone(current_phrase))
- # 末尾に来る終了記号、無視（文中の疑問文は`_`になる）
- if letter in ("$", "?"):
- assert i == len(prosodies) - 1, f"Unexpected {letter}"
- # あとは"_"（ポーズ）と"#"（アクセント句の境界）のみ
- # これらは残さず、次のアクセント句に備える。
- current_phrase = []
- # 0を基準点にしてそこから上昇・下降する（負の場合は上の`fix_phone_tone`で直る）
- current_tone = 0
- # アクセント上昇記号
- elif letter == "[":
- current_tone = current_tone + 1
- # アクセント下降記号
- elif letter == "]":
- current_tone = current_tone - 1
- # それ以外は通常の音素
- else:
- if letter == "cl": # 「っ」の処理
- letter = "q"
- # elif letter == "N": # 「ん」の処理
- # letter = "n"
- current_phrase.append((letter, current_tone))
- return result
-
-
-def text2sep_kata(
- norm_text: str, raise_yomi_error: bool = False
-) -> tuple[list[str], list[str]]:
- """
- `text_normalize`で正規化済みの`norm_text`を受け取り、それを単語分割し、
- 分割された単語リストとその読み（カタカナor記号1文字）のリストのタプルを返す。
- 単語分割結果は、`g2p()`の`word2ph`で1文字あたりに割り振る音素記号の数を決めるために使う。
- 例:
- `私はそう思う!って感じ?` →
- ["私", "は", "そう", "思う", "!", "って", "感じ", "?"], ["ワタシ", "ワ", "ソー", "オモウ", "!", "ッテ", "カンジ", "?"]
-
- raise_yomi_error: Trueの場合、読めない文字があるときに例外を発生させる。
- Falseの場合は読めない文字が消えたような扱いとして処理される。
- """
- # parsed: OpenJTalkの解析結果
- parsed = pyopenjtalk.run_frontend(norm_text)
- sep_text: list[str] = []
- sep_kata: list[str] = []
- for parts in parsed:
- # word: 実際の単語の文字列
- # yomi: その読み、但し無声化サインの`’`は除去
- word, yomi = replace_punctuation(parts["string"]), parts["pron"].replace(
- "’", ""
- )
- """
- ここで`yomi`の取りうる値は以下の通りのはず。
- - `word`が通常単語 → 通常の読み（カタカナ）
- （カタカナからなり、長音記号も含みうる、`アー` 等）
- - `word`が`ー` から始まる → `ーラー` や `ーーー` など
- - `word`が句読点や空白等 → `、`
- - `word`がpunctuationの繰り返し → 全角にしたもの
- 基本的にpunctuationは1文字ずつ分かれるが、何故かある程度連続すると1つにまとまる。
- 他にも`word`が読めないキリル文字アラビア文字等が来ると`、`になるが、正規化でこの場合は起きないはず。
- また元のコードでは`yomi`が空白の場合の処理があったが、これは起きないはず。
- 処理すべきは`yomi`が`、`の場合のみのはず。
- """
- assert yomi != "", f"Empty yomi: {word}"
- if yomi == "、":
- # wordは正規化されているので、`.`, `,`, `!`, `'`, `-`, `--` のいずれか
- if not set(word).issubset(set(punctuation)): # 記号繰り返しか判定
- # ここはpyopenjtalkが読めない文字等のときに起こる
- if raise_yomi_error:
- raise YomiError(f"Cannot read: {word} in:\n{norm_text}")
- logger.warning(f"Ignoring unknown: {word} in:\n{norm_text}")
- continue
- # yomiは元の記号のままに変更
- yomi = word
- elif yomi == "？":
- assert word == "?", f"yomi `？` comes from: {word}"
- yomi = "?"
- sep_text.append(word)
- sep_kata.append(yomi)
- return sep_text, sep_kata
-
-
-# ESPnetの実装から引用、変更点無し。「ん」は「N」なことに注意。
-# https://github.com/espnet/espnet/blob/master/espnet2/text/phoneme_tokenizer.py
-def pyopenjtalk_g2p_prosody(text: str, drop_unvoiced_vowels: bool = True) -> list[str]:
- """Extract phoneme + prosoody symbol sequence from input full-context labels.
-
- The algorithm is based on `Prosodic features control by symbols as input of
- sequence-to-sequence acoustic modeling for neural TTS`_ with some r9y9's tweaks.
-
- Args:
- text (str): Input text.
- drop_unvoiced_vowels (bool): whether to drop unvoiced vowels.
-
- Returns:
- List[str]: List of phoneme + prosody symbols.
-
- Examples:
- >>> from espnet2.text.phoneme_tokenizer import pyopenjtalk_g2p_prosody
- >>> pyopenjtalk_g2p_prosody("こんにちは。")
- ['^', 'k', 'o', '[', 'N', 'n', 'i', 'ch', 'i', 'w', 'a', '$']
-
- .. _`Prosodic features control by symbols as input of sequence-to-sequence acoustic
- modeling for neural TTS`: https://doi.org/10.1587/transinf.2020EDP7104
-
- """
- labels = pyopenjtalk.make_label(pyopenjtalk.run_frontend(text))
- N = len(labels)
-
- phones = []
- for n in range(N):
- lab_curr = labels[n]
-
- # current phoneme
- p3 = re.search(r"\-(.*?)\+", lab_curr).group(1)
- # deal unvoiced vowels as normal vowels
- if drop_unvoiced_vowels and p3 in "AEIOU":
- p3 = p3.lower()
-
- # deal with sil at the beginning and the end of text
- if p3 == "sil":
- assert n == 0 or n == N - 1
- if n == 0:
- phones.append("^")
- elif n == N - 1:
- # check question form or not
- e3 = _numeric_feature_by_regex(r"!(\d+)_", lab_curr)
- if e3 == 0:
- phones.append("$")
- elif e3 == 1:
- phones.append("?")
- continue
- elif p3 == "pau":
- phones.append("_")
- continue
- else:
- phones.append(p3)
-
- # accent type and position info (forward or backward)
- a1 = _numeric_feature_by_regex(r"/A:([0-9\-]+)\+", lab_curr)
- a2 = _numeric_feature_by_regex(r"\+(\d+)\+", lab_curr)
- a3 = _numeric_feature_by_regex(r"\+(\d+)/", lab_curr)
-
- # number of mora in accent phrase
- f1 = _numeric_feature_by_regex(r"/F:(\d+)_", lab_curr)
-
- a2_next = _numeric_feature_by_regex(r"\+(\d+)\+", labels[n + 1])
- # accent phrase border
- if a3 == 1 and a2_next == 1 and p3 in "aeiouAEIOUNcl":
- phones.append("#")
- # pitch falling
- elif a1 == 0 and a2_next == a2 + 1 and a2 != f1:
- phones.append("]")
- # pitch rising
- elif a2 == 1 and a2_next == 2:
- phones.append("[")
-
- return phones
-
-
-def _numeric_feature_by_regex(regex, s):
- match = re.search(regex, s)
- if match is None:
- return -50
- return int(match.group(1))
-
-
-def fix_phone_tone(phone_tone_list: list[tuple[str, int]]) -> list[tuple[str, int]]:
- """
- `phone_tone_list`のtone（アクセントの値）を0か1の範囲に修正する。
- 例: [(a, 0), (i, -1), (u, -1)] → [(a, 1), (i, 0), (u, 0)]
- """
- tone_values = set(tone for _, tone in phone_tone_list)
- if len(tone_values) == 1:
- assert tone_values == {0}, tone_values
- return phone_tone_list
- elif len(tone_values) == 2:
- if tone_values == {0, 1}:
- return phone_tone_list
- elif tone_values == {-1, 0}:
- return [
- (letter, 0 if tone == -1 else 1) for letter, tone in phone_tone_list
- ]
- else:
- raise ValueError(f"Unexpected tone values: {tone_values}")
- else:
- raise ValueError(f"Unexpected tone values: {tone_values}")
-
-
-def distribute_phone(n_phone: int, n_word: int) -> list[int]:
- """
- 左から右に1ずつ振り分け、次にまた左から右に1ずつ増やし、というふうに、
- 音素の数`n_phone`を単語の数`n_word`に分配する。
- """
- phones_per_word = [0] * n_word
- for _ in range(n_phone):
- min_tasks = min(phones_per_word)
- min_index = phones_per_word.index(min_tasks)
- phones_per_word[min_index] += 1
- return phones_per_word
-
-
-def handle_long(sep_phonemes: list[list[str]]) -> list[list[str]]:
- """
- フレーズごとに分かれた音素（長音記号がそのまま）のリストのリスト`sep_phonemes`を受け取り、
- その長音記号を処理して、音素のリストのリストを返す。
- 基本的には直前の音素を伸ばすが、直前の音素が母音でない場合もしくは冒頭の場合は、
- おそらく長音記号とダッシュを勘違いしていると思われるので、ダッシュに対応する音素`-`に変換する。
- """
- for i in range(len(sep_phonemes)):
- if len(sep_phonemes[i]) == 0:
- # 空白文字等でリストが空の場合
- continue
- if sep_phonemes[i][0] == "ー":
- if i != 0:
- prev_phoneme = sep_phonemes[i - 1][-1]
- if prev_phoneme in VOWELS:
- # 母音と「ん」のあとの伸ばし棒なので、その母音に変換
- sep_phonemes[i][0] = sep_phonemes[i - 1][-1]
- else:
- # 「。ーー」等おそらく予期しない長音記号
- # ダッシュの勘違いだと思われる
- sep_phonemes[i][0] = "-"
- else:
- # 冒頭に長音記号が来ていおり、これはダッシュの勘違いと思われる
- sep_phonemes[i][0] = "-"
- if "ー" in sep_phonemes[i]:
- for j in range(len(sep_phonemes[i])):
- if sep_phonemes[i][j] == "ー":
- sep_phonemes[i][j] = sep_phonemes[i][j - 1][-1]
- return sep_phonemes
-
-
-tokenizer = AutoTokenizer.from_pretrained("./bert/deberta-v2-large-japanese-char-wwm")
-
-
-def align_tones(
- phones_with_punct: list[str], phone_tone_list: list[tuple[str, int]]
-) -> list[tuple[str, int]]:
- """
- 例:
- …私は、、そう思う。
- phones_with_punct:
- [".", ".", ".", "w", "a", "t", "a", "sh", "i", "w", "a", ",", ",", "s", "o", "o", "o", "m", "o", "u", "."]
- phone_tone_list:
- [("w", 0), ("a", 0), ("t", 1), ("a", 1), ("sh", 1), ("i", 1), ("w", 1), ("a", 1), ("_", 0), ("s", 0), ("o", 0), ("o", 1), ("o", 1), ("m", 1), ("o", 1), ("u", 0))]
- Return:
- [(".", 0), (".", 0), (".", 0), ("w", 0), ("a", 0), ("t", 1), ("a", 1), ("sh", 1), ("i", 1), ("w", 1), ("a", 1), (",", 0), (",", 0), ("s", 0), ("o", 0), ("o", 1), ("o", 1), ("m", 1), ("o", 1), ("u", 0), (".", 0)]
- """
- result: list[tuple[str, int]] = []
- tone_index = 0
- for phone in phones_with_punct:
- if tone_index >= len(phone_tone_list):
- # 余ったpunctuationがある場合 → (punctuation, 0)を追加
- result.append((phone, 0))
- elif phone == phone_tone_list[tone_index][0]:
- # phone_tone_listの現在の音素と一致する場合 → toneをそこから取得、(phone, tone)を追加
- result.append((phone, phone_tone_list[tone_index][1]))
- # 探すindexを1つ進める
- tone_index += 1
- elif phone in punctuation:
- # phoneがpunctuationの場合 → (phone, 0)を追加
- result.append((phone, 0))
- else:
- logger.debug(f"phones: {phones_with_punct}")
- logger.debug(f"phone_tone_list: {phone_tone_list}")
- logger.debug(f"result: {result}")
- logger.debug(f"tone_index: {tone_index}")
- logger.debug(f"phone: {phone}")
- raise ValueError(f"Unexpected phone: {phone}")
- return result
-
-
-def kata2phoneme_list(text: str) -> list[str]:
- """
- 原則カタカナの`text`を受け取り、それをそのままいじらずに音素記号のリストに変換。
- 注意点：
- - punctuationかその繰り返しが来た場合、punctuationたちをそのままリストにして返す。
- - 冒頭に続く「ー」はそのまま「ー」のままにする（`handle_long()`で処理される）
- - 文中の「ー」は前の音素記号の最後の音素記号に変換される。
- 例：
- `ーーソーナノカーー` → ["ー", "ー", "s", "o", "o", "n", "a", "n", "o", "k", "a", "a", "a"]
- `?` → ["?"]
- `!?!?!?!?!` → ["!", "?", "!", "?", "!", "?", "!", "?", "!"]
- """
- if set(text).issubset(set(punctuation)):
- return list(text)
- # `text`がカタカナ（`ー`含む）のみからなるかどうかをチェック
- if re.fullmatch(r"[\u30A0-\u30FF]+", text) is None:
- raise ValueError(f"Input must be katakana only: {text}")
- sorted_keys = sorted(mora_kata_to_mora_phonemes.keys(), key=len, reverse=True)
- pattern = "|".join(map(re.escape, sorted_keys))
-
- def mora2phonemes(mora: str) -> str:
- cosonant, vowel = mora_kata_to_mora_phonemes[mora]
- if cosonant is None:
- return f" {vowel}"
- return f" {cosonant} {vowel}"
-
- spaced_phonemes = re.sub(pattern, lambda m: mora2phonemes(m.group()), text)
-
- # 長音記号「ー」の処理
- long_pattern = r"(\w)(ー*)"
- long_replacement = lambda m: m.group(1) + (" " + m.group(1)) * len(m.group(2))
- spaced_phonemes = re.sub(long_pattern, long_replacement, spaced_phonemes)
- return spaced_phonemes.strip().split(" ")
-
-
-if __name__ == "__main__":
- tokenizer = AutoTokenizer.from_pretrained(
- "./bert/deberta-v2-large-japanese-char-wwm"
- )
- text = "こんにちは、世界。"
- from text.japanese_bert import get_bert_feature
-
- text = text_normalize(text)
-
- phones, tones, word2ph = g2p(text)
- bert = get_bert_feature(text, word2ph)
-
- print(phones, tones, word2ph, bert.shape)
diff --git a/text/japanese_bert.py b/text/japanese_bert.py
deleted file mode 100644
index 9ebc64a..0000000
--- a/text/japanese_bert.py
+++ /dev/null
@@ -1,70 +0,0 @@
-import sys
-
-import torch
-from transformers import AutoModelForMaskedLM, AutoTokenizer
-
-from config import config
-from text.japanese import text2sep_kata, text_normalize
-
-LOCAL_PATH = "./bert/deberta-v2-large-japanese-char-wwm"
-
-tokenizer = AutoTokenizer.from_pretrained(LOCAL_PATH)
-
-models = dict()
-
-
-def get_bert_feature(
- text,
- word2ph,
- device=config.bert_gen_config.device,
- assist_text=None,
- assist_text_weight=0.7,
-):
- # 各単語が何文字かを作る`word2ph`を使う必要があるので、読めない文字は必ず無視する
- # でないと`word2ph`の結果とテキストの文字数結果が整合性が取れない
- text = "".join(text2sep_kata(text, raise_yomi_error=False)[0])
-
- if assist_text:
- assist_text = "".join(text2sep_kata(assist_text, raise_yomi_error=False)[0])
- if (
- sys.platform == "darwin"
- and torch.backends.mps.is_available()
- and device == "cpu"
- ):
- device = "mps"
- if not device:
- device = "cuda"
- if device == "cuda" and not torch.cuda.is_available():
- device = "cpu"
- if device not in models.keys():
- models[device] = AutoModelForMaskedLM.from_pretrained(LOCAL_PATH).to(device)
- with torch.no_grad():
- inputs = tokenizer(text, return_tensors="pt")
- for i in inputs:
- inputs[i] = inputs[i].to(device)
- res = models[device](**inputs, output_hidden_states=True)
- res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
- if assist_text:
- style_inputs = tokenizer(assist_text, return_tensors="pt")
- for i in style_inputs:
- style_inputs[i] = style_inputs[i].to(device)
- style_res = models[device](**style_inputs, output_hidden_states=True)
- style_res = torch.cat(style_res["hidden_states"][-3:-2], -1)[0].cpu()
- style_res_mean = style_res.mean(0)
-
- assert len(word2ph) == len(text) + 2, f"word2ph: {word2ph}, text: {text}"
- word2phone = word2ph
- phone_level_feature = []
- for i in range(len(word2phone)):
- if assist_text:
- repeat_feature = (
- res[i].repeat(word2phone[i], 1) * (1 - assist_text_weight)
- + style_res_mean.repeat(word2phone[i], 1) * assist_text_weight
- )
- else:
- repeat_feature = res[i].repeat(word2phone[i], 1)
- phone_level_feature.append(repeat_feature)
-
- phone_level_feature = torch.cat(phone_level_feature, dim=0)
-
- return phone_level_feature.T
diff --git a/text/pyopenjtalk_worker/__init__.py b/text/pyopenjtalk_worker/__init__.py
deleted file mode 100644
index 7aa0d0e..0000000
--- a/text/pyopenjtalk_worker/__init__.py
+++ /dev/null
@@ -1,126 +0,0 @@
-"""
-Run the pyopenjtalk worker in a separate process
-to avoid user dictionary access error
-"""
-
-from typing import Optional, Any
-
-from .worker_common import WORKER_PORT
-from .worker_client import WorkerClient
-
-from common.log import logger
-
-WORKER_CLIENT: Optional[WorkerClient] = None
-
-# pyopenjtalk interface
-
-# g2p: not used
-
-
-def run_frontend(text: str) -> list[dict[str, Any]]:
- assert WORKER_CLIENT
- ret = WORKER_CLIENT.dispatch_pyopenjtalk("run_frontend", text)
- assert isinstance(ret, list)
- return ret
-
-
-def make_label(njd_features) -> list[str]:
- assert WORKER_CLIENT
- ret = WORKER_CLIENT.dispatch_pyopenjtalk("make_label", njd_features)
- assert isinstance(ret, list)
- return ret
-
-
-def mecab_dict_index(path: str, out_path: str, dn_mecab: Optional[str] = None):
- assert WORKER_CLIENT
- WORKER_CLIENT.dispatch_pyopenjtalk("mecab_dict_index", path, out_path, dn_mecab)
-
-
-def update_global_jtalk_with_user_dict(path: str):
- assert WORKER_CLIENT
- WORKER_CLIENT.dispatch_pyopenjtalk("update_global_jtalk_with_user_dict", path)
-
-
-def unset_user_dict():
- assert WORKER_CLIENT
- WORKER_CLIENT.dispatch_pyopenjtalk("unset_user_dict")
-
-
-# initialize module when imported
-
-
-def initialize(port: int = WORKER_PORT):
- import time
- import socket
- import sys
- import atexit
- import signal
-
- logger.debug("initialize")
- global WORKER_CLIENT
- if WORKER_CLIENT:
- return
-
- client = None
- try:
- client = WorkerClient(port)
- except (socket.timeout, socket.error):
- logger.debug("try starting pyopenjtalk worker server")
- import os
- import subprocess
-
- worker_pkg_path = os.path.relpath(
- os.path.dirname(__file__), os.getcwd()
- ).replace(os.sep, ".")
- args = [sys.executable, "-m", worker_pkg_path, "--port", str(port)]
- # new session, new process group
- if sys.platform.startswith("win"):
- cf = subprocess.CREATE_NEW_CONSOLE | subprocess.CREATE_NEW_PROCESS_GROUP # type: ignore
- si = subprocess.STARTUPINFO() # type: ignore
- si.dwFlags |= subprocess.STARTF_USESHOWWINDOW # type: ignore
- si.wShowWindow = subprocess.SW_HIDE # type: ignore
- subprocess.Popen(args, creationflags=cf, startupinfo=si)
- else:
- # align with Windows behavior
- # start_new_session is same as specifying setsid in preexec_fn
- subprocess.Popen(args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True) # type: ignore
-
- # wait until server listening
- count = 0
- while True:
- try:
- client = WorkerClient(port)
- break
- except socket.error:
- time.sleep(1)
- count += 1
- # 10: max number of retries
- if count == 10:
- raise TimeoutError("サーバーに接続できませんでした")
-
- WORKER_CLIENT = client
- atexit.register(terminate)
-
- # when the process is killed
- def signal_handler(signum, frame):
- terminate()
-
- signal.signal(signal.SIGTERM, signal_handler)
-
-
-# top-level declaration
-def terminate():
- logger.debug("terminate")
- global WORKER_CLIENT
- if not WORKER_CLIENT:
- return
-
- # repare for unexpected errors
- try:
- if WORKER_CLIENT.status() == 1:
- WORKER_CLIENT.quit_server()
- except Exception as e:
- logger.error(e)
-
- WORKER_CLIENT.close()
- WORKER_CLIENT = None
diff --git a/text/user_dict/README.md b/text/user_dict/README.md
deleted file mode 100644
index 6f5618e..0000000
--- a/text/user_dict/README.md
+++ /dev/null
@@ -1,19 +0,0 @@
-このフォルダに含まれるユーザー辞書関連のコードは、[VOICEVOX engine](https://github.com/VOICEVOX/voicevox_engine)プロジェクトのコードを改変したものを使用しています。VOICEVOXプロジェクトのチームに深く感謝し、その貢献を尊重します。
-
-**元のコード**:
-
-- [voicevox_engine/user_dict/](https://github.com/VOICEVOX/voicevox_engine/tree/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/user_dict)
-- [voicevox_engine/model.py](https://github.com/VOICEVOX/voicevox_engine/blob/f181411ec69812296989d9cc583826c22eec87ae/voicevox_engine/model.py#L207)
-
-**改変の詳細**:
-
-- ファイル名の書き換えおよびそれに伴うimport文の書き換え。
-- VOICEVOX固有の部分をコメントアウト。
-- mutexを使用している部分をコメントアウト。
-- 参照しているpyopenjtalkの違いによるメソッド名の書き換え。
-- UserDictWordのmora_countのデフォルト値をNoneに指定。
-- Pydanticのモデルで必要な箇所のみを抽出。
-
-**ライセンス**:
-
-元のVOICEVOX engineのリポジトリのコードは、LGPL v3 と、ソースコードの公開が不要な別ライセンスのデュアルライセンスの下で使用されています。当プロジェクトにおけるこのモジュールもLGPLライセンスの下にあります。詳細については、プロジェクトのルートディレクトリにある[LGPL_LICENSE](/LGPL_LICENSE)ファイルをご参照ください。また、元のVOICEVOX engineプロジェクトのライセンスについては、[こちら](https://github.com/VOICEVOX/voicevox_engine/blob/master/LICENSE)をご覧ください。
diff --git a/tools/__init__.py b/tools/__init__.py
deleted file mode 100644
index b68d332..0000000
--- a/tools/__init__.py
+++ /dev/null
@@ -1,3 +0,0 @@
-"""
-工具包
-"""
diff --git a/tools/classify_language.py b/tools/classify_language.py
deleted file mode 100644
index 2b8a7ab..0000000
--- a/tools/classify_language.py
+++ /dev/null
@@ -1,197 +0,0 @@
-import regex as re
-
-try:
- from config import config
-
- LANGUAGE_IDENTIFICATION_LIBRARY = (
- config.webui_config.language_identification_library
- )
-except:
- LANGUAGE_IDENTIFICATION_LIBRARY = "langid"
-
-module = LANGUAGE_IDENTIFICATION_LIBRARY.lower()
-
-langid_languages = [
- "af",
- "am",
- "an",
- "ar",
- "as",
- "az",
- "be",
- "bg",
- "bn",
- "br",
- "bs",
- "ca",
- "cs",
- "cy",
- "da",
- "de",
- "dz",
- "el",
- "en",
- "eo",
- "es",
- "et",
- "eu",
- "fa",
- "fi",
- "fo",
- "fr",
- "ga",
- "gl",
- "gu",
- "he",
- "hi",
- "hr",
- "ht",
- "hu",
- "hy",
- "id",
- "is",
- "it",
- "ja",
- "jv",
- "ka",
- "kk",
- "km",
- "kn",
- "ko",
- "ku",
- "ky",
- "la",
- "lb",
- "lo",
- "lt",
- "lv",
- "mg",
- "mk",
- "ml",
- "mn",
- "mr",
- "ms",
- "mt",
- "nb",
- "ne",
- "nl",
- "nn",
- "no",
- "oc",
- "or",
- "pa",
- "pl",
- "ps",
- "pt",
- "qu",
- "ro",
- "ru",
- "rw",
- "se",
- "si",
- "sk",
- "sl",
- "sq",
- "sr",
- "sv",
- "sw",
- "ta",
- "te",
- "th",
- "tl",
- "tr",
- "ug",
- "uk",
- "ur",
- "vi",
- "vo",
- "wa",
- "xh",
- "zh",
- "zu",
-]
-
-
-def classify_language(text: str, target_languages: list = None) -> str:
- if module == "fastlid" or module == "fasttext":
- from fastlid import fastlid, supported_langs
-
- classifier = fastlid
- if target_languages != None:
- target_languages = [
- lang for lang in target_languages if lang in supported_langs
- ]
- fastlid.set_languages = target_languages
- elif module == "langid":
- import langid
-
- classifier = langid.classify
- if target_languages != None:
- target_languages = [
- lang for lang in target_languages if lang in langid_languages
- ]
- langid.set_languages(target_languages)
- else:
- raise ValueError(f"Wrong module {module}")
-
- lang = classifier(text)[0]
-
- return lang
-
-
-def classify_zh_ja(text: str) -> str:
- for idx, char in enumerate(text):
- unicode_val = ord(char)
-
- # 检测日语字符
- if 0x3040 <= unicode_val <= 0x309F or 0x30A0 <= unicode_val <= 0x30FF:
- return "ja"
-
- # 检测汉字字符
- if 0x4E00 <= unicode_val <= 0x9FFF:
- # 检查周围的字符
- next_char = text[idx + 1] if idx + 1 < len(text) else None
-
- if next_char and (
- 0x3040 <= ord(next_char) <= 0x309F or 0x30A0 <= ord(next_char) <= 0x30FF
- ):
- return "ja"
-
- return "zh"
-
-
-def split_alpha_nonalpha(text, mode=1):
- if mode == 1:
- pattern = r"(?<=[\u4e00-\u9fff\u3040-\u30FF\d\s])(?=[\p{Latin}])|(?<=[\p{Latin}\s])(?=[\u4e00-\u9fff\u3040-\u30FF\d])"
- elif mode == 2:
- pattern = r"(?<=[\u4e00-\u9fff\u3040-\u30FF\s])(?=[\p{Latin}\d])|(?<=[\p{Latin}\d\s])(?=[\u4e00-\u9fff\u3040-\u30FF])"
- else:
- raise ValueError("Invalid mode. Supported modes are 1 and 2.")
-
- return re.split(pattern, text)
-
-
-if __name__ == "__main__":
- text = "这是一个测试文本"
- print(classify_language(text))
- print(classify_zh_ja(text)) # "zh"
-
- text = "これはテストテキストです"
- print(classify_language(text))
- print(classify_zh_ja(text)) # "ja"
-
- text = "vits和Bert-VITS2是tts模型。花费3days.花费3天。Take 3 days"
-
- print(split_alpha_nonalpha(text, mode=1))
- # output: ['vits', '和', 'Bert-VITS', '2是', 'tts', '模型。花费3', 'days.花费3天。Take 3 days']
-
- print(split_alpha_nonalpha(text, mode=2))
- # output: ['vits', '和', 'Bert-VITS2', '是', 'tts', '模型。花费', '3days.花费', '3', '天。Take 3 days']
-
- text = "vits 和 Bert-VITS2 是 tts 模型。花费3days.花费3天。Take 3 days"
- print(split_alpha_nonalpha(text, mode=1))
- # output: ['vits ', '和 ', 'Bert-VITS', '2 ', '是 ', 'tts ', '模型。花费3', 'days.花费3天。Take ', '3 ', 'days']
-
- text = "vits 和 Bert-VITS2 是 tts 模型。花费3days.花费3天。Take 3 days"
- print(split_alpha_nonalpha(text, mode=2))
- # output: ['vits ', '和 ', 'Bert-VITS2 ', '是 ', 'tts ', '模型。花费', '3days.花费', '3', '天。Take ', '3 ', 'days']
diff --git a/tools/sentence.py b/tools/sentence.py
deleted file mode 100644
index b66864c..0000000
--- a/tools/sentence.py
+++ /dev/null
@@ -1,173 +0,0 @@
-import logging
-
-import regex as re
-
-from tools.classify_language import classify_language, split_alpha_nonalpha
-
-
-def check_is_none(item) -> bool:
- """none -> True, not none -> False"""
- return (
- item is None
- or (isinstance(item, str) and str(item).isspace())
- or str(item) == ""
- )
-
-
-def markup_language(text: str, target_languages: list = None) -> str:
- pattern = (
- r"[\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\>\=\?\@\[\]\{\}\\\\\^\_\`"
- r"\！？。＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」"
- r"『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘\'\‛\“\”\„\‟…‧﹏.]+"
- )
- sentences = re.split(pattern, text)
-
- pre_lang = ""
- p = 0
-
- if target_languages is not None:
- sorted_target_languages = sorted(target_languages)
- if sorted_target_languages in [["en", "zh"], ["en", "ja"], ["en", "ja", "zh"]]:
- new_sentences = []
- for sentence in sentences:
- new_sentences.extend(split_alpha_nonalpha(sentence))
- sentences = new_sentences
-
- for sentence in sentences:
- if check_is_none(sentence):
- continue
-
- lang = classify_language(sentence, target_languages)
-
- if pre_lang == "":
- text = text[:p] + text[p:].replace(
- sentence, f"[{lang.upper()}]{sentence}", 1
- )
- p += len(f"[{lang.upper()}]")
- elif pre_lang != lang:
- text = text[:p] + text[p:].replace(
- sentence, f"[{pre_lang.upper()}][{lang.upper()}]{sentence}", 1
- )
- p += len(f"[{pre_lang.upper()}][{lang.upper()}]")
- pre_lang = lang
- p += text[p:].index(sentence) + len(sentence)
- text += f"[{pre_lang.upper()}]"
-
- return text
-
-
-def split_by_language(text: str, target_languages: list = None) -> list:
- pattern = (
- r"[\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\>\=\?\@\[\]\{\}\\\\\^\_\`"
- r"\！？\。＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」"
- r"『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘\'\‛\“\”\„\‟…‧﹏.]+"
- )
- sentences = re.split(pattern, text)
-
- pre_lang = ""
- start = 0
- end = 0
- sentences_list = []
-
- if target_languages is not None:
- sorted_target_languages = sorted(target_languages)
- if sorted_target_languages in [["en", "zh"], ["en", "ja"], ["en", "ja", "zh"]]:
- new_sentences = []
- for sentence in sentences:
- new_sentences.extend(split_alpha_nonalpha(sentence))
- sentences = new_sentences
-
- for sentence in sentences:
- if check_is_none(sentence):
- continue
-
- lang = classify_language(sentence, target_languages)
-
- end += text[end:].index(sentence)
- if pre_lang != "" and pre_lang != lang:
- sentences_list.append((text[start:end], pre_lang))
- start = end
- end += len(sentence)
- pre_lang = lang
- sentences_list.append((text[start:], pre_lang))
-
- return sentences_list
-
-
-def sentence_split(text: str, max: int) -> list:
- pattern = r"[!(),—+\-.:;?？。，、；：]+"
- sentences = re.split(pattern, text)
- discarded_chars = re.findall(pattern, text)
-
- sentences_list, count, p = [], 0, 0
-
- # 按被分割的符号遍历
- for i, discarded_chars in enumerate(discarded_chars):
- count += len(sentences[i]) + len(discarded_chars)
- if count >= max:
- sentences_list.append(text[p : p + count].strip())
- p += count
- count = 0
-
- # 加入最后剩余的文本
- if p < len(text):
- sentences_list.append(text[p:])
-
- return sentences_list
-
-
-def sentence_split_and_markup(text, max=50, lang="auto", speaker_lang=None):
- # 如果该speaker只支持一种语言
- if speaker_lang is not None and len(speaker_lang) == 1:
- if lang.upper() not in ["AUTO", "MIX"] and lang.lower() != speaker_lang[0]:
- logging.debug(
- f'lang "{lang}" is not in speaker_lang {speaker_lang},automatically set lang={speaker_lang[0]}'
- )
- lang = speaker_lang[0]
-
- sentences_list = []
- if lang.upper() != "MIX":
- if max <= 0:
- sentences_list.append(
- markup_language(text, speaker_lang)
- if lang.upper() == "AUTO"
- else f"[{lang.upper()}]{text}[{lang.upper()}]"
- )
- else:
- for i in sentence_split(text, max):
- if check_is_none(i):
- continue
- sentences_list.append(
- markup_language(i, speaker_lang)
- if lang.upper() == "AUTO"
- else f"[{lang.upper()}]{i}[{lang.upper()}]"
- )
- else:
- sentences_list.append(text)
-
- for i in sentences_list:
- logging.debug(i)
-
- return sentences_list
-
-
-if __name__ == "__main__":
- text = "这几天心里颇不宁静。今晚在院子里坐着乘凉，忽然想起日日走过的荷塘，在这满月的光里，总该另有一番样子吧。月亮渐渐地升高了，墙外马路上孩子们的欢笑，已经听不见了；妻在屋里拍着闰儿，迷迷糊糊地哼着眠歌。我悄悄地披了大衫，带上门出去。"
- print(markup_language(text, target_languages=None))
- print(sentence_split(text, max=50))
- print(sentence_split_and_markup(text, max=50, lang="auto", speaker_lang=None))
-
- text = "你好，这是一段用来测试自动标注的文本。こんにちは,これは自動ラベリングのテスト用テキストです.Hello, this is a piece of text to test autotagging.你好！今天我们要介绍VITS项目，其重点是使用了GAN Duration predictor和transformer flow,并且接入了Bert模型来提升韵律。Bert embedding会在稍后介绍。"
- print(split_by_language(text, ["zh", "ja", "en"]))
-
- text = "vits和Bert-VITS2是tts模型。花费3days.花费3天。Take 3 days"
-
- print(split_by_language(text, ["zh", "ja", "en"]))
- # output: [('vits', 'en'), ('和', 'ja'), ('Bert-VITS', 'en'), ('2是', 'zh'), ('tts', 'en'), ('模型。花费3', 'zh'), ('days.', 'en'), ('花费3天。', 'zh'), ('Take 3 days', 'en')]
-
- print(split_by_language(text, ["zh", "en"]))
- # output: [('vits', 'en'), ('和', 'zh'), ('Bert-VITS', 'en'), ('2是', 'zh'), ('tts', 'en'), ('模型。花费3', 'zh'), ('days.', 'en'), ('花费3天。', 'zh'), ('Take 3 days', 'en')]
-
- text = "vits 和 Bert-VITS2 是 tts 模型。花费 3 days. 花费 3天。Take 3 days"
- print(split_by_language(text, ["zh", "en"]))
- # output: [('vits ', 'en'), ('和 ', 'zh'), ('Bert-VITS2 ', 'en'), ('是 ', 'zh'), ('tts ', 'en'), ('模型。花费 ', 'zh'), ('3 days. ', 'en'), ('花费 3天。', 'zh'), ('Take 3 days', 'en')]
diff --git a/tools/translate.py b/tools/translate.py
deleted file mode 100644
index be0f7ea..0000000
--- a/tools/translate.py
+++ /dev/null
@@ -1,62 +0,0 @@
-"""
-翻译api
-"""
-
-from config import config
-
-import random
-import hashlib
-import requests
-
-
-def translate(Sentence: str, to_Language: str = "jp", from_Language: str = ""):
- """
- :param Sentence: 待翻译语句
- :param from_Language: 待翻译语句语言
- :param to_Language: 目标语言
- :return: 翻译后语句 出错时返回None
-
- 常见语言代码：中文 zh 英语 en 日语 jp
- """
- appid = config.translate_config.app_key
- key = config.translate_config.secret_key
- if appid == "" or key == "":
- return "请开发者在config.yml中配置app_key与secret_key"
- url = "https://fanyi-api.baidu.com/api/trans/vip/translate"
- texts = Sentence.splitlines()
- outTexts = []
- for t in texts:
- if t != "":
- # 签名计算 参考文档 https://api.fanyi.baidu.com/product/113
- salt = str(random.randint(1, 100000))
- signString = appid + t + salt + key
- hs = hashlib.md5()
- hs.update(signString.encode("utf-8"))
- signString = hs.hexdigest()
- if from_Language == "":
- from_Language = "auto"
- headers = {"Content-Type": "application/x-www-form-urlencoded"}
- payload = {
- "q": t,
- "from": from_Language,
- "to": to_Language,
- "appid": appid,
- "salt": salt,
- "sign": signString,
- }
- # 发送请求
- try:
- response = requests.post(
- url=url, data=payload, headers=headers, timeout=3
- )
- response = response.json()
- if "trans_result" in response.keys():
- result = response["trans_result"][0]
- if "dst" in result.keys():
- dst = result["dst"]
- outTexts.append(dst)
- except Exception:
- return Sentence
- else:
- outTexts.append(t)
- return "\n".join(outTexts)
diff --git a/train_ms.py b/train_ms.py
index 07b5100..c2538da 100644
--- a/train_ms.py
+++ b/train_ms.py
@@ -15,11 +15,7 @@ from torch.utils.tensorboard import SummaryWriter
from tqdm import tqdm
# logging.getLogger("numba").setLevel(logging.WARNING)
-import commons
import default_style
-import utils
-from common.log import logger
-from common.stdout_wrapper import SAFE_STDOUT
from config import config
from data_utils import (
DistributedBucketSampler,
@@ -28,8 +24,17 @@ from data_utils import (
)
from losses import discriminator_loss, feature_loss, generator_loss, kl_loss
from mel_processing import mel_spectrogram_torch, spec_to_mel_torch
-from models import DurationDiscriminator, MultiPeriodDiscriminator, SynthesizerTrn
-from text.symbols import symbols
+from style_bert_vits2.logging import logger
+from style_bert_vits2.models import commons, utils
+from style_bert_vits2.models.hyper_parameters import HyperParameters
+from style_bert_vits2.models.models import (
+ DurationDiscriminator,
+ MultiPeriodDiscriminator,
+ SynthesizerTrn,
+)
+from style_bert_vits2.nlp.symbols import SYMBOLS
+from style_bert_vits2.utils.stdout_wrapper import SAFE_STDOUT
+
torch.backends.cuda.matmul.allow_tf32 = True
torch.backends.cudnn.allow_tf32 = (
@@ -127,7 +132,7 @@ def run():
local_rank = int(os.environ["LOCAL_RANK"])
n_gpus = dist.get_world_size()
- hps = utils.get_hparams_from_file(args.config)
+ hps = HyperParameters.load_from_json(args.config)
# This is needed because we have to pass values to `train_and_evaluate()`
hps.model_dir = model_dir
hps.speedup = args.speedup
@@ -244,10 +249,7 @@ def run():
drop_last=False,
collate_fn=collate_fn,
)
- if (
- "use_noise_scaled_mas" in hps.model.keys()
- and hps.model.use_noise_scaled_mas is True
- ):
+ if hps.model.use_noise_scaled_mas is True:
logger.info("Using noise scaled MAS for VITS2")
mas_noise_scale_initial = 0.01
noise_scale_delta = 2e-6
@@ -255,10 +257,7 @@ def run():
logger.info("Using normal MAS for VITS1")
mas_noise_scale_initial = 0.0
noise_scale_delta = 0.0
- if (
- "use_duration_discriminator" in hps.model.keys()
- and hps.model.use_duration_discriminator is True
- ):
+ if hps.model.use_duration_discriminator is True:
logger.info("Using duration discriminator for VITS2")
net_dur_disc = DurationDiscriminator(
hps.model.hidden_channels,
@@ -267,10 +266,7 @@ def run():
0.1,
gin_channels=hps.model.gin_channels if hps.data.n_speakers != 0 else 0,
).cuda(local_rank)
- if (
- "use_spk_conditioned_encoder" in hps.model.keys()
- and hps.model.use_spk_conditioned_encoder is True
- ):
+ if hps.model.use_spk_conditioned_encoder is True:
if hps.data.n_speakers == 0:
raise ValueError(
"n_speakers must be > 0 when using spk conditioned encoder to train multi-speaker model"
@@ -279,13 +275,35 @@ def run():
logger.info("Using normal encoder for VITS1")
net_g = SynthesizerTrn(
- len(symbols),
+ len(SYMBOLS),
hps.data.filter_length // 2 + 1,
hps.train.segment_size // hps.data.hop_length,
n_speakers=hps.data.n_speakers,
mas_noise_scale_initial=mas_noise_scale_initial,
noise_scale_delta=noise_scale_delta,
- **hps.model,
+ # hps.model 以下のすべての値を引数に渡す
+ use_spk_conditioned_encoder=hps.model.use_spk_conditioned_encoder,
+ use_noise_scaled_mas=hps.model.use_noise_scaled_mas,
+ use_mel_posterior_encoder=hps.model.use_mel_posterior_encoder,
+ use_duration_discriminator=hps.model.use_duration_discriminator,
+ use_wavlm_discriminator=hps.model.use_wavlm_discriminator,
+ inter_channels=hps.model.inter_channels,
+ hidden_channels=hps.model.hidden_channels,
+ filter_channels=hps.model.filter_channels,
+ n_heads=hps.model.n_heads,
+ n_layers=hps.model.n_layers,
+ kernel_size=hps.model.kernel_size,
+ p_dropout=hps.model.p_dropout,
+ resblock=hps.model.resblock,
+ resblock_kernel_sizes=hps.model.resblock_kernel_sizes,
+ resblock_dilation_sizes=hps.model.resblock_dilation_sizes,
+ upsample_rates=hps.model.upsample_rates,
+ upsample_initial_channel=hps.model.upsample_initial_channel,
+ upsample_kernel_sizes=hps.model.upsample_kernel_sizes,
+ n_layers_q=hps.model.n_layers_q,
+ use_spectral_norm=hps.model.use_spectral_norm,
+ gin_channels=hps.model.gin_channels,
+ slm=hps.model.slm,
).cuda(local_rank)
if getattr(hps.train, "freeze_ZH_bert", False):
@@ -344,31 +362,25 @@ def run():
if utils.is_resuming(model_dir):
if net_dur_disc is not None:
- _, _, dur_resume_lr, epoch_str = utils.load_checkpoint(
- utils.latest_checkpoint_path(model_dir, "DUR_*.pth"),
+ _, _, dur_resume_lr, epoch_str = utils.checkpoints.load_checkpoint(
+ utils.checkpoints.get_latest_checkpoint_path(model_dir, "DUR_*.pth"),
net_dur_disc,
optim_dur_disc,
- skip_optimizer=(
- hps.train.skip_optimizer if "skip_optimizer" in hps.train else True
- ),
+ skip_optimizer=hps.train.skip_optimizer,
)
if not optim_dur_disc.param_groups[0].get("initial_lr"):
optim_dur_disc.param_groups[0]["initial_lr"] = dur_resume_lr
- _, optim_g, g_resume_lr, epoch_str = utils.load_checkpoint(
- utils.latest_checkpoint_path(model_dir, "G_*.pth"),
+ _, optim_g, g_resume_lr, epoch_str = utils.checkpoints.load_checkpoint(
+ utils.checkpoints.get_latest_checkpoint_path(model_dir, "G_*.pth"),
net_g,
optim_g,
- skip_optimizer=(
- hps.train.skip_optimizer if "skip_optimizer" in hps.train else True
- ),
+ skip_optimizer=hps.train.skip_optimizer,
)
- _, optim_d, d_resume_lr, epoch_str = utils.load_checkpoint(
- utils.latest_checkpoint_path(model_dir, "D_*.pth"),
+ _, optim_d, d_resume_lr, epoch_str = utils.checkpoints.load_checkpoint(
+ utils.checkpoints.get_latest_checkpoint_path(model_dir, "D_*.pth"),
net_d,
optim_d,
- skip_optimizer=(
- hps.train.skip_optimizer if "skip_optimizer" in hps.train else True
- ),
+ skip_optimizer=hps.train.skip_optimizer,
)
if not optim_g.param_groups[0].get("initial_lr"):
optim_g.param_groups[0]["initial_lr"] = g_resume_lr
@@ -378,21 +390,23 @@ def run():
epoch_str = max(epoch_str, 1)
# global_step = (epoch_str - 1) * len(train_loader)
global_step = int(
- utils.get_steps(utils.latest_checkpoint_path(model_dir, "G_*.pth"))
+ utils.get_steps(
+ utils.checkpoints.get_latest_checkpoint_path(model_dir, "G_*.pth")
+ )
)
logger.info(
f"******************Found the model. Current epoch is {epoch_str}, gloabl step is {global_step}*********************"
)
else:
try:
- _ = utils.load_safetensors(
+ _ = utils.safetensors.load_safetensors(
os.path.join(model_dir, "G_0.safetensors"), net_g
)
- _ = utils.load_safetensors(
+ _ = utils.safetensors.load_safetensors(
os.path.join(model_dir, "D_0.safetensors"), net_d
)
if net_dur_disc is not None:
- _ = utils.load_safetensors(
+ _ = utils.safetensors.load_safetensors(
os.path.join(model_dir, "DUR_0.safetensors"), net_dur_disc
)
logger.info("Loaded the pretrained models.")
@@ -485,14 +499,16 @@ def run():
if epoch == hps.train.epochs:
# Save the final models
- utils.save_checkpoint(
+ assert optim_g is not None
+ utils.checkpoints.save_checkpoint(
net_g,
optim_g,
hps.train.learning_rate,
epoch,
os.path.join(model_dir, "G_{}.pth".format(global_step)),
)
- utils.save_checkpoint(
+ assert optim_d is not None
+ utils.checkpoints.save_checkpoint(
net_d,
optim_d,
hps.train.learning_rate,
@@ -500,14 +516,15 @@ def run():
os.path.join(model_dir, "D_{}.pth".format(global_step)),
)
if net_dur_disc is not None:
- utils.save_checkpoint(
+ assert optim_dur_disc is not None
+ utils.checkpoints.save_checkpoint(
net_dur_disc,
optim_dur_disc,
hps.train.learning_rate,
epoch,
os.path.join(model_dir, "DUR_{}.pth".format(global_step)),
)
- utils.save_safetensors(
+ utils.safetensors.save_safetensors(
net_g,
epoch,
os.path.join(
@@ -544,7 +561,7 @@ def train_and_evaluate(
rank,
local_rank,
epoch,
- hps,
+ hps: HyperParameters,
nets,
optims,
schedulers,
@@ -778,14 +795,15 @@ def train_and_evaluate(
):
if not hps.speedup:
evaluate(hps, net_g, eval_loader, writer_eval)
- utils.save_checkpoint(
+ assert hps.model_dir is not None
+ utils.checkpoints.save_checkpoint(
net_g,
optim_g,
hps.train.learning_rate,
epoch,
os.path.join(hps.model_dir, "G_{}.pth".format(global_step)),
)
- utils.save_checkpoint(
+ utils.checkpoints.save_checkpoint(
net_d,
optim_d,
hps.train.learning_rate,
@@ -793,7 +811,7 @@ def train_and_evaluate(
os.path.join(hps.model_dir, "D_{}.pth".format(global_step)),
)
if net_dur_disc is not None:
- utils.save_checkpoint(
+ utils.checkpoints.save_checkpoint(
net_dur_disc,
optim_dur_disc,
hps.train.learning_rate,
@@ -802,13 +820,13 @@ def train_and_evaluate(
)
keep_ckpts = config.train_ms_config.keep_ckpts
if keep_ckpts > 0:
- utils.clean_checkpoints(
- path_to_models=hps.model_dir,
+ utils.checkpoints.clean_checkpoints(
+ model_dir_path=hps.model_dir,
n_ckpts_to_keep=keep_ckpts,
sort_by_time=True,
)
# Save safetensors (for inference) to `model_assets/{model_name}`
- utils.save_safetensors(
+ utils.safetensors.save_safetensors(
net_g,
epoch,
os.path.join(
diff --git a/train_ms_jp_extra.py b/train_ms_jp_extra.py
index bae922d..7b2cb42 100644
--- a/train_ms_jp_extra.py
+++ b/train_ms_jp_extra.py
@@ -6,20 +6,16 @@ import platform
import torch
import torch.distributed as dist
+from huggingface_hub import HfApi
from torch.cuda.amp import GradScaler, autocast
from torch.nn import functional as F
from torch.nn.parallel import DistributedDataParallel as DDP
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter
from tqdm import tqdm
-from huggingface_hub import HfApi
# logging.getLogger("numba").setLevel(logging.WARNING)
-import commons
import default_style
-import utils
-from common.log import logger
-from common.stdout_wrapper import SAFE_STDOUT
from config import config
from data_utils import (
DistributedBucketSampler,
@@ -28,13 +24,18 @@ from data_utils import (
)
from losses import WavLMLoss, discriminator_loss, feature_loss, generator_loss, kl_loss
from mel_processing import mel_spectrogram_torch, spec_to_mel_torch
-from models_jp_extra import (
+from style_bert_vits2.logging import logger
+from style_bert_vits2.models import commons, utils
+from style_bert_vits2.models.hyper_parameters import HyperParameters
+from style_bert_vits2.models.models_jp_extra import (
DurationDiscriminator,
MultiPeriodDiscriminator,
SynthesizerTrn,
WavLMDiscriminator,
)
-from text.symbols import symbols
+from style_bert_vits2.nlp.symbols import SYMBOLS
+from style_bert_vits2.utils.stdout_wrapper import SAFE_STDOUT
+
torch.backends.cuda.matmul.allow_tf32 = True
torch.backends.cudnn.allow_tf32 = (
@@ -130,7 +131,7 @@ def run():
local_rank = int(os.environ["LOCAL_RANK"])
n_gpus = dist.get_world_size()
- hps = utils.get_hparams_from_file(args.config)
+ hps = HyperParameters.load_from_json(args.config)
# This is needed because we have to pass values to `train_and_evaluate()
hps.model_dir = model_dir
hps.speedup = args.speedup
@@ -247,10 +248,7 @@ def run():
drop_last=False,
collate_fn=collate_fn,
)
- if (
- "use_noise_scaled_mas" in hps.model.keys()
- and hps.model.use_noise_scaled_mas is True
- ):
+ if hps.model.use_noise_scaled_mas is True:
logger.info("Using noise scaled MAS for VITS2")
mas_noise_scale_initial = 0.01
noise_scale_delta = 2e-6
@@ -258,10 +256,7 @@ def run():
logger.info("Using normal MAS for VITS1")
mas_noise_scale_initial = 0.0
noise_scale_delta = 0.0
- if (
- "use_duration_discriminator" in hps.model.keys()
- and hps.model.use_duration_discriminator is True
- ):
+ if hps.model.use_duration_discriminator is True:
logger.info("Using duration discriminator for VITS2")
net_dur_disc = DurationDiscriminator(
hps.model.hidden_channels,
@@ -272,19 +267,13 @@ def run():
).cuda(local_rank)
else:
net_dur_disc = None
- if (
- "use_wavlm_discriminator" in hps.model.keys()
- and hps.model.use_wavlm_discriminator is True
- ):
+ if hps.model.use_wavlm_discriminator is True:
net_wd = WavLMDiscriminator(
hps.model.slm.hidden, hps.model.slm.nlayers, hps.model.slm.initial_channel
).cuda(local_rank)
else:
net_wd = None
- if (
- "use_spk_conditioned_encoder" in hps.model.keys()
- and hps.model.use_spk_conditioned_encoder is True
- ):
+ if hps.model.use_spk_conditioned_encoder is True:
if hps.data.n_speakers == 0:
raise ValueError(
"n_speakers must be > 0 when using spk conditioned encoder to train multi-speaker model"
@@ -293,13 +282,35 @@ def run():
logger.info("Using normal encoder for VITS1")
net_g = SynthesizerTrn(
- len(symbols),
+ len(SYMBOLS),
hps.data.filter_length // 2 + 1,
hps.train.segment_size // hps.data.hop_length,
n_speakers=hps.data.n_speakers,
mas_noise_scale_initial=mas_noise_scale_initial,
noise_scale_delta=noise_scale_delta,
- **hps.model,
+ # hps.model 以下のすべての値を引数に渡す
+ use_spk_conditioned_encoder=hps.model.use_spk_conditioned_encoder,
+ use_noise_scaled_mas=hps.model.use_noise_scaled_mas,
+ use_mel_posterior_encoder=hps.model.use_mel_posterior_encoder,
+ use_duration_discriminator=hps.model.use_duration_discriminator,
+ use_wavlm_discriminator=hps.model.use_wavlm_discriminator,
+ inter_channels=hps.model.inter_channels,
+ hidden_channels=hps.model.hidden_channels,
+ filter_channels=hps.model.filter_channels,
+ n_heads=hps.model.n_heads,
+ n_layers=hps.model.n_layers,
+ kernel_size=hps.model.kernel_size,
+ p_dropout=hps.model.p_dropout,
+ resblock=hps.model.resblock,
+ resblock_kernel_sizes=hps.model.resblock_kernel_sizes,
+ resblock_dilation_sizes=hps.model.resblock_dilation_sizes,
+ upsample_rates=hps.model.upsample_rates,
+ upsample_initial_channel=hps.model.upsample_initial_channel,
+ upsample_kernel_sizes=hps.model.upsample_kernel_sizes,
+ n_layers_q=hps.model.n_layers_q,
+ use_spectral_norm=hps.model.use_spectral_norm,
+ gin_channels=hps.model.gin_channels,
+ slm=hps.model.slm,
).cuda(local_rank)
if getattr(hps.train, "freeze_JP_bert", False):
logger.info("Freezing (JP) bert encoder !!!")
@@ -372,15 +383,13 @@ def run():
if utils.is_resuming(model_dir):
if net_dur_disc is not None:
try:
- _, _, dur_resume_lr, epoch_str = utils.load_checkpoint(
- utils.latest_checkpoint_path(model_dir, "DUR_*.pth"),
+ _, _, dur_resume_lr, epoch_str = utils.checkpoints.load_checkpoint(
+ utils.checkpoints.get_latest_checkpoint_path(
+ model_dir, "DUR_*.pth"
+ ),
net_dur_disc,
optim_dur_disc,
- skip_optimizer=(
- hps.train.skip_optimizer
- if "skip_optimizer" in hps.train
- else True
- ),
+ skip_optimizer=hps.train.skip_optimizer,
)
if not optim_dur_disc.param_groups[0].get("initial_lr"):
optim_dur_disc.param_groups[0]["initial_lr"] = dur_resume_lr
@@ -390,15 +399,15 @@ def run():
print("Initialize dur_disc")
if net_wd is not None:
try:
- _, optim_wd, wd_resume_lr, epoch_str = utils.load_checkpoint(
- utils.latest_checkpoint_path(model_dir, "WD_*.pth"),
- net_wd,
- optim_wd,
- skip_optimizer=(
- hps.train.skip_optimizer
- if "skip_optimizer" in hps.train
- else True
- ),
+ _, optim_wd, wd_resume_lr, epoch_str = (
+ utils.checkpoints.load_checkpoint(
+ utils.checkpoints.get_latest_checkpoint_path(
+ model_dir, "WD_*.pth"
+ ),
+ net_wd,
+ optim_wd,
+ skip_optimizer=hps.train.skip_optimizer,
+ )
)
if not optim_wd.param_groups[0].get("initial_lr"):
optim_wd.param_groups[0]["initial_lr"] = wd_resume_lr
@@ -408,21 +417,17 @@ def run():
logger.info("Initialize wavlm")
try:
- _, optim_g, g_resume_lr, epoch_str = utils.load_checkpoint(
- utils.latest_checkpoint_path(model_dir, "G_*.pth"),
+ _, optim_g, g_resume_lr, epoch_str = utils.checkpoints.load_checkpoint(
+ utils.checkpoints.get_latest_checkpoint_path(model_dir, "G_*.pth"),
net_g,
optim_g,
- skip_optimizer=(
- hps.train.skip_optimizer if "skip_optimizer" in hps.train else True
- ),
+ skip_optimizer=hps.train.skip_optimizer,
)
- _, optim_d, d_resume_lr, epoch_str = utils.load_checkpoint(
- utils.latest_checkpoint_path(model_dir, "D_*.pth"),
+ _, optim_d, d_resume_lr, epoch_str = utils.checkpoints.load_checkpoint(
+ utils.checkpoints.get_latest_checkpoint_path(model_dir, "D_*.pth"),
net_d,
optim_d,
- skip_optimizer=(
- hps.train.skip_optimizer if "skip_optimizer" in hps.train else True
- ),
+ skip_optimizer=hps.train.skip_optimizer,
)
if not optim_g.param_groups[0].get("initial_lr"):
optim_g.param_groups[0]["initial_lr"] = g_resume_lr
@@ -432,7 +437,9 @@ def run():
epoch_str = max(epoch_str, 1)
# global_step = (epoch_str - 1) * len(train_loader)
global_step = int(
- utils.get_steps(utils.latest_checkpoint_path(model_dir, "G_*.pth"))
+ utils.get_steps(
+ utils.checkpoints.get_latest_checkpoint_path(model_dir, "G_*.pth")
+ )
)
logger.info(
f"******************Found the model. Current epoch is {epoch_str}, gloabl step is {global_step}*********************"
@@ -446,18 +453,18 @@ def run():
global_step = 0
else:
try:
- _ = utils.load_safetensors(
+ _ = utils.safetensors.load_safetensors(
os.path.join(model_dir, "G_0.safetensors"), net_g
)
- _ = utils.load_safetensors(
+ _ = utils.safetensors.load_safetensors(
os.path.join(model_dir, "D_0.safetensors"), net_d
)
if net_dur_disc is not None:
- _ = utils.load_safetensors(
+ _ = utils.safetensors.load_safetensors(
os.path.join(model_dir, "DUR_0.safetensors"), net_dur_disc
)
if net_wd is not None:
- _ = utils.load_safetensors(
+ _ = utils.safetensors.load_safetensors(
os.path.join(model_dir, "WD_0.safetensors"), net_wd
)
logger.info("Loaded the pretrained models.")
@@ -564,14 +571,16 @@ def run():
scheduler_wd.step()
if epoch == hps.train.epochs:
# Save the final models
- utils.save_checkpoint(
+ assert optim_g is not None
+ utils.checkpoints.save_checkpoint(
net_g,
optim_g,
hps.train.learning_rate,
epoch,
os.path.join(model_dir, "G_{}.pth".format(global_step)),
)
- utils.save_checkpoint(
+ assert optim_d is not None
+ utils.checkpoints.save_checkpoint(
net_d,
optim_d,
hps.train.learning_rate,
@@ -579,7 +588,8 @@ def run():
os.path.join(model_dir, "D_{}.pth".format(global_step)),
)
if net_dur_disc is not None:
- utils.save_checkpoint(
+ assert optim_dur_disc is not None
+ utils.checkpoints.save_checkpoint(
net_dur_disc,
optim_dur_disc,
hps.train.learning_rate,
@@ -587,14 +597,15 @@ def run():
os.path.join(model_dir, "DUR_{}.pth".format(global_step)),
)
if net_wd is not None:
- utils.save_checkpoint(
+ assert optim_wd is not None
+ utils.checkpoints.save_checkpoint(
net_wd,
optim_wd,
hps.train.learning_rate,
epoch,
os.path.join(model_dir, "WD_{}.pth".format(global_step)),
)
- utils.save_safetensors(
+ utils.safetensors.save_safetensors(
net_g,
epoch,
os.path.join(
@@ -927,14 +938,14 @@ def train_and_evaluate(
):
if not hps.speedup:
evaluate(hps, net_g, eval_loader, writer_eval)
- utils.save_checkpoint(
+ utils.checkpoints.save_checkpoint(
net_g,
optim_g,
hps.train.learning_rate,
epoch,
os.path.join(hps.model_dir, "G_{}.pth".format(global_step)),
)
- utils.save_checkpoint(
+ utils.checkpoints.save_checkpoint(
net_d,
optim_d,
hps.train.learning_rate,
@@ -942,7 +953,7 @@ def train_and_evaluate(
os.path.join(hps.model_dir, "D_{}.pth".format(global_step)),
)
if net_dur_disc is not None:
- utils.save_checkpoint(
+ utils.checkpoints.save_checkpoint(
net_dur_disc,
optim_dur_disc,
hps.train.learning_rate,
@@ -950,7 +961,7 @@ def train_and_evaluate(
os.path.join(hps.model_dir, "DUR_{}.pth".format(global_step)),
)
if net_wd is not None:
- utils.save_checkpoint(
+ utils.checkpoints.save_checkpoint(
net_wd,
optim_wd,
hps.train.learning_rate,
@@ -959,13 +970,13 @@ def train_and_evaluate(
)
keep_ckpts = config.train_ms_config.keep_ckpts
if keep_ckpts > 0:
- utils.clean_checkpoints(
- path_to_models=hps.model_dir,
+ utils.checkpoints.clean_checkpoints(
+ model_dir_path=hps.model_dir,
n_ckpts_to_keep=keep_ckpts,
sort_by_time=True,
)
# Save safetensors (for inference) to `model_assets/{model_name}`
- utils.save_safetensors(
+ utils.safetensors.save_safetensors(
net_g,
epoch,
os.path.join(
@@ -998,9 +1009,9 @@ def train_and_evaluate(
)
)
pbar.update()
- # 本家ではこれをスピードアップのために消すと書かれていたので、一応消してみる
- # gc.collect()
- # torch.cuda.empty_cache()
+
+ gc.collect()
+ torch.cuda.empty_cache()
if pbar is None and rank == 0:
logger.info(f"====> Epoch: {epoch}, step: {global_step}")
diff --git a/transcribe.py b/transcribe.py
index b3f9014..f44ecc3 100644
--- a/transcribe.py
+++ b/transcribe.py
@@ -2,17 +2,20 @@ import argparse
import os
import sys
from pathlib import Path
+from typing import Optional
import yaml
from faster_whisper import WhisperModel
from tqdm import tqdm
-from common.constants import Languages
-from common.log import logger
-from common.stdout_wrapper import SAFE_STDOUT
+from style_bert_vits2.constants import Languages
+from style_bert_vits2.logging import logger
+from style_bert_vits2.utils.stdout_wrapper import SAFE_STDOUT
-def transcribe(wav_path: Path, initial_prompt=None, language="ja"):
+def transcribe(
+ wav_path: Path, initial_prompt: Optional[str] = None, language: str = "ja"
+):
segments, _ = model.transcribe(
str(wav_path), beam_size=5, language=language, initial_prompt=initial_prompt
)
@@ -45,10 +48,10 @@ if __name__ == "__main__":
input_dir = dataset_root / model_name / "raw"
output_file = dataset_root / model_name / "esd.list"
- initial_prompt = args.initial_prompt
- language = args.language
- device = args.device
- compute_type = args.compute_type
+ initial_prompt: str = args.initial_prompt
+ language: str = args.language
+ device: str = args.device
+ compute_type: str = args.compute_type
output_file.parent.mkdir(parents=True, exist_ok=True)
diff --git a/update_status.py b/update_status.py
deleted file mode 100644
index 7d768c6..0000000
--- a/update_status.py
+++ /dev/null
@@ -1,93 +0,0 @@
-import os
-import gradio as gr
-
-lang_dict = {"EN(英文)": "_en", "ZH(中文)": "_zh", "JP(日语)": "_jp"}
-
-
-def raw_dir_convert_to_path(target_dir: str, lang):
- res = target_dir.rstrip("/").rstrip("\\")
- if (not target_dir.startswith("raw")) and (not target_dir.startswith("./raw")):
- res = os.path.join("./raw", res)
- if (
- (not res.endswith("_zh"))
- and (not res.endswith("_jp"))
- and (not res.endswith("_en"))
- ):
- res += lang_dict[lang]
- return res
-
-
-def update_g_files():
- g_files = []
- cnt = 0
- for root, dirs, files in os.walk(os.path.abspath("./logs")):
- for file in files:
- if file.startswith("G_") and file.endswith(".pth"):
- g_files.append(os.path.join(root, file))
- cnt += 1
- print(g_files)
- return f"更新模型列表完成, 共找到{cnt}个模型", gr.Dropdown.update(choices=g_files)
-
-
-def update_c_files():
- c_files = []
- cnt = 0
- for root, dirs, files in os.walk(os.path.abspath("./logs")):
- for file in files:
- if file.startswith("config.json"):
- c_files.append(os.path.join(root, file))
- cnt += 1
- print(c_files)
- return f"更新模型列表完成, 共找到{cnt}个配置文件", gr.Dropdown.update(
- choices=c_files
- )
-
-
-def update_model_folders():
- subdirs = []
- cnt = 0
- for root, dirs, files in os.walk(os.path.abspath("./logs")):
- for dir_name in dirs:
- if os.path.basename(dir_name) != "eval":
- subdirs.append(os.path.join(root, dir_name))
- cnt += 1
- print(subdirs)
- return f"更新模型文件夹列表完成, 共找到{cnt}个文件夹", gr.Dropdown.update(
- choices=subdirs
- )
-
-
-def update_wav_lab_pairs():
- wav_count = tot_count = 0
- for root, _, files in os.walk("./raw"):
- for file in files:
- # print(file)
- file_path = os.path.join(root, file)
- if file.lower().endswith(".wav"):
- lab_file = os.path.splitext(file_path)[0] + ".lab"
- if os.path.exists(lab_file):
- wav_count += 1
- tot_count += 1
- return f"{wav_count} / {tot_count}"
-
-
-def update_raw_folders():
- subdirs = []
- cnt = 0
- script_path = os.path.dirname(os.path.abspath(__file__)) # 获取当前脚本的绝对路径
- raw_path = os.path.join(script_path, "raw")
- print(raw_path)
- os.makedirs(raw_path, exist_ok=True)
- for root, dirs, files in os.walk(raw_path):
- for dir_name in dirs:
- relative_path = os.path.relpath(
- os.path.join(root, dir_name), script_path
- ) # 获取相对路径
- subdirs.append(relative_path)
- cnt += 1
- print(subdirs)
- return (
- f"更新raw音频文件夹列表完成, 共找到{cnt}个文件夹",
- gr.Dropdown.update(choices=subdirs),
- gr.Textbox.update(value=update_wav_lab_pairs()),
- )
diff --git a/utils.py b/utils.py
deleted file mode 100644
index ca194bf..0000000
--- a/utils.py
+++ /dev/null
@@ -1,501 +0,0 @@
-import argparse
-import glob
-import json
-import logging
-import os
-import re
-import subprocess
-
-import numpy as np
-import torch
-from huggingface_hub import hf_hub_download
-from safetensors import safe_open
-from safetensors.torch import save_file
-from scipy.io.wavfile import read
-
-from common.log import logger
-
-MATPLOTLIB_FLAG = False
-
-
-def download_checkpoint(
- dir_path, repo_config, token=None, regex="G_*.pth", mirror="openi"
-):
- repo_id = repo_config["repo_id"]
- f_list = glob.glob(os.path.join(dir_path, regex))
- if f_list:
- print("Use existed model, skip downloading.")
- return
- for file in ["DUR_0.pth", "D_0.pth", "G_0.pth"]:
- hf_hub_download(repo_id, file, local_dir=dir_path, local_dir_use_symlinks=False)
-
-
-def load_checkpoint(
- checkpoint_path, model, optimizer=None, skip_optimizer=False, for_infer=False
-):
- assert os.path.isfile(checkpoint_path)
- checkpoint_dict = torch.load(checkpoint_path, map_location="cpu")
- iteration = checkpoint_dict["iteration"]
- learning_rate = checkpoint_dict["learning_rate"]
- logger.info(
- f"Loading model and optimizer at iteration {iteration} from {checkpoint_path}"
- )
- if (
- optimizer is not None
- and not skip_optimizer
- and checkpoint_dict["optimizer"] is not None
- ):
- optimizer.load_state_dict(checkpoint_dict["optimizer"])
- elif optimizer is None and not skip_optimizer:
- # else: Disable this line if Infer and resume checkpoint,then enable the line upper
- new_opt_dict = optimizer.state_dict()
- new_opt_dict_params = new_opt_dict["param_groups"][0]["params"]
- new_opt_dict["param_groups"] = checkpoint_dict["optimizer"]["param_groups"]
- new_opt_dict["param_groups"][0]["params"] = new_opt_dict_params
- optimizer.load_state_dict(new_opt_dict)
-
- saved_state_dict = checkpoint_dict["model"]
- if hasattr(model, "module"):
- state_dict = model.module.state_dict()
- else:
- state_dict = model.state_dict()
-
- new_state_dict = {}
- for k, v in state_dict.items():
- try:
- # assert "emb_g" not in k
- new_state_dict[k] = saved_state_dict[k]
- assert saved_state_dict[k].shape == v.shape, (
- saved_state_dict[k].shape,
- v.shape,
- )
- except:
- # For upgrading from the old version
- if "ja_bert_proj" in k:
- v = torch.zeros_like(v)
- logger.warning(
- f"Seems you are using the old version of the model, the {k} is automatically set to zero for backward compatibility"
- )
- elif "enc_q" in k and for_infer:
- continue
- else:
- logger.error(f"{k} is not in the checkpoint {checkpoint_path}")
-
- new_state_dict[k] = v
-
- if hasattr(model, "module"):
- model.module.load_state_dict(new_state_dict, strict=False)
- else:
- model.load_state_dict(new_state_dict, strict=False)
-
- logger.info("Loaded '{}' (iteration {})".format(checkpoint_path, iteration))
-
- return model, optimizer, learning_rate, iteration
-
-
-def save_checkpoint(model, optimizer, learning_rate, iteration, checkpoint_path):
- logger.info(
- "Saving model and optimizer state at iteration {} to {}".format(
- iteration, checkpoint_path
- )
- )
- if hasattr(model, "module"):
- state_dict = model.module.state_dict()
- else:
- state_dict = model.state_dict()
- torch.save(
- {
- "model": state_dict,
- "iteration": iteration,
- "optimizer": optimizer.state_dict(),
- "learning_rate": learning_rate,
- },
- checkpoint_path,
- )
-
-
-def save_safetensors(model, iteration, checkpoint_path, is_half=False, for_infer=False):
- """
- Save model with safetensors.
- """
- if hasattr(model, "module"):
- state_dict = model.module.state_dict()
- else:
- state_dict = model.state_dict()
- keys = []
- for k in state_dict:
- if "enc_q" in k and for_infer:
- continue # noqa: E701
- keys.append(k)
-
- new_dict = (
- {k: state_dict[k].half() for k in keys}
- if is_half
- else {k: state_dict[k] for k in keys}
- )
- new_dict["iteration"] = torch.LongTensor([iteration])
- logger.info(f"Saved safetensors to {checkpoint_path}")
- save_file(new_dict, checkpoint_path)
-
-
-def load_safetensors(checkpoint_path, model, for_infer=False):
- """
- Load safetensors model.
- """
-
- tensors = {}
- iteration = None
- with safe_open(checkpoint_path, framework="pt", device="cpu") as f:
- for key in f.keys():
- if key == "iteration":
- iteration = f.get_tensor(key).item()
- tensors[key] = f.get_tensor(key)
- if hasattr(model, "module"):
- result = model.module.load_state_dict(tensors, strict=False)
- else:
- result = model.load_state_dict(tensors, strict=False)
- for key in result.missing_keys:
- if key.startswith("enc_q") and for_infer:
- continue
- logger.warning(f"Missing key: {key}")
- for key in result.unexpected_keys:
- if key == "iteration":
- continue
- logger.warning(f"Unexpected key: {key}")
- if iteration is None:
- logger.info(f"Loaded '{checkpoint_path}'")
- else:
- logger.info(f"Loaded '{checkpoint_path}' (iteration {iteration})")
- return model, iteration
-
-
-def summarize(
- writer,
- global_step,
- scalars={},
- histograms={},
- images={},
- audios={},
- audio_sampling_rate=22050,
-):
- for k, v in scalars.items():
- writer.add_scalar(k, v, global_step)
- for k, v in histograms.items():
- writer.add_histogram(k, v, global_step)
- for k, v in images.items():
- writer.add_image(k, v, global_step, dataformats="HWC")
- for k, v in audios.items():
- writer.add_audio(k, v, global_step, audio_sampling_rate)
-
-
-def is_resuming(dir_path):
- # JP-ExtraバージョンではDURがなくWDがあったり変わるため、Gのみで判断する
- g_list = glob.glob(os.path.join(dir_path, "G_*.pth"))
- # d_list = glob.glob(os.path.join(dir_path, "D_*.pth"))
- # dur_list = glob.glob(os.path.join(dir_path, "DUR_*.pth"))
- return len(g_list) > 0
-
-
-def latest_checkpoint_path(dir_path, regex="G_*.pth"):
- f_list = glob.glob(os.path.join(dir_path, regex))
- f_list.sort(key=lambda f: int("".join(filter(str.isdigit, f))))
- try:
- x = f_list[-1]
- except IndexError:
- raise ValueError(f"No checkpoint found in {dir_path} with regex {regex}")
- return x
-
-
-def plot_spectrogram_to_numpy(spectrogram):
- global MATPLOTLIB_FLAG
- if not MATPLOTLIB_FLAG:
- import matplotlib
-
- matplotlib.use("Agg")
- MATPLOTLIB_FLAG = True
- mpl_logger = logging.getLogger("matplotlib")
- mpl_logger.setLevel(logging.WARNING)
- import matplotlib.pylab as plt
- import numpy as np
-
- fig, ax = plt.subplots(figsize=(10, 2))
- im = ax.imshow(spectrogram, aspect="auto", origin="lower", interpolation="none")
- plt.colorbar(im, ax=ax)
- plt.xlabel("Frames")
- plt.ylabel("Channels")
- plt.tight_layout()
-
- fig.canvas.draw()
- data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep="")
- data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
- plt.close()
- return data
-
-
-def plot_alignment_to_numpy(alignment, info=None):
- global MATPLOTLIB_FLAG
- if not MATPLOTLIB_FLAG:
- import matplotlib
-
- matplotlib.use("Agg")
- MATPLOTLIB_FLAG = True
- mpl_logger = logging.getLogger("matplotlib")
- mpl_logger.setLevel(logging.WARNING)
- import matplotlib.pylab as plt
- import numpy as np
-
- fig, ax = plt.subplots(figsize=(6, 4))
- im = ax.imshow(
- alignment.transpose(), aspect="auto", origin="lower", interpolation="none"
- )
- fig.colorbar(im, ax=ax)
- xlabel = "Decoder timestep"
- if info is not None:
- xlabel += "\n\n" + info
- plt.xlabel(xlabel)
- plt.ylabel("Encoder timestep")
- plt.tight_layout()
-
- fig.canvas.draw()
- data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep="")
- data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
- plt.close()
- return data
-
-
-def load_wav_to_torch(full_path):
- sampling_rate, data = read(full_path)
- return torch.FloatTensor(data.astype(np.float32)), sampling_rate
-
-
-def load_filepaths_and_text(filename, split="|"):
- with open(filename, encoding="utf-8") as f:
- filepaths_and_text = [line.strip().split(split) for line in f]
- return filepaths_and_text
-
-
-def get_hparams(init=True):
- parser = argparse.ArgumentParser()
- parser.add_argument(
- "-c",
- "--config",
- type=str,
- default="./configs/base.json",
- help="JSON file for configuration",
- )
- parser.add_argument("-m", "--model", type=str, required=True, help="Model name")
-
- args = parser.parse_args()
- model_dir = os.path.join("./logs", args.model)
-
- if not os.path.exists(model_dir):
- os.makedirs(model_dir)
-
- config_path = args.config
- config_save_path = os.path.join(model_dir, "config.json")
- if init:
- with open(config_path, "r", encoding="utf-8") as f:
- data = f.read()
- with open(config_save_path, "w", encoding="utf-8") as f:
- f.write(data)
- else:
- with open(config_save_path, "r", vencoding="utf-8") as f:
- data = f.read()
- config = json.loads(data)
- hparams = HParams(**config)
- hparams.model_dir = model_dir
- return hparams
-
-
-def clean_checkpoints(path_to_models="logs/44k/", n_ckpts_to_keep=2, sort_by_time=True):
- """Freeing up space by deleting saved ckpts
-
- Arguments:
- path_to_models -- Path to the model directory
- n_ckpts_to_keep -- Number of ckpts to keep, excluding G_0.pth and D_0.pth
- sort_by_time -- True -> chronologically delete ckpts
- False -> lexicographically delete ckpts
- """
- import re
-
- ckpts_files = [
- f
- for f in os.listdir(path_to_models)
- if os.path.isfile(os.path.join(path_to_models, f))
- ]
-
- def name_key(_f):
- return int(re.compile("._(\\d+)\\.pth").match(_f).group(1))
-
- def time_key(_f):
- return os.path.getmtime(os.path.join(path_to_models, _f))
-
- sort_key = time_key if sort_by_time else name_key
-
- def x_sorted(_x):
- return sorted(
- [f for f in ckpts_files if f.startswith(_x) and not f.endswith("_0.pth")],
- key=sort_key,
- )
-
- to_del = [
- os.path.join(path_to_models, fn)
- for fn in (
- x_sorted("G_")[:-n_ckpts_to_keep]
- + x_sorted("D_")[:-n_ckpts_to_keep]
- + x_sorted("WD_")[:-n_ckpts_to_keep]
- + x_sorted("DUR_")[:-n_ckpts_to_keep]
- )
- ]
-
- def del_info(fn):
- return logger.info(f"Free up space by deleting ckpt {fn}")
-
- def del_routine(x):
- return [os.remove(x), del_info(x)]
-
- [del_routine(fn) for fn in to_del]
-
-
-def get_hparams_from_dir(model_dir):
- config_save_path = os.path.join(model_dir, "config.json")
- with open(config_save_path, "r", encoding="utf-8") as f:
- data = f.read()
- config = json.loads(data)
-
- hparams = HParams(**config)
- hparams.model_dir = model_dir
- return hparams
-
-
-def get_hparams_from_file(config_path):
- # print("config_path: ", config_path)
- with open(config_path, "r", encoding="utf-8") as f:
- data = f.read()
- config = json.loads(data)
-
- hparams = HParams(**config)
- return hparams
-
-
-def check_git_hash(model_dir):
- source_dir = os.path.dirname(os.path.realpath(__file__))
- if not os.path.exists(os.path.join(source_dir, ".git")):
- logger.warning(
- "{} is not a git repository, therefore hash value comparison will be ignored.".format(
- source_dir
- )
- )
- return
-
- cur_hash = subprocess.getoutput("git rev-parse HEAD")
-
- path = os.path.join(model_dir, "githash")
- if os.path.exists(path):
- saved_hash = open(path).read()
- if saved_hash != cur_hash:
- logger.warning(
- "git hash values are different. {}(saved) != {}(current)".format(
- saved_hash[:8], cur_hash[:8]
- )
- )
- else:
- open(path, "w").write(cur_hash)
-
-
-def get_logger(model_dir, filename="train.log"):
- global logger
- logger = logging.getLogger(os.path.basename(model_dir))
- logger.setLevel(logging.DEBUG)
-
- formatter = logging.Formatter("%(asctime)s\t%(name)s\t%(levelname)s\t%(message)s")
- if not os.path.exists(model_dir):
- os.makedirs(model_dir)
- h = logging.FileHandler(os.path.join(model_dir, filename))
- h.setLevel(logging.DEBUG)
- h.setFormatter(formatter)
- logger.addHandler(h)
- return logger
-
-
-class HParams:
- def __init__(self, **kwargs):
- for k, v in kwargs.items():
- if type(v) == dict:
- v = HParams(**v)
- self[k] = v
-
- def keys(self):
- return self.__dict__.keys()
-
- def items(self):
- return self.__dict__.items()
-
- def values(self):
- return self.__dict__.values()
-
- def __len__(self):
- return len(self.__dict__)
-
- def __getitem__(self, key):
- return getattr(self, key)
-
- def __setitem__(self, key, value):
- return setattr(self, key, value)
-
- def __contains__(self, key):
- return key in self.__dict__
-
- def __repr__(self):
- return self.__dict__.__repr__()
-
-
-def load_model(model_path, config_path):
- hps = get_hparams_from_file(config_path)
- net = SynthesizerTrn(
- # len(symbols),
- 108,
- hps.data.filter_length // 2 + 1,
- hps.train.segment_size // hps.data.hop_length,
- n_speakers=hps.data.n_speakers,
- **hps.model,
- ).to("cpu")
- _ = net.eval()
- _ = load_checkpoint(model_path, net, None, skip_optimizer=True)
- return net
-
-
-def mix_model(
- network1, network2, output_path, voice_ratio=(0.5, 0.5), tone_ratio=(0.5, 0.5)
-):
- if hasattr(network1, "module"):
- state_dict1 = network1.module.state_dict()
- state_dict2 = network2.module.state_dict()
- else:
- state_dict1 = network1.state_dict()
- state_dict2 = network2.state_dict()
- for k in state_dict1.keys():
- if k not in state_dict2.keys():
- continue
- if "enc_p" in k:
- state_dict1[k] = (
- state_dict1[k].clone() * tone_ratio[0]
- + state_dict2[k].clone() * tone_ratio[1]
- )
- else:
- state_dict1[k] = (
- state_dict1[k].clone() * voice_ratio[0]
- + state_dict2[k].clone() * voice_ratio[1]
- )
- for k in state_dict2.keys():
- if k not in state_dict1.keys():
- state_dict1[k] = state_dict2[k].clone()
- torch.save(
- {"model": state_dict1, "iteration": 0, "optimizer": None, "learning_rate": 0},
- output_path,
- )
-
-
-def get_steps(model_path):
- matches = re.findall(r"\d+", model_path)
- return matches[-1] if matches else None
diff --git a/webui.py b/webui.py
deleted file mode 100644
index 90318a1..0000000
--- a/webui.py
+++ /dev/null
@@ -1,558 +0,0 @@
-"""
-Original `webui.py` for Bert-VITS2, not working with Style-Bert-VITS2 yet.
-"""
-
-# flake8: noqa: E402
-import os
-import logging
-import re_matching
-from tools.sentence import split_by_language
-
-logging.getLogger("numba").setLevel(logging.WARNING)
-logging.getLogger("markdown_it").setLevel(logging.WARNING)
-logging.getLogger("urllib3").setLevel(logging.WARNING)
-logging.getLogger("matplotlib").setLevel(logging.WARNING)
-
-logging.basicConfig(
- level=logging.INFO, format="| %(name)s | %(levelname)s | %(message)s"
-)
-
-logger = logging.getLogger(__name__)
-
-import torch
-import utils
-from infer import infer, latest_version, get_net_g, infer_multilang
-import gradio as gr
-import webbrowser
-import numpy as np
-from config import config
-from tools.translate import translate
-import librosa
-
-net_g = None
-
-device = config.webui_config.device
-if device == "mps":
- os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
-
-
-def generate_audio(
- slices,
- sdp_ratio,
- noise_scale,
- noise_scale_w,
- length_scale,
- speaker,
- language,
- reference_audio,
- emotion,
- style_text,
- style_weight,
- skip_start=False,
- skip_end=False,
-):
- audio_list = []
- # silence = np.zeros(hps.data.sampling_rate // 2, dtype=np.int16)
- with torch.no_grad():
- for idx, piece in enumerate(slices):
- skip_start = idx != 0
- skip_end = idx != len(slices) - 1
- audio = infer(
- piece,
- reference_audio=reference_audio,
- emotion=emotion,
- sdp_ratio=sdp_ratio,
- noise_scale=noise_scale,
- noise_scale_w=noise_scale_w,
- length_scale=length_scale,
- sid=speaker,
- language=language,
- hps=hps,
- net_g=net_g,
- device=device,
- skip_start=skip_start,
- skip_end=skip_end,
- assist_text=style_text,
- assist_text_weight=style_weight,
- )
- audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
- audio_list.append(audio16bit)
- return audio_list
-
-
-def generate_audio_multilang(
- slices,
- sdp_ratio,
- noise_scale,
- noise_scale_w,
- length_scale,
- speaker,
- language,
- reference_audio,
- emotion,
- skip_start=False,
- skip_end=False,
-):
- audio_list = []
- # silence = np.zeros(hps.data.sampling_rate // 2, dtype=np.int16)
- with torch.no_grad():
- for idx, piece in enumerate(slices):
- skip_start = idx != 0
- skip_end = idx != len(slices) - 1
- audio = infer_multilang(
- piece,
- reference_audio=reference_audio,
- emotion=emotion,
- sdp_ratio=sdp_ratio,
- noise_scale=noise_scale,
- noise_scale_w=noise_scale_w,
- length_scale=length_scale,
- sid=speaker,
- language=language[idx],
- hps=hps,
- net_g=net_g,
- device=device,
- skip_start=skip_start,
- skip_end=skip_end,
- )
- audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
- audio_list.append(audio16bit)
- return audio_list
-
-
-def tts_split(
- text: str,
- speaker,
- sdp_ratio,
- noise_scale,
- noise_scale_w,
- length_scale,
- language,
- cut_by_sent,
- interval_between_para,
- interval_between_sent,
- reference_audio,
- emotion,
- style_text,
- style_weight,
-):
- while text.find("\n\n") != -1:
- text = text.replace("\n\n", "\n")
- text = text.replace("|", "")
- para_list = re_matching.cut_para(text)
- para_list = [p for p in para_list if p != ""]
- audio_list = []
- for p in para_list:
- if not cut_by_sent:
- audio_list += process_text(
- p,
- speaker,
- sdp_ratio,
- noise_scale,
- noise_scale_w,
- length_scale,
- language,
- reference_audio,
- emotion,
- style_text,
- style_weight,
- )
- silence = np.zeros((int)(44100 * interval_between_para), dtype=np.int16)
- audio_list.append(silence)
- else:
- audio_list_sent = []
- sent_list = re_matching.cut_sent(p)
- sent_list = [s for s in sent_list if s != ""]
- for s in sent_list:
- audio_list_sent += process_text(
- s,
- speaker,
- sdp_ratio,
- noise_scale,
- noise_scale_w,
- length_scale,
- language,
- reference_audio,
- emotion,
- style_text,
- style_weight,
- )
- silence = np.zeros((int)(44100 * interval_between_sent))
- audio_list_sent.append(silence)
- if (interval_between_para - interval_between_sent) > 0:
- silence = np.zeros(
- (int)(44100 * (interval_between_para - interval_between_sent))
- )
- audio_list_sent.append(silence)
- audio16bit = gr.processing_utils.convert_to_16_bit_wav(
- np.concatenate(audio_list_sent)
- ) # 对完整句子做音量归一
- audio_list.append(audio16bit)
- audio_concat = np.concatenate(audio_list)
- return ("Success", (hps.data.sampling_rate, audio_concat))
-
-
-def process_mix(slice):
- _speaker = slice.pop()
- _text, _lang = [], []
- for lang, content in slice:
- content = content.split("|")
- content = [part for part in content if part != ""]
- if len(content) == 0:
- continue
- if len(_text) == 0:
- _text = [[part] for part in content]
- _lang = [[lang] for part in content]
- else:
- _text[-1].append(content[0])
- _lang[-1].append(lang)
- if len(content) > 1:
- _text += [[part] for part in content[1:]]
- _lang += [[lang] for part in content[1:]]
- return _text, _lang, _speaker
-
-
-def process_auto(text):
- _text, _lang = [], []
- for slice in text.split("|"):
- if slice == "":
- continue
- temp_text, temp_lang = [], []
- sentences_list = split_by_language(slice, target_languages=["zh", "ja", "en"])
- for sentence, lang in sentences_list:
- if sentence == "":
- continue
- temp_text.append(sentence)
- if lang == "ja":
- lang = "jp"
- temp_lang.append(lang.upper())
- _text.append(temp_text)
- _lang.append(temp_lang)
- return _text, _lang
-
-
-def process_text(
- text: str,
- speaker,
- sdp_ratio,
- noise_scale,
- noise_scale_w,
- length_scale,
- language,
- reference_audio,
- emotion,
- style_text=None,
- style_weight=0,
-):
- audio_list = []
- if language == "mix":
- bool_valid, str_valid = re_matching.validate_text(text)
- if not bool_valid:
- return str_valid, (
- hps.data.sampling_rate,
- np.concatenate([np.zeros(hps.data.sampling_rate // 2)]),
- )
- for slice in re_matching.text_matching(text):
- _text, _lang, _speaker = process_mix(slice)
- if _speaker is None:
- continue
- print(f"Text: {_text}\nLang: {_lang}")
- audio_list.extend(
- generate_audio_multilang(
- _text,
- sdp_ratio,
- noise_scale,
- noise_scale_w,
- length_scale,
- _speaker,
- _lang,
- reference_audio,
- emotion,
- )
- )
- elif language.lower() == "auto":
- _text, _lang = process_auto(text)
- print(f"Text: {_text}\nLang: {_lang}")
- audio_list.extend(
- generate_audio_multilang(
- _text,
- sdp_ratio,
- noise_scale,
- noise_scale_w,
- length_scale,
- speaker,
- _lang,
- reference_audio,
- emotion,
- )
- )
- else:
- audio_list.extend(
- generate_audio(
- text.split("|"),
- sdp_ratio,
- noise_scale,
- noise_scale_w,
- length_scale,
- speaker,
- language,
- reference_audio,
- emotion,
- style_text,
- style_weight,
- )
- )
- return audio_list
-
-
-def tts_fn(
- text: str,
- speaker,
- sdp_ratio,
- noise_scale,
- noise_scale_w,
- length_scale,
- language,
- reference_audio,
- emotion,
- prompt_mode,
- style_text=None,
- style_weight=0,
-):
- if style_text == "":
- style_text = None
- if prompt_mode == "Audio prompt":
- if reference_audio == None:
- return ("Invalid audio prompt", None)
- else:
- reference_audio = load_audio(reference_audio)[1]
- else:
- reference_audio = None
-
- audio_list = process_text(
- text,
- speaker,
- sdp_ratio,
- noise_scale,
- noise_scale_w,
- length_scale,
- language,
- reference_audio,
- emotion,
- style_text,
- style_weight,
- )
-
- audio_concat = np.concatenate(audio_list)
- return "Success", (hps.data.sampling_rate, audio_concat)
-
-
-def format_utils(text, speaker):
- _text, _lang = process_auto(text)
- res = f"[{speaker}]"
- for lang_s, content_s in zip(_lang, _text):
- for lang, content in zip(lang_s, content_s):
- res += f"<{lang.lower()}>{content}"
- res += "|"
- return "mix", res[:-1]
-
-
-def load_audio(path):
- audio, sr = librosa.load(path, 48000)
- # audio = librosa.resample(audio, 44100, 48000)
- return sr, audio
-
-
-def gr_util(item):
- if item == "Text prompt":
- return {"visible": True, "__type__": "update"}, {
- "visible": False,
- "__type__": "update",
- }
- else:
- return {"visible": False, "__type__": "update"}, {
- "visible": True,
- "__type__": "update",
- }
-
-
-if __name__ == "__main__":
- if config.webui_config.debug:
- logger.info("Enable DEBUG-LEVEL log")
- logging.basicConfig(level=logging.DEBUG)
- hps = utils.get_hparams_from_file(config.webui_config.config_path)
- # 若config.json中未指定版本则默认为最新版本
- version = hps.version if hasattr(hps, "version") else latest_version
- net_g = get_net_g(
- model_path=config.webui_config.model, version=version, device=device, hps=hps
- )
- speaker_ids = hps.data.spk2id
- speakers = list(speaker_ids.keys())
- languages = ["ZH", "JP", "EN", "mix", "auto"]
- with gr.Blocks() as app:
- with gr.Row():
- with gr.Column():
- text = gr.TextArea(
- label="输入文本内容",
- placeholder="""
- 如果你选择语言为\'mix\'，必须按照格式输入，否则报错:
- 格式举例(zh是中文，jp是日语，不区分大小写；说话人举例:gongzi):
- [说话人1]你好，こんにちは！ こんにちは，世界。
- [说话人2]你好吗？元気ですか？
- [说话人3]谢谢。どういたしまして。
- ...
- 另外，所有的语言选项都可以用'|'分割长段实现分句生成。
- """,
- )
- trans = gr.Button("中翻日", variant="primary")
- slicer = gr.Button("快速切分", variant="primary")
- formatter = gr.Button("检测语言，并整理为 MIX 格式", variant="primary")
- speaker = gr.Dropdown(
- choices=speakers, value=speakers[0], label="Speaker"
- )
- _ = gr.Markdown(
- value="提示模式（Prompt mode）：可选文字提示或音频提示，用于生成文字或音频指定风格的声音。\n",
- visible=False,
- )
- prompt_mode = gr.Radio(
- ["Text prompt", "Audio prompt"],
- label="Prompt Mode",
- value="Text prompt",
- visible=False,
- )
- text_prompt = gr.Textbox(
- label="Text prompt",
- placeholder="用文字描述生成风格。如：Happy",
- value="Happy",
- visible=False,
- )
- audio_prompt = gr.Audio(
- label="Audio prompt", type="filepath", visible=False
- )
- sdp_ratio = gr.Slider(
- minimum=0, maximum=1, value=0.5, step=0.1, label="SDP Ratio"
- )
- noise_scale = gr.Slider(
- minimum=0.1, maximum=2, value=0.6, step=0.1, label="Noise"
- )
- noise_scale_w = gr.Slider(
- minimum=0.1, maximum=2, value=0.9, step=0.1, label="Noise_W"
- )
- length_scale = gr.Slider(
- minimum=0.1, maximum=2, value=1.0, step=0.1, label="Length"
- )
- language = gr.Dropdown(
- choices=languages, value=languages[0], label="Language"
- )
- btn = gr.Button("生成音频！", variant="primary")
- with gr.Column():
- with gr.Accordion("融合文本语义", open=False):
- gr.Markdown(
- value="使用辅助文本的语意来辅助生成对话（语言保持与主文本相同）\n\n"
- "**注意**：不要使用**指令式文本**（如：开心），要使用**带有强烈情感的文本**（如：我好快乐！！！）\n\n"
- "效果较不明确，留空即为不使用该功能"
- )
- style_text = gr.Textbox(label="辅助文本")
- style_weight = gr.Slider(
- minimum=0,
- maximum=1,
- value=0.7,
- step=0.1,
- label="Weight",
- info="主文本和辅助文本的bert混合比率，0表示仅主文本，1表示仅辅助文本",
- )
- with gr.Row():
- with gr.Column():
- interval_between_sent = gr.Slider(
- minimum=0,
- maximum=5,
- value=0.2,
- step=0.1,
- label="句间停顿(秒)，勾选按句切分才生效",
- )
- interval_between_para = gr.Slider(
- minimum=0,
- maximum=10,
- value=1,
- step=0.1,
- label="段间停顿(秒)，需要大于句间停顿才有效",
- )
- opt_cut_by_sent = gr.Checkbox(
- label="按句切分 在按段落切分的基础上再按句子切分文本"
- )
- slicer = gr.Button("切分生成", variant="primary")
- text_output = gr.Textbox(label="状态信息")
- audio_output = gr.Audio(label="输出音频")
- # explain_image = gr.Image(
- # label="参数解释信息",
- # show_label=True,
- # show_share_button=False,
- # show_download_button=False,
- # value=os.path.abspath("./img/参数说明.png"),
- # )
- btn.click(
- tts_fn,
- inputs=[
- text,
- speaker,
- sdp_ratio,
- noise_scale,
- noise_scale_w,
- length_scale,
- language,
- audio_prompt,
- text_prompt,
- prompt_mode,
- style_text,
- style_weight,
- ],
- outputs=[text_output, audio_output],
- )
-
- trans.click(
- translate,
- inputs=[text],
- outputs=[text],
- )
- slicer.click(
- tts_split,
- inputs=[
- text,
- speaker,
- sdp_ratio,
- noise_scale,
- noise_scale_w,
- length_scale,
- language,
- opt_cut_by_sent,
- interval_between_para,
- interval_between_sent,
- audio_prompt,
- text_prompt,
- style_text,
- style_weight,
- ],
- outputs=[text_output, audio_output],
- )
-
- prompt_mode.change(
- lambda x: gr_util(x),
- inputs=[prompt_mode],
- outputs=[text_prompt, audio_prompt],
- )
-
- audio_prompt.upload(
- lambda x: load_audio(x),
- inputs=[audio_prompt],
- outputs=[audio_prompt],
- )
-
- formatter.click(
- format_utils,
- inputs=[text, speaker],
- outputs=[language, text],
- )
-
- print("推理页面已开启!")
- webbrowser.open(f"http://127.0.0.1:{config.webui_config.port}")
- app.launch(share=config.webui_config.share, server_port=config.webui_config.port)
diff --git a/webui/__init__.py b/webui/__init__.py
deleted file mode 100644
index 4bc8efa..0000000
--- a/webui/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-from .dataset import create_dataset_app
-from .inference import create_inference_app
-from .merge import create_merge_app
-from .style_vectors import create_style_vectors_app
-from .train import create_train_app
-
-
-class TrainSettings:
- def __init__(self, setting_json):
- self.setting_json = setting_json
-
- def __enter__(self):
- return self
-
- def __exit__(self, exc_type, exc_value, traceback):
- pass
diff --git a/webui/dataset.py b/webui/dataset.py
index 3169fd2..acaef99 100644
--- a/webui/dataset.py
+++ b/webui/dataset.py
@@ -1,12 +1,7 @@
-import argparse
-import os
-
import gradio as gr
-import yaml
-from common.constants import GRADIO_THEME
-from common.log import logger
-from common.subprocess_utils import run_script_with_log
+from style_bert_vits2.logging import logger
+from style_bert_vits2.utils.subprocess import run_script_with_log
def do_slice(
diff --git a/webui/inference.py b/webui/inference.py
index 94124b9..d711846 100644
--- a/webui/inference.py
+++ b/webui/inference.py
@@ -1,11 +1,10 @@
-import argparse
import datetime
import json
from typing import Optional
import gradio as gr
-from common.constants import (
+from style_bert_vits2.constants import (
DEFAULT_ASSIST_TEXT_WEIGHT,
DEFAULT_LENGTH,
DEFAULT_LINE_SPLIT,
@@ -18,13 +17,24 @@ from common.constants import (
GRADIO_THEME,
Languages,
)
-from common.log import logger
-from common.tts_model import ModelHolder
-from infer import InvalidToneError
-from text.japanese import g2kata_tone, kata_tone2phone_tone, text_normalize
+from style_bert_vits2.logging import logger
+from style_bert_vits2.models.infer import InvalidToneError
+from style_bert_vits2.nlp.japanese import pyopenjtalk_worker as pyopenjtalk
+from style_bert_vits2.nlp.japanese.g2p_utils import g2kata_tone, kata_tone2phone_tone
+from style_bert_vits2.nlp.japanese.normalizer import normalize_text
+from style_bert_vits2.tts_model import TTSModelHolder
-languages = [l.value for l in Languages]
+# pyopenjtalk_worker を起動
+## pyopenjtalk_worker は TCP ソケットサーバーのため、ここで起動する
+pyopenjtalk.initialize_worker()
+
+# Web UI での学習時の無駄な GPU VRAM 消費を避けるため、あえてここでは BERT モデルの事前ロードを行わない
+# データセットの BERT 特徴量は事前に bert_gen.py により抽出されているため、学習時に BERT モデルをロードしておく必要はない
+# BERT モデルの事前ロードは「ロード」ボタン押下時に実行される TTSModelHolder.get_model_for_gradio() 内で行われる
+# Web UI での学習時、音声合成タブの「ロード」ボタンを押さなければ、BERT モデルが VRAM にロードされていない状態で学習を開始できる
+
+languages = [lang.value for lang in Languages]
initial_text = "こんにちは、初めまして。あなたの名前はなんていうの？"
@@ -85,7 +95,7 @@ examples = [
],
]
-initial_md = f"""
+initial_md = """
- Ver 2.3で追加されたエディターのほうが実際に読み上げさせるには使いやすいかもしれません。`Editor.bat`か`python server_editor.py --inbrowser`で起動できます。
- 初期からある[jvnvのモデル](https://huggingface.co/litagin/style_bert_vits2_jvnv)は、[JVNVコーパス（言語音声と非言語音声を持つ日本語感情音声コーパス）](https://sites.google.com/site/shinnosuketakamichi/research-topics/jvnv_corpus)で学習されたモデルです。ライセンスは[CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/deed.ja)です。
@@ -136,7 +146,7 @@ def gr_util(item):
return (gr.update(visible=False), gr.update(visible=True))
-def create_inference_app(model_holder: ModelHolder) -> gr.Blocks:
+def create_inference_app(model_holder: TTSModelHolder) -> gr.Blocks:
def tts_fn(
model_name,
model_path,
@@ -160,7 +170,7 @@ def create_inference_app(model_holder: ModelHolder) -> gr.Blocks:
pitch_scale,
intonation_scale,
):
- model_holder.load_model(model_name, model_path)
+ model_holder.get_model(model_name, model_path)
assert model_holder.current_model is not None
wrong_tone_message = ""
@@ -203,7 +213,7 @@ def create_inference_app(model_holder: ModelHolder) -> gr.Blocks:
reference_audio_path=reference_audio_path,
sdp_ratio=sdp_ratio,
noise=noise_scale,
- noisew=noise_scale_w,
+ noise_w=noise_scale_w,
length=length_scale,
line_split=line_split,
split_interval=split_interval,
@@ -213,7 +223,7 @@ def create_inference_app(model_holder: ModelHolder) -> gr.Blocks:
style=style,
style_weight=style_weight,
given_tone=tone,
- sid=speaker_id,
+ speaker_id=speaker_id,
pitch_scale=pitch_scale,
intonation_scale=intonation_scale,
)
@@ -229,7 +239,7 @@ def create_inference_app(model_holder: ModelHolder) -> gr.Blocks:
if tone is None and language == "JP":
# アクセント指定に使えるようにアクセント情報を返す
- norm_text = text_normalize(text)
+ norm_text = normalize_text(text)
kata_tone = g2kata_tone(norm_text)
kata_tone_json_str = json.dumps(kata_tone, ensure_ascii=False)
elif tone is None:
@@ -279,7 +289,6 @@ def create_inference_app(model_holder: ModelHolder) -> gr.Blocks:
value=1,
step=0.05,
label="音程(1以外では音質劣化)",
- visible=False, # pyworldが必要
)
intonation_scale = gr.Slider(
minimum=0,
@@ -287,7 +296,6 @@ def create_inference_app(model_holder: ModelHolder) -> gr.Blocks:
value=1,
step=0.1,
label="抑揚(1以外では音質劣化)",
- visible=False, # pyworldが必要
)
line_split = gr.Checkbox(
@@ -431,7 +439,7 @@ def create_inference_app(model_holder: ModelHolder) -> gr.Blocks:
)
model_name.change(
- model_holder.update_model_files_gr,
+ model_holder.update_model_files_for_gradio,
inputs=[model_name],
outputs=[model_path],
)
@@ -439,12 +447,12 @@ def create_inference_app(model_holder: ModelHolder) -> gr.Blocks:
model_path.change(make_non_interactive, outputs=[tts_button])
refresh_button.click(
- model_holder.update_model_names_gr,
+ model_holder.update_model_names_for_gradio,
outputs=[model_name, model_path, tts_button],
)
load_button.click(
- model_holder.load_model_gr,
+ model_holder.get_model_for_gradio,
inputs=[model_name, model_path],
outputs=[style, tts_button, speaker],
)
diff --git a/webui/merge.py b/webui/merge.py
index aeab08d..18454c9 100644
--- a/webui/merge.py
+++ b/webui/merge.py
@@ -1,4 +1,3 @@
-import argparse
import json
import os
from pathlib import Path
@@ -10,9 +9,10 @@ import yaml
from safetensors import safe_open
from safetensors.torch import save_file
-from common.constants import DEFAULT_STYLE, GRADIO_THEME
-from common.log import logger
-from common.tts_model import Model, ModelHolder
+from style_bert_vits2.constants import DEFAULT_STYLE, GRADIO_THEME
+from style_bert_vits2.logging import logger
+from style_bert_vits2.tts_model import TTSModel, TTSModelHolder
+
voice_keys = ["dec"]
voice_pitch_keys = ["flow"]
@@ -47,11 +47,11 @@ def merge_style(model_name_a, model_name_b, weight, output_name, style_triple_li
os.path.join(assets_root, model_name_b, "style_vectors.npy")
) # (style_num_b, 256)
with open(
- os.path.join(assets_root, model_name_a, "config.json"), encoding="utf-8"
+ os.path.join(assets_root, model_name_a, "config.json"), "r", encoding="utf-8"
) as f:
config_a = json.load(f)
with open(
- os.path.join(assets_root, model_name_b, "config.json"), encoding="utf-8"
+ os.path.join(assets_root, model_name_b, "config.json"), "r", encoding="utf-8"
) as f:
config_b = json.load(f)
style2id_a = config_a["data"]["style2id"]
@@ -88,7 +88,7 @@ def merge_style(model_name_a, model_name_b, weight, output_name, style_triple_li
# recipe.jsonを読み込んで、style_triple_listを追記
info_path = os.path.join(assets_root, output_name, "recipe.json")
if os.path.exists(info_path):
- with open(info_path, encoding="utf-8") as f:
+ with open(info_path, "r", encoding="utf-8") as f:
info = json.load(f)
else:
info = {}
@@ -250,23 +250,23 @@ def simple_tts(model_name, text, style=DEFAULT_STYLE, style_weight=1.0):
config_path = os.path.join(assets_root, model_name, "config.json")
style_vec_path = os.path.join(assets_root, model_name, "style_vectors.npy")
- model = Model(Path(model_path), Path(config_path), Path(style_vec_path), device)
+ model = TTSModel(Path(model_path), Path(config_path), Path(style_vec_path), device)
return model.infer(text, style=style, style_weight=style_weight)
-def update_two_model_names_dropdown(model_holder: ModelHolder):
- new_names, new_files, _ = model_holder.update_model_names_gr()
+def update_two_model_names_dropdown(model_holder: TTSModelHolder):
+ new_names, new_files, _ = model_holder.update_model_names_for_gradio()
return new_names, new_files, new_names, new_files
def load_styles_gr(model_name_a, model_name_b):
config_path_a = os.path.join(assets_root, model_name_a, "config.json")
- with open(config_path_a, encoding="utf-8") as f:
+ with open(config_path_a, "r", encoding="utf-8") as f:
config_a = json.load(f)
styles_a = list(config_a["data"]["style2id"].keys())
config_path_b = os.path.join(assets_root, model_name_b, "config.json")
- with open(config_path_b, encoding="utf-8") as f:
+ with open(config_path_b, "r", encoding="utf-8") as f:
config_b = json.load(f)
styles_b = list(config_b["data"]["style2id"].keys())
@@ -328,7 +328,7 @@ Happy, Surprise, HappySurprise
"""
-def create_merge_app(model_holder: ModelHolder) -> gr.Blocks:
+def create_merge_app(model_holder: TTSModelHolder) -> gr.Blocks:
model_names = model_holder.model_names
if len(model_names) == 0:
logger.error(
@@ -444,12 +444,12 @@ def create_merge_app(model_holder: ModelHolder) -> gr.Blocks:
audio_output = gr.Audio(label="結果")
model_name_a.change(
- model_holder.update_model_files_gr,
+ model_holder.update_model_files_for_gradio,
inputs=[model_name_a],
outputs=[model_path_a],
)
model_name_b.change(
- model_holder.update_model_files_gr,
+ model_holder.update_model_files_for_gradio,
inputs=[model_name_b],
outputs=[model_path_b],
)
diff --git a/webui/style_vectors.py b/webui/style_vectors.py
index 8effa37..05056eb 100644
--- a/webui/style_vectors.py
+++ b/webui/style_vectors.py
@@ -1,4 +1,3 @@
-import argparse
import json
import os
import shutil
@@ -12,9 +11,10 @@ from sklearn.cluster import DBSCAN, AgglomerativeClustering, KMeans
from sklearn.manifold import TSNE
from umap import UMAP
-from common.constants import DEFAULT_STYLE, GRADIO_THEME
-from common.log import logger
from config import config
+from style_bert_vits2.constants import DEFAULT_STYLE, GRADIO_THEME
+from style_bert_vits2.logging import logger
+
# Get path settings
with open(os.path.join("configs", "paths.yml"), "r", encoding="utf-8") as f:
@@ -152,7 +152,7 @@ def do_dbscan_gradio(eps=2.5, min_samples=15):
return [
plt,
gr.Slider(maximum=MAX_CLUSTER_NUM),
- f"クラスタが数が0です。パラメータを変えてみてください。",
+ "クラスタが数が0です。パラメータを変えてみてください。",
] + [gr.Audio(visible=False)] * MAX_AUDIO_NUM
return [plt, gr.Slider(maximum=n_clusters, value=1), n_clusters] + [
@@ -211,7 +211,7 @@ def save_style_vectors_from_clustering(model_name, style_names_str: str):
if len(style_name_list) != len(centroids) + 1:
return f"スタイルの数が合いません。`,`で正しく{len(centroids)}個に区切られているか確認してください: {style_names_str}"
if len(set(style_names)) != len(style_names):
- return f"スタイル名が重複しています。"
+ return "スタイル名が重複しています。"
logger.info(f"Backup {config_path} to {config_path}.bak")
shutil.copy(config_path, f"{config_path}.bak")
@@ -242,7 +242,7 @@ def save_style_vectors_from_files(
return f"音声ファイルとスタイル名の数が合いません。`,`で正しく{len(style_names)}個に区切られているか確認してください: {audio_files_str}と{style_names_str}"
style_name_list = [DEFAULT_STYLE] + style_names
if len(set(style_names)) != len(style_names):
- return f"スタイル名が重複しています。"
+ return "スタイル名が重複しています。"
style_vectors = [mean]
wavs_dir = os.path.join(dataset_root, model_name, "wavs")
diff --git a/webui/train.py b/webui/train.py
index dee018e..6d72061 100644
--- a/webui/train.py
+++ b/webui/train.py
@@ -1,4 +1,3 @@
-import argparse
import json
import os
import shutil
@@ -14,9 +13,10 @@ from pathlib import Path
import gradio as gr
import yaml
-from common.log import logger
-from common.stdout_wrapper import SAFE_STDOUT
-from common.subprocess_utils import run_script_with_log, second_elem_of
+from style_bert_vits2.logging import logger
+from style_bert_vits2.utils.stdout_wrapper import SAFE_STDOUT
+from style_bert_vits2.utils.subprocess import run_script_with_log, second_elem_of
+
logger_handler = None
tensorboard_executed = False
@@ -87,6 +87,9 @@ def initialize(
config["train"]["bf16_run"] = False # デフォルトでFalseのはずだが念のため
+ # 今はデフォルトであるが、以前は非JP-Extra版になくバグの原因になるので念のため
+ config["data"]["use_jp_extra"] = use_jp_extra
+
model_path = os.path.join(dataset_path, "models")
if os.path.exists(model_path):
logger.warning(
@@ -145,10 +148,10 @@ def resample(model_name, normalize, trim, num_processes):
cmd.append("--trim")
success, message = run_script_with_log(cmd)
if not success:
- logger.error(f"Step 2: resampling failed.")
+ logger.error("Step 2: resampling failed.")
return False, f"Step 2, Error: 音声ファイルの前処理に失敗しました:\n{message}"
elif message:
- logger.warning(f"Step 2: resampling finished with stderr.")
+ logger.warning("Step 2: resampling finished with stderr.")
return True, f"Step 2, Success: 音声ファイルの前処理が完了しました:\n{message}"
logger.success("Step 2: resampling finished.")
return True, "Step 2, Success: 音声ファイルの前処理が完了しました"
@@ -195,13 +198,13 @@ def preprocess_text(model_name, use_jp_extra, val_per_lang, yomi_error):
cmd.append("--use_jp_extra")
success, message = run_script_with_log(cmd)
if not success:
- logger.error(f"Step 3: preprocessing text failed.")
+ logger.error("Step 3: preprocessing text failed.")
return (
False,
f"Step 3, Error: 書き起こしファイルの前処理に失敗しました:\n{message}",
)
elif message:
- logger.warning(f"Step 3: preprocessing text finished with stderr.")
+ logger.warning("Step 3: preprocessing text finished with stderr.")
return (
True,
f"Step 3, Success: 書き起こしファイルの前処理が完了しました:\n{message}",
@@ -223,10 +226,10 @@ def bert_gen(model_name):
]
)
if not success:
- logger.error(f"Step 4: bert_gen failed.")
+ logger.error("Step 4: bert_gen failed.")
return False, f"Step 4, Error: BERT特徴ファイルの生成に失敗しました:\n{message}"
elif message:
- logger.warning(f"Step 4: bert_gen finished with stderr.")
+ logger.warning("Step 4: bert_gen finished with stderr.")
return (
True,
f"Step 4, Success: BERT特徴ファイルの生成が完了しました:\n{message}",
@@ -248,13 +251,13 @@ def style_gen(model_name, num_processes):
]
)
if not success:
- logger.error(f"Step 5: style_gen failed.")
+ logger.error("Step 5: style_gen failed.")
return (
False,
f"Step 5, Error: スタイル特徴ファイルの生成に失敗しました:\n{message}",
)
elif message:
- logger.warning(f"Step 5: style_gen finished with stderr.")
+ logger.warning("Step 5: style_gen finished with stderr.")
return (
True,
f"Step 5, Success: スタイル特徴ファイルの生成が完了しました:\n{message}",
@@ -348,10 +351,10 @@ def train(model_name, skip_style=False, use_jp_extra=True, speedup=False):
cmd.append("--speedup")
success, message = run_script_with_log(cmd, ignore_warning=True)
if not success:
- logger.error(f"Train failed.")
+ logger.error("Train failed.")
return False, f"Error: 学習に失敗しました:\n{message}"
elif message:
- logger.warning(f"Train finished with stderr.")
+ logger.warning("Train finished with stderr.")
return True, f"Success: 学習が完了しました:\n{message}"
logger.success("Train finished.")
return True, "Success: 学習が完了しました"
@@ -397,7 +400,7 @@ def run_tensorboard(model_name):
yield gr.Button("Tensorboardを開く")
-how_to_md = f"""
+how_to_md = """
## 使い方
- データを準備して、モデル名を入力して、必要なら設定を調整してから、「自動前処理を実行」ボタンを押してください。進捗状況等はターミナルに表示されます。
@@ -792,5 +795,4 @@ def create_train_app():
outputs=[use_jp_extra_train],
)
- # app.launch(inbrowser=not args.no_autolaunch, server_name=args.server_name)
return app