Docs: CLI doc and paperspace guide

Dockerfile

@@ -1,127 +0,0 @@
-# For paperspace gradient
-# Based on https://github.com/gradient-ai/base-container
-# Style-Bert-VITS2 are NOT included in this image, only for environment setup
-
-# ==================================================================
-# Details
-# ------------------------------------------------------------------
-# Ubuntu 22.04, Python 3.10
-# CUDA Toolkit 12.1, CUDNN 8.9.7
-# PyTorch 2.1.2 (cuda 12.1)
-# Jupyter Lab
-# Huggingface CLI
-# Other Python packages in requirements.txt
-
-# ==================================================================
-# Initial setup
-# ------------------------------------------------------------------
-
-# Ubuntu 22.04 as base image
-FROM ubuntu:22.04
-# RUN yes| unminimize
-
-RUN sed -i 's@archive.ubuntu.com@ftp.jaist.ac.jp/pub/Linux@g' /etc/apt/sources.list
-
-# Set ENV variables
-ENV LANG C.UTF-8
-ENV SHELL=/bin/bash
-ENV DEBIAN_FRONTEND=noninteractive
-
-ENV APT_INSTALL="apt-get install -y --no-install-recommends"
-ENV PIP_INSTALL="python3 -m pip --no-cache-dir install --upgrade"
-ENV GIT_CLONE="git clone --depth 10"
-
-
-# ==================================================================
-# Tools
-# ------------------------------------------------------------------
-
-RUN apt-get update && \
-    $APT_INSTALL \
-    build-essential \
-    ca-certificates \
-    wget \
-    git \
-    curl \
-    unzip \
-    zip \
-    nano \
-    ffmpeg \
-    sudo \
-    software-properties-common \
-    gnupg \
-    python3 \
-    python3-pip \
-    python3-dev
-
-# ==================================================================
-# Git-lfs
-# ------------------------------------------------------------------
-
-RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash && \
-    $APT_INSTALL git-lfs
-
-# Add symlink so python and python3 commands use same python3.9 executable
-RUN ln -s /usr/bin/python3 /usr/local/bin/python    
-
-
-# ==================================================================
-# Installing CUDA packages (CUDA Toolkit 12.1 and CUDNN 8.9.7)
-# ------------------------------------------------------------------
-RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-ubuntu2204.pin && \
-    mv cuda-ubuntu2204.pin /etc/apt/preferences.d/cuda-repository-pin-600 && \
-    wget https://developer.download.nvidia.com/compute/cuda/12.1.0/local_installers/cuda-repo-ubuntu2204-12-1-local_12.1.0-530.30.02-1_amd64.deb && \
-    dpkg -i cuda-repo-ubuntu2204-12-1-local_12.1.0-530.30.02-1_amd64.deb && \
-    cp /var/cuda-repo-ubuntu2204-12-1-local/cuda-*-keyring.gpg /usr/share/keyrings/ && \
-    apt-get update && \
-    $APT_INSTALL cuda && \  
-    rm cuda-repo-ubuntu2204-12-1-local_12.1.0-530.30.02-1_amd64.deb
-
-# Installing CUDNN
-RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/3bf863cc.pub && \
-    add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/ /" && \
-    apt-get update && \
-    $APT_INSTALL libcudnn8=8.9.7.29-1+cuda12.2  \
-    libcudnn8-dev=8.9.7.29-1+cuda12.2
-
-
-ENV PATH=$PATH:/usr/local/cuda/bin
-ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH
-
-
-# ==================================================================
-# PyTorch
-# ------------------------------------------------------------------
-
-# Based on https://pytorch.org/get-started/locally/
-
-RUN $PIP_INSTALL torch==2.1.2 torchvision==0.16.2 torchaudio==2.1.2 --index-url https://download.pytorch.org/whl/cu121
-
-# ==================================================================
-# Jupyter Lab
-# ------------------------------------------------------------------
-
-RUN $PIP_INSTALL jupyterlab
-
-# ==================================================================
-# huggingface_cli
-# ------------------------------------------------------------------
-
-RUN $PIP_INSTALL "huggingface_hub[cli]"
-
-# ==================================================================
-# Other Python packages
-# ------------------------------------------------------------------
-
-COPY requirements.txt /tmp/requirements.txt
-
-RUN $PIP_INSTALL -r /tmp/requirements.txt && \
-    rm /tmp/requirements.txt
-
-# ==================================================================
-# Startup
-# ------------------------------------------------------------------
-
-EXPOSE 8888 6006
-
-CMD jupyter lab --allow-root --ip=0.0.0.0 --no-browser --ServerApp.trust_xheaders=True --ServerApp.disable_check_xsrf=False --ServerApp.allow_remote_access=True --ServerApp.allow_origin='*' --ServerApp.allow_credentials=True

README.md

@@ -90,6 +90,9 @@ model_assets
 
 ### 学習
 
+- CLIでの学習の詳細は[こちら](docs/CLI.md)を参照してください。
+- paperspace上での学習の詳細は[こちら](docs/paperspace.md)を参照してください。
+
 学習には2-14秒程度の音声ファイルが複数と、それらの書き起こしデータが必要です。
 
 - 既存コーパスなどですでに分割された音声ファイルと書き起こしデータがある場合はそのまま（必要に応じて書き起こしファイルを修正して）使えます。下の「学習WebUI」を参照してください。

colab.ipynb

@@ -4,7 +4,7 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "# Style-Bert-VITS2 (ver 2.2) のGoogle Colabでの学習\n",
+        "# Style-Bert-VITS2 (ver 2.3) のGoogle Colabでの学習\n",
         "\n",
         "Google Colab上でStyle-Bert-VITS2の学習を行うことができます。\n",
         "\n",

common/constants.py

@@ -4,7 +4,7 @@ import enum
 # See https://huggingface.co/spaces/gradio/theme-gallery for more themes
 GRADIO_THEME: str = "NoCrypt/miku"
 
-LATEST_VERSION: str = "2.2"
+LATEST_VERSION: str = "2.3"
 DEFAULT_STYLE: str = "Neutral"
 DEFAULT_STYLE_WEIGHT: float = 5.0
 

configs/config.json

@@ -67,5 +67,5 @@
     "use_spectral_norm": false,
     "gin_channels": 256
   },
-  "version": "2.2"
+  "version": "2.3"
 }

configs/configs_jp_extra.json

@@ -74,5 +74,5 @@
       "initial_channel": 64
     }
   },
-  "version": "2.2-JP-Extra"
+  "version": "2.3-JP-Extra"
 }

docs/CLI.md

@@ -1,35 +1,49 @@
 # CLI
 
-**WIP**
+## 0. Install and global paths settings
-
-## Dataset
-
-`Dataset.bat` webui (`python webui_dataset.py`) consists of **slice audio** and **transcribe wavs**.
-
-### Slice audio
 
 ```bash
-python slice.py -i <input_dir> -o <output_dir> -m <min_sec> -M <max_sec>
+git clone https://github.com/litagin02/Style-Bert-VITS2.git
+cd Style-Bert-VITS2
+python -m venv venv
+venv\Scripts\activate
+pip install torch==2.1.2 torchvision==0.16.2 torchaudio==2.1.2 --index-url https://download.pytorch.org/whl/cu118
+pip install -r requirements.txt
 ```
 
-Required:
+Then download the necessary models and the default TTS model, and set the global paths.
-- `input_dir`: Path to the directory containing the audio files to slice.
+```bash
-- `output_dir`: Path to the directory where the sliced audio files will be saved.
+python initialize.py [--skip_jvnv] [--dataset_root <path>] [--assets_root <path>]
+```
 
 Optional:
-- `min_sec`: Minimum duration of the sliced audio files in seconds (default 2).
+- `--skip_jvnv`: Skip downloading the default JVNV voice models (use this if you only have to train your own models).
-- `max_sec`: Maximum duration of the sliced audio files in seconds (default 12).
+- `--dataset_root`: Default: `Data`. Root directory of the training dataset. The training dataset of `{model_name}` should be placed in `{dataset_root}/{model_name}`.
+- `--assets_root`: Default: `model_assets`. Root directory of the model assets (for inference). In training, the model assets will be saved to `{assets_root}/{model_name}`, and in inference, we load all the models from `{assets_root}`.
 
-### Transcribe wavs
 
+## 1. Dataset preparation
+
+### 1.1. Slice wavs
 ```bash
-python transcribe.py -i <input_dir> -o <output_file> --speaker_name <speaker_name>
+python slice.py --model_name <model_name> [-i <input_dir>] [-m <min_sec>] [-M <max_sec>]
 ```
 
 Required:
-- `input_dir`: Path to the directory containing the audio files to transcribe.
+- `model_name`: Name of the speaker (to be used as the name of the trained model).
-- `output_file`: Path to the file where the transcriptions will be saved.
+
-- `speaker_name`: Name of the speaker.
+Optional:
+- `input_dir`: Path to the directory containing the audio files to slice (default: `inputs`)
+- `min_sec`: Minimum duration of the sliced audio files in seconds (default: 2).
+- `max_sec`: Maximum duration of the sliced audio files in seconds (default: 12).
+
+### 1.2. Transcribe wavs
+
+```bash
+python transcribe.py --model_name <model_name>
+```
+Required:
+- `model_name`: Name of the speaker (to be used as the name of the trained model).
 
 Optional
 - `--initial_prompt`: Initial prompt to use for the transcription (default value is specific to Japanese).
@@ -38,19 +52,43 @@ Optional
 - `--model`: Whisper model, default: `large-v3`
 - `--compute_type`: default: `bfloat16`
 
-## Train
+## 2. Preprocess
 
-`Train.bat` webui (`python webui_train.py`) consists of the following.
-
-### Preprocess audio
 ```bash
-python resample.py -i <input_dir> -o <output_dir> [--normalize] [--trim]
+python preprocess_all.py -m <model_name> [--use_jp_extra] [-b <batch_size>] [-e <epochs>] [-s <save_every_steps>] [--num_processes <num_processes>] [--normalize] [--trim] [--val_per_lang <val_per_lang>] [--log_interval <log_interval>] [--freeze_EN_bert] [--freeze_JP_bert] [--freeze_ZH_bert] [--freeze_style] 
 ```
 
 Required:
-- `input_dir`: Path to the directory containing the audio files to preprocess.
+- `model_name`: Name of the speaker (to be used as the name of the trained model).
-- `output_dir`: Path to the directory where the preprocessed audio files will be saved.
 
-TO BE WRITTEN (WIP)
+Optional:
+- `--batch_size`, `-b`: Batch size (default: 2).
+- `--epochs`, `-e`: Number of epochs (default: 100).
+- `--save_every_steps`, `-s`: Save every steps (default: 1000).
+- `--num_processes`: Number of processes (default: half of the number of CPU cores).
+- `--normalize`: Loudness normalize audio.
+- `--trim`: Trim silence.
+- `--freeze_EN_bert`: Freeze English BERT.
+- `--freeze_JP_bert`: Freeze Japanese BERT.
+- `--freeze_ZH_bert`: Freeze Chinese BERT.
+- `--freeze_style`: Freeze style vector.
+- `--use_jp_extra`: Use JP-Extra model.
+- `--val_per_lang`: Validation data per language (default: 0).
+- `--log_interval`: Log interval (default: 200).
 
-これいる？
+## 3. Train
+
+Training settings are automatically loaded from the above process.
+
+If NOT using JP-Extra model:
+```bash
+python train_ms.py [--repo_id <username>/<repo_name>]
+```
+
+If using JP-Extra model:
+```bash
+python train_ms_jp_extra.py [--repo_id <username>/<repo_name>]
+```
+
+Optional:
+- `--repo_id`: Hugging Face repository ID to upload the trained model to. You should have logged in using `huggingface-cli login` before running this command.

docs/paperspace.md

@@ -0,0 +1,72 @@
+# Paperspace gradient で学習する
+
+詳しいコマンドの叩き方は[こちら](CLI.md)を参照してください。
+
+## 事前準備
+- Paperspace のアカウントを作成し必要なら課金する
+- Projectを作る
+- NotebookはStart from Scratchを選択して空いてるGPUマシンを選ぶ
+
+## 使い方
+
+以下では次のような方針でやっています。
+
+- `/storage/`は永続ストレージなので、事前学習モデルとかを含めてリポジトリをクローンするとよい。
+- `/notebooks/`は一時ストレージなので、データセットやその結果を保存する。
+- hugging faceアカウントを作り、（プライベートな）リポジトリを作って、学習元データを置いたり、学習結果を随時アップロードする。
+
+### 1. 環境を作る
+
+まずは永続ストレージにgit clone
+```bash
+mkdir -p /storage/sbv2
+cd /storage/sbv2
+git clone https://github.com/litagin02/Style-Bert-VITS2.git
+```
+環境構築（デフォルトはPyTorch 1.x系、Python 3.9の模様）
+```bash
+pip install torch==2.1.2 torchvision==0.16.2 torchaudio==2.1.2 --index-url https://download.pytorch.org/whl/cu118
+pip install -r requirements.txt
+```
+事前学習済みモデル等のダウンロード、またパスを`/notebooks/`以下のものに設定
+```bash
+python initialize.py --skip_jvnv --dataset_root /notebooks/Data --assets_root /notebooks/model_assets
+```
+
+### 2. データセットの準備
+以下では`username/voices`というデータセットリポジトリにある`Foo.zip`というデータセットを使うことを想定しています。
+```bash
+cd /nodtebooks
+huggingface-cli login  # 事前にトークンが必要
+huggingface-cli download username/voices Foo.zip --repo-type dataset --local-dir .
+```
+
+- zipファイル中身が既に`raw`と`esd.list`があるデータ（スライス・書き起こし済み）の場合
+```bash
+mkdir -p Data/Foo
+unzip Foo.zip -d Data/Foo
+rm Foo.zip
+cd /storage/sbv2/Style-Bert-VITS2
+```
+
+- zipファイルが音声ファイルのみの場合
+```bash
+mkdir inputs
+unzip Foo.zip -d inputs
+cd /storage/sbv2/Style-Bert-VITS2
+python slice.py --model_name Foo -i /notebooks/inputs
+python transcribe.py --model_name Foo
+```
+
+それが終わったら、以下のコマンドで一括前処理を行う（パラメータは各自お好み、バッチサイズ6でVRAM 16GBギリくらい）。
+```bash
+python preprocess_all.py --model_name Foo -b 6 -e 300 --use_jp_extra
+```
+
+### 3. 学習
+
+Hugging faceの`username/sbv2-private`というモデルリポジトリに学習済みモデルをアップロードすることを想定しています。事前に`huggingface-cli login`でログインしておくこと。
+```bash
+python train_ms_jp_extra.py --repo_id username/sbv2-private
+```
+(JP-Extraでない場合は`train_ms.py`を使う)

preprocess_all.py

@@ -69,11 +69,6 @@ if __name__ == "__main__":
         help="Log interval",
         default=200,
     )
-    parser.add_argument(
-        "--skip_invalid",
-        action="store_true",
-        help="Skip invalid",
-    )
 
     args = parser.parse_args()
 

requirements.txt

@@ -17,7 +17,8 @@ numpy
 psutil
 pyannote.audio>=3.1.0
 pyloudnorm
-pyopenjtalk-prebuilt
+# pyopenjtalk-prebuilt  # Should be manually uninstalled
+pyopenjtalk-dict
 pypinyin
 pyworld
 PyYAML