init (not checked bat script yet)
This commit is contained in:
litagin02
43
.github/workflows/pull_format.yml
vendored
43
.github/workflows/pull_format.yml
vendored
@@ -1,43 +0,0 @@
|
||||
name: pull format
|
||||
|
||||
on: [pull_request]
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
pull_format:
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ["3.10"]
|
||||
os: [ubuntu-latest]
|
||||
fail-fast: false
|
||||
|
||||
continue-on-error: true
|
||||
|
||||
steps:
|
||||
- name: checkout
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
ref: ${{ github.head_ref }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install Black
|
||||
run: pip install "black[jupyter]"
|
||||
|
||||
- name: Run Black
|
||||
# run: black $(git ls-files '*.py')
|
||||
run: black .
|
||||
|
||||
- name: Commit Back
|
||||
uses: stefanzweifel/git-auto-commit-action@v4
|
||||
with:
|
||||
commit_message: Apply Code Formatter Change
|
||||
57
.github/workflows/push_format.yml
vendored
57
.github/workflows/push_format.yml
vendored
@@ -1,57 +0,0 @@
|
||||
name: push format
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
- dev
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
push_format:
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ["3.10"]
|
||||
os: [ubuntu-latest]
|
||||
fail-fast: false
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
ref: ${{github.ref_name}}
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install Black
|
||||
run: pip install "black[jupyter]"
|
||||
|
||||
- name: Run Black
|
||||
# run: black $(git ls-files '*.py')
|
||||
run: black .
|
||||
|
||||
- name: Commit Back
|
||||
continue-on-error: true
|
||||
id: commitback
|
||||
run: |
|
||||
git config --local user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git config --local user.name "github-actions[bot]"
|
||||
git add --all
|
||||
git commit -m "Format code"
|
||||
|
||||
- name: Create Pull Request
|
||||
if: steps.commitback.outcome == 'success'
|
||||
continue-on-error: true
|
||||
uses: peter-evans/create-pull-request@v5
|
||||
with:
|
||||
delete-branch: true
|
||||
body: Apply Code Formatter Change
|
||||
title: Apply Code Formatter Change
|
||||
commit-message: Automatic code format
|
||||
184
.gitignore
vendored
184
.gitignore
vendored
@@ -1,188 +1,16 @@
|
||||
# Byte-compiled / optimized / DLL files
|
||||
.vscode/
|
||||
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
cover/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
.pybuilder/
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
# For a library or package, you might want to ignore these files since the code is
|
||||
# intended to run in multiple environments; otherwise, check them in:
|
||||
# .python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# poetry
|
||||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||
#poetry.lock
|
||||
|
||||
# pdm
|
||||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||
#pdm.lock
|
||||
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
||||
# in version control.
|
||||
# https://pdm.fming.dev/#use-with-ide
|
||||
.pdm.toml
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
.ipynb_checkpoints/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# pytype static type analyzer
|
||||
.pytype/
|
||||
|
||||
# Cython debug symbols
|
||||
cython_debug/
|
||||
|
||||
# PyCharm
|
||||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||
#.idea/
|
||||
|
||||
.DS_Store
|
||||
/models
|
||||
/logs
|
||||
|
||||
filelists/*
|
||||
!/filelists/esd.list
|
||||
data/*
|
||||
/*.yml
|
||||
!/default_config.yml
|
||||
/Web/
|
||||
/emotional/*/*.bin
|
||||
/slm/*/*.bin
|
||||
/bert/*/*.bin
|
||||
/bert/*/*.h5
|
||||
/bert/*/*.model
|
||||
/bert/*/*.safetensors
|
||||
/bert/*/*.msgpack
|
||||
asr_transcript.py
|
||||
extract_list.py
|
||||
dataset
|
||||
/Data
|
||||
Model
|
||||
raw/
|
||||
logs/
|
||||
Data/*
|
||||
/onnx
|
||||
/.vs
|
||||
|
||||
/pretrained/*.safetensors
|
||||
/pretrained/*.pth
|
||||
|
||||
0
.gitmodules
vendored
0
.gitmodules
vendored
@@ -1,25 +0,0 @@
|
||||
repos:
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v4.5.0
|
||||
hooks:
|
||||
- id: check-yaml
|
||||
- id: end-of-file-fixer
|
||||
- id: trailing-whitespace
|
||||
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.1.8
|
||||
hooks:
|
||||
- id: ruff
|
||||
args: [ --fix ]
|
||||
|
||||
- repo: https://github.com/psf/black
|
||||
rev: 23.12.0
|
||||
hooks:
|
||||
- id: black
|
||||
|
||||
- repo: https://github.com/codespell-project/codespell
|
||||
rev: v2.2.6
|
||||
hooks:
|
||||
- id: codespell
|
||||
files: ^.*\.(py|md|rst|yml)$
|
||||
args: [-L=fro]
|
||||
8
App.bat
Normal file
8
App.bat
Normal file
@@ -0,0 +1,8 @@
|
||||
@echo off
|
||||
|
||||
echo Running app.py...
|
||||
venv\Scripts\python app.py
|
||||
|
||||
if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% )
|
||||
|
||||
pause
|
||||
2
Data/.gitignore
vendored
Normal file
2
Data/.gitignore
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
*
|
||||
!.gitignore
|
||||
93
README.md
93
README.md
@@ -1,17 +1,86 @@
|
||||
# Bert-VITS2-litaginフォーク
|
||||
# Style-Bert-VITS2
|
||||
|
||||
## 変更点
|
||||
- `keep_ckpts`のバグを修正
|
||||
- 圧縮モデルを保存する設定を追加: `config.json`の`save_compressed_models`を`true`にすると、modelの保存時に圧縮モデルも保存される（これと`keep_ckpts`を`1`とかにしとけばかなり容量の節約に）
|
||||
- Ver 2.1での学習をサポート（`train_ms_V210.py`）
|
||||
Bert-VITS2 with more controllable voice styles.
|
||||
|
||||
## TODO
|
||||
- [x] Ver 2.2での学習をサポート←たぶんやった、まだ確認してない
|
||||
- [ ] Ver 2.1での感情のクラス数を10から少なくして実験
|
||||
- [ ] Ver 2.1, 2.2での学習でのbf16対応
|
||||
- [ ] 推論のWebUIでのバージョンに応じた感情指定のサポート
|
||||
- [ ] より良い推論WebUI？
|
||||
- [ ] 学習のWebUI?
|
||||
This repository is based on [Bert-VITS2](https://github.com/fishaudio/Bert-VITS2) v2.1, so many thanks to the original author!
|
||||
|
||||
## これは何？
|
||||
- [Bert-VITS2](https://github.com/fishaudio/Bert-VITS2)のv2.1を元に、正確に感情や発話スタイルを強弱混みで指定して音声を生成することができるようにしたものです。
|
||||
- [EasyBertVits2](https://github.com/Zuntan03/EasyBertVits2/)のように、GitやPythonがない人でも簡単にインストールできるやつもあります。
|
||||
|
||||
## 使い方
|
||||
|
||||
詳しくは[こちら](docs/tutorial.md)を参照してください。
|
||||
|
||||
### インストール
|
||||
|
||||
Windows環境で最近のNVIDIA製グラボがあることを前提にしています。
|
||||
|
||||
#### GitやPythonに馴染みが無い方
|
||||
|
||||
[これ]をダウンロードして、スペースを含まない英数字のみのパスで実行してください（まだ動作未確認なので後でちゃんとチェックします）。
|
||||
|
||||
#### GitやPython使える人
|
||||
|
||||
Python 3.10で動作確認しています。
|
||||
|
||||
```bash
|
||||
git clone https://github.com/litagin02/Style-Bert-VITS2.git
|
||||
cd Style-Bert-VITS2
|
||||
python -m venv venv
|
||||
venv\Scripts\activate
|
||||
pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu121
|
||||
pip install -r requirements.txt
|
||||
python initialize.py
|
||||
```
|
||||
最後を忘れずに。
|
||||
|
||||
### 音声合成
|
||||
|
||||
`App.bat`をダブルクリックするとWebUIが起動します。
|
||||
TODO: デフォルトモデルをいくつかダウンロードするようにする
|
||||
|
||||
ディレクトリ構造:
|
||||
```
|
||||
model_assets
|
||||
├── your_model
|
||||
│ ├── config.json
|
||||
│ ├── your_model_file1.safetensors
|
||||
│ ├── your_model_file2.safetensors
|
||||
│ ├── ...
|
||||
│ └── style_vectors.npy
|
||||
└── another_model
|
||||
├── ...
|
||||
```
|
||||
このように、推論には`config.json`と`*.safetensors`と`style_vectors.npy`が必要です。学習の段階で前者の2つは自動で作成されますが、`style_vectors.npy`は自分で作成する必要があります: 下の「スタイルの生成」を参照してください。
|
||||
|
||||
### 学習
|
||||
|
||||
`Train.bat`をダブルクリックするとWebUIが起動します。
|
||||
|
||||
### スタイルの生成
|
||||
|
||||
- `Style.bat`をダブルクリックするとWebUIが起動します。
|
||||
- この手順は、音声ファイルたちからスタイルを作るのに必要な手順です。
|
||||
- 学習とは独立しているので、学習中でもできるし、学習が終わっても何度もやりなおせます。
|
||||
|
||||
|
||||
## Bert-VITS2 v2.1と違う点
|
||||
- 感情埋め込みのモデルを変更（[wav2vec2-large-robust-12-ft-emotion-msp-dim](https://huggingface.co/audeering/wav2vec2-large-robust-12-ft-emotion-msp-dim)から[wespeaker-voxceleb-resnet34-LM](https://huggingface.co/pyannote/wespeaker-voxceleb-resnet34-LM)へ、感情埋め込みというより正確には話者埋め込みが近い）
|
||||
- 埋め込みもベクトル量子化を取り払い、単なる全結合層に。
|
||||
- スタイルベクトルファイル`style_vectors.npy`を作ることで、そのスタイルを使って効果の強さも連続的に指定しつつ音声を生成することができる。
|
||||
- 各種WebUIを作成（事前準備・学習・スタイルベクトルの生成・音声合成）
|
||||
- bf16での学習のサポート
|
||||
- safetensors形式のサポート、デフォルトでsafetensorsを使用するように
|
||||
- その他軽微なbugfixやリファクタリング
|
||||
|
||||
## Bert-VITS2 v2.1と同じ点
|
||||
- [事前学習モデル](https://huggingface.co/litagin/style_bert_vits2_jvnv)は、実質Bert-VITS2 v2.1と同じものを使用しています（不要な重みを削ってsafetensorsに変換したもの）。
|
||||
|
||||
## 実験したいこと
|
||||
- [ ] 複数話者での学習の実験（原理的にはできるはず、スタイルがどう効くかが未知）
|
||||
- [ ] むしろ複数話者で単一話者扱いで学習しても、スタイル埋め込みが話者埋め込みでそこに話者の情報が含まれているので、スタイルベクトルを作ることで複数話者の音声を生成できるのではないか？
|
||||
- [ ] もしそうなら、大量の人数の音声を学習させれば、ある意味「話者空間から適当に選んだ話者（連続的に変えられる）の音声合成」ができるのでは？リファレンス音声も使えばゼロショットでの音声合成もできるのでは？
|
||||
|
||||
|
||||
以下本家のREADME.md
|
||||
|
||||
8
Style.bat
Normal file
8
Style.bat
Normal file
@@ -0,0 +1,8 @@
|
||||
@echo off
|
||||
|
||||
echo Running webui_style_vectors.py...
|
||||
venv\Scripts\python webui_style_vectors.py
|
||||
|
||||
if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% )
|
||||
|
||||
pause
|
||||
8
Train.bat
Normal file
8
Train.bat
Normal file
@@ -0,0 +1,8 @@
|
||||
@echo off
|
||||
|
||||
echo Running webui_train.py...
|
||||
venv\Scripts\python webui_train.py
|
||||
|
||||
if %errorlevel% neq 0 ( pause & popd & exit /b %errorlevel% )
|
||||
|
||||
pause
|
||||
493
app.py
Normal file
493
app.py
Normal file
@@ -0,0 +1,493 @@
|
||||
import argparse
|
||||
import os
|
||||
|
||||
import gradio as gr
|
||||
import numpy as np
|
||||
import torch
|
||||
import warnings
|
||||
from gradio.processing_utils import convert_to_16_bit_wav
|
||||
|
||||
import utils
|
||||
from infer import get_net_g, infer
|
||||
from tools.log import logger
|
||||
from config import config
|
||||
|
||||
is_hf_spaces = os.getenv("SYSTEM") == "spaces"
|
||||
limit = 100
|
||||
|
||||
|
||||
class Model:
|
||||
def __init__(self, model_path, config_path, style_vec_path, device):
|
||||
self.model_path = model_path
|
||||
self.config_path = config_path
|
||||
self.device = device
|
||||
self.style_vec_path = style_vec_path
|
||||
self.load()
|
||||
|
||||
def load(self):
|
||||
self.hps = utils.get_hparams_from_file(self.config_path)
|
||||
self.spk2id = self.hps.data.spk2id
|
||||
self.num_styles = self.hps.data.num_styles
|
||||
if hasattr(self.hps.data, "style2id"):
|
||||
self.style2id = self.hps.data.style2id
|
||||
else:
|
||||
self.style2id = {str(i): i for i in range(self.num_styles)}
|
||||
|
||||
self.style_vectors = np.load(self.style_vec_path)
|
||||
self.net_g = None
|
||||
|
||||
def load_net_g(self):
|
||||
self.net_g = get_net_g(
|
||||
model_path=self.model_path,
|
||||
version=self.hps.version,
|
||||
device=self.device,
|
||||
hps=self.hps,
|
||||
)
|
||||
|
||||
def get_style_vector(self, style_id, weight=1.0):
|
||||
mean = self.style_vectors[0]
|
||||
style_vec = self.style_vectors[style_id]
|
||||
style_vec = mean + (style_vec - mean) * weight
|
||||
return style_vec
|
||||
|
||||
def get_style_vector_from_audio(self, audio_path, weight=1.0):
|
||||
from style_gen import extract_style_vector
|
||||
|
||||
xvec = extract_style_vector(audio_path)
|
||||
mean = self.style_vectors[0]
|
||||
xvec = mean + (xvec - mean) * weight
|
||||
return xvec
|
||||
|
||||
def infer(
|
||||
self,
|
||||
text,
|
||||
language="JP",
|
||||
sid=0,
|
||||
reference_audio_path=None,
|
||||
sdp_ratio=0.2,
|
||||
noise=0.6,
|
||||
noisew=0.8,
|
||||
length=1.0,
|
||||
line_split=True,
|
||||
split_interval=0.2,
|
||||
style_text="",
|
||||
style_weight=0.7,
|
||||
use_style_text=False,
|
||||
style="0",
|
||||
emotion_weight=1.0,
|
||||
):
|
||||
if reference_audio_path == "":
|
||||
reference_audio_path = None
|
||||
if style_text == "" or not use_style_text:
|
||||
style_text = None
|
||||
|
||||
if self.net_g is None:
|
||||
self.load_net_g()
|
||||
if reference_audio_path is None:
|
||||
style_id = self.style2id[style]
|
||||
style_vector = self.get_style_vector(style_id, emotion_weight)
|
||||
else:
|
||||
style_vector = self.get_style_vector_from_audio(
|
||||
reference_audio_path, emotion_weight
|
||||
)
|
||||
if not line_split:
|
||||
with torch.no_grad():
|
||||
audio = infer(
|
||||
text=text,
|
||||
sdp_ratio=sdp_ratio,
|
||||
noise_scale=noise,
|
||||
noise_scale_w=noisew,
|
||||
length_scale=length,
|
||||
sid=sid,
|
||||
language=language,
|
||||
hps=self.hps,
|
||||
net_g=self.net_g,
|
||||
device=self.device,
|
||||
style_text=style_text,
|
||||
style_weight=style_weight,
|
||||
style_vec=style_vector,
|
||||
)
|
||||
else:
|
||||
texts = text.split("\n")
|
||||
texts = [t for t in texts if t != ""]
|
||||
audios = []
|
||||
with torch.no_grad():
|
||||
for i, t in enumerate(texts):
|
||||
audios.append(
|
||||
infer(
|
||||
text=t,
|
||||
sdp_ratio=sdp_ratio,
|
||||
noise_scale=noise,
|
||||
noise_scale_w=noisew,
|
||||
length_scale=length,
|
||||
sid=sid,
|
||||
language=language,
|
||||
hps=self.hps,
|
||||
net_g=self.net_g,
|
||||
device=self.device,
|
||||
style_text=style_text,
|
||||
style_weight=style_weight,
|
||||
style_vec=style_vector,
|
||||
)
|
||||
)
|
||||
if i != len(texts) - 1:
|
||||
audios.append(np.zeros(int(44100 * split_interval)))
|
||||
audio = np.concatenate(audios)
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
audio = convert_to_16_bit_wav(audio)
|
||||
return (self.hps.data.sampling_rate, audio)
|
||||
|
||||
|
||||
class ModelHolder:
|
||||
def __init__(self, root_dir, device):
|
||||
self.root_dir = root_dir
|
||||
self.device = device
|
||||
self.model_files_dict = {}
|
||||
self.current_model = None
|
||||
self.model_names = []
|
||||
self.models = []
|
||||
self.refresh()
|
||||
|
||||
def refresh(self):
|
||||
self.model_files_dict = {}
|
||||
self.model_names = []
|
||||
self.current_model = None
|
||||
model_dirs = [
|
||||
d
|
||||
for d in os.listdir(self.root_dir)
|
||||
if os.path.isdir(os.path.join(self.root_dir, d))
|
||||
]
|
||||
for model_name in model_dirs:
|
||||
model_dir = os.path.join(self.root_dir, model_name)
|
||||
model_files = [
|
||||
os.path.join(model_dir, f)
|
||||
for f in os.listdir(model_dir)
|
||||
if f.endswith(".pth") or f.endswith(".pt") or f.endswith(".safetensors")
|
||||
]
|
||||
if len(model_files) == 0:
|
||||
logger.info(
|
||||
f"No model files found in {self.root_dir}/{model_name}, so skip it"
|
||||
)
|
||||
self.model_files_dict[model_name] = model_files
|
||||
self.model_names.append(model_name)
|
||||
|
||||
def load_model(self, model_name, model_path):
|
||||
if model_name not in self.model_files_dict:
|
||||
raise Exception(f"モデル名{model_name}は存在しません")
|
||||
if model_path not in self.model_files_dict[model_name]:
|
||||
raise Exception(f"pthファイル{model_path}は存在しません")
|
||||
self.current_model = Model(
|
||||
model_path=model_path,
|
||||
config_path=os.path.join(self.root_dir, model_name, "config.json"),
|
||||
style_vec_path=os.path.join(self.root_dir, model_name, "style_vectors.npy"),
|
||||
device=self.device,
|
||||
)
|
||||
styles = list(self.current_model.style2id.keys())
|
||||
return (
|
||||
gr.Dropdown(choices=styles, value=styles[0]),
|
||||
gr.update(interactive=True, value="音声合成"),
|
||||
)
|
||||
|
||||
def update_model_files_dropdown(self, model_name):
|
||||
model_files = self.model_files_dict[model_name]
|
||||
return gr.Dropdown(choices=model_files, value=model_files[0])
|
||||
|
||||
def update_model_names_dropdown(self):
|
||||
self.refresh()
|
||||
initial_model_name = self.model_names[0]
|
||||
initial_model_files = self.model_files_dict[initial_model_name]
|
||||
return (
|
||||
gr.Dropdown(choices=self.model_names, value=initial_model_name),
|
||||
gr.Dropdown(choices=initial_model_files, value=initial_model_files[0]),
|
||||
gr.update(interactive=False), # For tts_button
|
||||
)
|
||||
|
||||
|
||||
def tts_fn(
|
||||
text,
|
||||
language,
|
||||
reference_audio_path,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
line_split,
|
||||
split_interval,
|
||||
style_text,
|
||||
style_weight,
|
||||
use_style_text,
|
||||
emotion,
|
||||
emotion_weight,
|
||||
):
|
||||
if is_hf_spaces and len(text) > limit:
|
||||
raise Exception(f"文字数が{limit}文字を超えています")
|
||||
|
||||
assert model_holder.current_model is not None
|
||||
|
||||
sr, audio = model_holder.current_model.infer(
|
||||
text=text,
|
||||
language=language,
|
||||
reference_audio_path=reference_audio_path,
|
||||
sdp_ratio=sdp_ratio,
|
||||
noise=noise_scale,
|
||||
noisew=noise_scale_w,
|
||||
length=length_scale,
|
||||
line_split=line_split,
|
||||
split_interval=split_interval,
|
||||
style_text=style_text,
|
||||
style_weight=style_weight,
|
||||
use_style_text=use_style_text,
|
||||
style=emotion,
|
||||
emotion_weight=emotion_weight,
|
||||
)
|
||||
return "Success", (sr, audio)
|
||||
|
||||
|
||||
initial_text = "こんにちは、初めまして。あなたの名前はなんていうの？"
|
||||
|
||||
example_local = [
|
||||
[initial_text, "JP"],
|
||||
[ # ChatGPTに考えてもらった告白セリフ
|
||||
"""私、ずっと前からあなたのことを見てきました。あなたの笑顔、優しさ、強さに、心惹かれていたんです。
|
||||
友達として過ごす中で、あなたのことがだんだんと特別な存在になっていくのがわかりました。
|
||||
えっと、私、あなたのことが好きです！もしよければ、私と付き合ってくれませんか？""",
|
||||
"JP",
|
||||
],
|
||||
[ # 夏目漱石『吾輩は猫である』
|
||||
"""吾輩は猫である。名前はまだ無い。
|
||||
どこで生れたかとんと見当がつかぬ。なんでも薄暗いじめじめした所でニャーニャー泣いていた事だけは記憶している。
|
||||
吾輩はここで始めて人間というものを見た。しかもあとで聞くと、それは書生という、人間中で一番獰悪な種族であったそうだ。
|
||||
この書生というのは時々我々を捕まえて煮て食うという話である。""",
|
||||
"JP",
|
||||
],
|
||||
[ # 梶井基次郎『桜の樹の下には』
|
||||
"""桜の樹の下には屍体が埋まっている！これは信じていいことなんだよ。
|
||||
何故って、桜の花があんなにも見事に咲くなんて信じられないことじゃないか。俺はあの美しさが信じられないので、このにさんにち不安だった。
|
||||
しかしいま、やっとわかるときが来た。桜の樹の下には屍体が埋まっている。これは信じていいことだ。""",
|
||||
"JP",
|
||||
],
|
||||
[ # ChatGPTと考えた、感情を表すセリフ
|
||||
"""やったー！テストで満点取れたよ！私とっても嬉しいな！
|
||||
どうして私の意見を無視するの？許せない！ムカつく！あんたなんか死ねばいいのに。
|
||||
あはははっ！この漫画めっちゃ笑える、見てよこれ、ふふふ、あはは。
|
||||
あなたがいなくなって、私は一人になっちゃって、泣いちゃいそうなほど悲しい。""",
|
||||
"JP",
|
||||
],
|
||||
[ # 上の丁寧語バージョン
|
||||
"""やりました！テストで満点取れましたよ！私とっても嬉しいです！
|
||||
どうして私の意見を無視するんですか？許せません！ムカつきます！あんたなんか死んでください。
|
||||
あはははっ！この漫画めっちゃ笑えます、見てくださいこれ、ふふふ、あはは。
|
||||
あなたがいなくなって、私は一人になっちゃって、泣いちゃいそうなほど悲しいです。""",
|
||||
"JP",
|
||||
],
|
||||
[ # ChatGPTに考えてもらった音声合成の説明文章
|
||||
"""音声合成は、機械学習を活用して、テキストから人の声を再現する技術です。この技術は、言語の構造を解析し、それに基づいて音声を生成します。
|
||||
この分野の最新の研究成果を使うと、より自然で表現豊かな音声の生成が可能である。深層学習の応用により、感情やアクセントを含む声質の微妙な変化も再現することが出来る。""",
|
||||
"JP",
|
||||
],
|
||||
[
|
||||
"Speech synthesis is the artificial production of human speech. A computer system used for this purpose is called a speech synthesizer, and can be implemented in software or hardware products.",
|
||||
"EN",
|
||||
],
|
||||
["语音合成是人工制造人类语音。用于此目的的计算机系统称为语音合成器，可以通过软件或硬件产品实现。", "ZH"],
|
||||
]
|
||||
|
||||
example_hf_spaces = [
|
||||
[initial_text, "JP"],
|
||||
["えっと、私、あなたのことが好きです！もしよければ付き合ってくれませんか？", "JP"],
|
||||
["吾輩は猫である。名前はまだ無い。", "JP"],
|
||||
["どこで生れたかとんと見当がつかぬ。なんでも薄暗いじめじめした所でニャーニャー泣いていた事だけは記憶している。", "JP"],
|
||||
["やったー！テストで満点取れたよ！私とっても嬉しいな！", "JP"],
|
||||
["どうして私の意見を無視するの？許せない！ムカつく！あんたなんか死ねばいいのに。", "JP"],
|
||||
["あはははっ！この漫画めっちゃ笑える、見てよこれ、ふふふ、あはは。", "JP"],
|
||||
["あなたがいなくなって、私は一人になっちゃって、泣いちゃいそうなほど悲しい。", "JP"],
|
||||
["深層学習の応用により、感情やアクセントを含む声質の微妙な変化も再現されている。", "JP"],
|
||||
]
|
||||
|
||||
initial_md = """
|
||||
# Bert-VITS2 okiba TTS デモ
|
||||
|
||||
[bert_vits2_okiba](https://huggingface.co/litagin/bert_vits2_okiba) のモデルのデモです。
|
||||
モデル名は[rvc_okiba](https://huggingface.co/litagin/rvc_okiba)のモデル名と対応しています。
|
||||
モデルは随時追加していきます。現在のモデルはすべてBert-VITS2のver 2.1のものです。
|
||||
|
||||
**定形サンプルは[こちら](https://huggingface.co/litagin/bert_vits2_okiba/blob/main/examples.md)から聴くほうが速いです。**
|
||||
|
||||
- huggingfaceのcpuで動くので、何故かやたら遅いことが多かったりなんか不安定で動かないときもあるみたいです。
|
||||
- huggingface上では最大100文字にしています。
|
||||
- Style textの実装あたりで本家の内部コードを改造しているので、このapp.pyをそのまま本家に使っても今のところは動きません。
|
||||
|
||||
現在のところはspeaker_id = 0に固定しています。
|
||||
"""
|
||||
|
||||
|
||||
def make_interactive():
|
||||
return gr.update(interactive=True, value="音声合成")
|
||||
|
||||
|
||||
def make_non_interactive():
|
||||
return gr.update(interactive=False, value="音声合成（モデルをロードしてください）")
|
||||
|
||||
|
||||
def gr_util(item):
|
||||
if item == "クラスタから選ぶ":
|
||||
return (gr.update(visible=True), gr.update(visible=False))
|
||||
else:
|
||||
return (gr.update(visible=False), gr.update(visible=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--cpu", action="store_true", help="Use CPU instead of GPU")
|
||||
parser.add_argument(
|
||||
"--dir", "-d", type=str, help="Model directory", default=config.out_dir
|
||||
)
|
||||
args = parser.parse_args()
|
||||
model_dir = args.dir
|
||||
|
||||
if args.cpu:
|
||||
device = "cpu"
|
||||
else:
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
|
||||
model_holder = ModelHolder(model_dir, device)
|
||||
|
||||
languages = ["JP", "EN", "ZH"]
|
||||
examples = example_hf_spaces if is_hf_spaces else example_local
|
||||
|
||||
model_names = model_holder.model_names
|
||||
initial_id = 1 if is_hf_spaces else 0
|
||||
initial_pth_files = model_holder.model_files_dict[model_names[initial_id]]
|
||||
|
||||
with gr.Blocks(theme="NoCrypt/miku") as app:
|
||||
gr.Markdown(initial_md)
|
||||
with gr.Row():
|
||||
with gr.Column():
|
||||
with gr.Row():
|
||||
with gr.Column(scale=3):
|
||||
model_name = gr.Dropdown(
|
||||
label="モデル一覧",
|
||||
choices=model_names,
|
||||
value=model_names[initial_id],
|
||||
)
|
||||
model_path = gr.Dropdown(
|
||||
label="モデルファイル",
|
||||
choices=initial_pth_files,
|
||||
value=initial_pth_files[0],
|
||||
)
|
||||
refresh_button = gr.Button(
|
||||
"モデル一覧を更新", scale=1, visible=not is_hf_spaces
|
||||
)
|
||||
load_button = gr.Button("モデルをロード", scale=1)
|
||||
text_input = gr.TextArea(label="テキスト", value=initial_text)
|
||||
use_style_text = gr.Checkbox(label="Style textを使う", value=False)
|
||||
style_text = gr.Textbox(
|
||||
label="Style text",
|
||||
placeholder="どうして私の意見を無視するの？許せない、ムカつく！死ねばいいのに。",
|
||||
info="このテキストの読み上げと似た声音・感情になりやすくなります。ただ抑揚やテンポ等が犠牲になるかも。",
|
||||
visible=False,
|
||||
)
|
||||
style_text_weight = gr.Slider(
|
||||
minimum=0,
|
||||
maximum=1,
|
||||
value=0.7,
|
||||
step=0.1,
|
||||
label="Style textの強さ",
|
||||
visible=False,
|
||||
)
|
||||
use_style_text.change(
|
||||
lambda x: (gr.Textbox(visible=x), gr.Slider(visible=x)),
|
||||
inputs=[use_style_text],
|
||||
outputs=[style_text, style_text_weight],
|
||||
)
|
||||
|
||||
line_split = gr.Checkbox(label="改行で分けて生成", value=True)
|
||||
split_interval = gr.Slider(
|
||||
minimum=0.1, maximum=2, value=0.5, step=0.1, label="分けた場合に挟む無音の長さ"
|
||||
)
|
||||
language = gr.Dropdown(choices=languages, value="JP", label="Language")
|
||||
with gr.Accordion(label="詳細設定", open=False):
|
||||
sdp_ratio = gr.Slider(
|
||||
minimum=0, maximum=1, value=0.2, step=0.1, label="SDP Ratio"
|
||||
)
|
||||
noise_scale = gr.Slider(
|
||||
minimum=0.1, maximum=2, value=0.6, step=0.1, label="Noise"
|
||||
)
|
||||
noise_scale_w = gr.Slider(
|
||||
minimum=0.1, maximum=2, value=0.8, step=0.1, label="Noise_W"
|
||||
)
|
||||
length_scale = gr.Slider(
|
||||
minimum=0.1, maximum=2, value=1.0, step=0.1, label="Length"
|
||||
)
|
||||
with gr.Column():
|
||||
style_mode = gr.Radio(
|
||||
["クラスタから選ぶ", "音声ファイルを入力"],
|
||||
label="スタイルの指定方法",
|
||||
value="クラスタから選ぶ",
|
||||
)
|
||||
style = gr.Dropdown(
|
||||
label="スタイル（0が平均スタイル）", choices=list(range(7)), value=0
|
||||
)
|
||||
style_weight = gr.Slider(
|
||||
minimum=0,
|
||||
maximum=20,
|
||||
value=1,
|
||||
step=0.1,
|
||||
label="スタイルの強さ",
|
||||
)
|
||||
ref_audio_path = gr.Audio(label="参照音声", type="filepath", visible=False)
|
||||
tts_button = gr.Button(
|
||||
"音声合成（モデルをロードしてください）", variant="primary", interactive=False
|
||||
)
|
||||
text_output = gr.Textbox(label="情報")
|
||||
audio_output = gr.Audio(label="結果")
|
||||
gr.Examples(examples, inputs=[text_input, language], label="テキスト例")
|
||||
|
||||
tts_button.click(
|
||||
tts_fn,
|
||||
inputs=[
|
||||
text_input,
|
||||
language,
|
||||
ref_audio_path,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
line_split,
|
||||
split_interval,
|
||||
style_text,
|
||||
style_text_weight,
|
||||
use_style_text,
|
||||
style,
|
||||
style_weight,
|
||||
],
|
||||
outputs=[text_output, audio_output],
|
||||
)
|
||||
|
||||
model_name.change(
|
||||
model_holder.update_model_files_dropdown,
|
||||
inputs=[model_name],
|
||||
outputs=[model_path],
|
||||
)
|
||||
|
||||
model_path.change(make_non_interactive, outputs=[tts_button])
|
||||
|
||||
refresh_button.click(
|
||||
model_holder.update_model_names_dropdown,
|
||||
outputs=[model_name, model_path, tts_button],
|
||||
)
|
||||
|
||||
load_button.click(
|
||||
model_holder.load_model,
|
||||
inputs=[model_name, model_path],
|
||||
outputs=[style, tts_button],
|
||||
)
|
||||
|
||||
style_mode.change(
|
||||
gr_util,
|
||||
inputs=[style_mode],
|
||||
outputs=[style, ref_audio_path],
|
||||
)
|
||||
|
||||
app.launch(inbrowser=True)
|
||||
@@ -4,9 +4,7 @@ from torch import nn
|
||||
from torch.nn import functional as F
|
||||
|
||||
import commons
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
from tools.log import logger as logging
|
||||
|
||||
|
||||
class LayerNorm(nn.Module):
|
||||
@@ -69,7 +67,7 @@ class Encoder(nn.Module):
|
||||
self.cond_layer_idx = (
|
||||
kwargs["cond_layer_idx"] if "cond_layer_idx" in kwargs else 2
|
||||
)
|
||||
logging.debug(self.gin_channels, self.cond_layer_idx)
|
||||
# logging.debug(self.gin_channels, self.cond_layer_idx)
|
||||
assert (
|
||||
self.cond_layer_idx < self.n_layers
|
||||
), "cond_layer_idx should be less than n_layers"
|
||||
|
||||
34
bert/bert-base-japanese-v3/.gitattributes
vendored
34
bert/bert-base-japanese-v3/.gitattributes
vendored
@@ -1,34 +0,0 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
@@ -1,53 +0,0 @@
|
||||
---
|
||||
license: apache-2.0
|
||||
datasets:
|
||||
- cc100
|
||||
- wikipedia
|
||||
language:
|
||||
- ja
|
||||
widget:
|
||||
- text: 東北大学で[MASK]の研究をしています。
|
||||
---
|
||||
|
||||
# BERT base Japanese (unidic-lite with whole word masking, CC-100 and jawiki-20230102)
|
||||
|
||||
This is a [BERT](https://github.com/google-research/bert) model pretrained on texts in the Japanese language.
|
||||
|
||||
This version of the model processes input texts with word-level tokenization based on the Unidic 2.1.2 dictionary (available in [unidic-lite](https://pypi.org/project/unidic-lite/) package), followed by the WordPiece subword tokenization.
|
||||
Additionally, the model is trained with the whole word masking enabled for the masked language modeling (MLM) objective.
|
||||
|
||||
The codes for the pretraining are available at [cl-tohoku/bert-japanese](https://github.com/cl-tohoku/bert-japanese/).
|
||||
|
||||
## Model architecture
|
||||
|
||||
The model architecture is the same as the original BERT base model; 12 layers, 768 dimensions of hidden states, and 12 attention heads.
|
||||
|
||||
## Training Data
|
||||
|
||||
The model is trained on the Japanese portion of [CC-100 dataset](https://data.statmt.org/cc-100/) and the Japanese version of Wikipedia.
|
||||
For Wikipedia, we generated a text corpus from the [Wikipedia Cirrussearch dump file](https://dumps.wikimedia.org/other/cirrussearch/) as of January 2, 2023.
|
||||
The corpus files generated from CC-100 and Wikipedia are 74.3GB and 4.9GB in size and consist of approximately 392M and 34M sentences, respectively.
|
||||
|
||||
For the purpose of splitting texts into sentences, we used [fugashi](https://github.com/polm/fugashi) with [mecab-ipadic-NEologd](https://github.com/neologd/mecab-ipadic-neologd) dictionary (v0.0.7).
|
||||
|
||||
## Tokenization
|
||||
|
||||
The texts are first tokenized by MeCab with the Unidic 2.1.2 dictionary and then split into subwords by the WordPiece algorithm.
|
||||
The vocabulary size is 32768.
|
||||
|
||||
We used [fugashi](https://github.com/polm/fugashi) and [unidic-lite](https://github.com/polm/unidic-lite) packages for the tokenization.
|
||||
|
||||
## Training
|
||||
|
||||
We trained the model first on the CC-100 corpus for 1M steps and then on the Wikipedia corpus for another 1M steps.
|
||||
For training of the MLM (masked language modeling) objective, we introduced whole word masking in which all of the subword tokens corresponding to a single word (tokenized by MeCab) are masked at once.
|
||||
|
||||
For training of each model, we used a v3-8 instance of Cloud TPUs provided by [TPU Research Cloud](https://sites.research.google/trc/about/).
|
||||
|
||||
## Licenses
|
||||
|
||||
The pretrained models are distributed under the Apache License 2.0.
|
||||
|
||||
## Acknowledgments
|
||||
|
||||
This model is trained with Cloud TPUs provided by [TPU Research Cloud](https://sites.research.google/trc/about/) program.
|
||||
@@ -1,19 +0,0 @@
|
||||
{
|
||||
"architectures": [
|
||||
"BertForPreTraining"
|
||||
],
|
||||
"attention_probs_dropout_prob": 0.1,
|
||||
"hidden_act": "gelu",
|
||||
"hidden_dropout_prob": 0.1,
|
||||
"hidden_size": 768,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 3072,
|
||||
"layer_norm_eps": 1e-12,
|
||||
"max_position_embeddings": 512,
|
||||
"model_type": "bert",
|
||||
"num_attention_heads": 12,
|
||||
"num_hidden_layers": 12,
|
||||
"pad_token_id": 0,
|
||||
"type_vocab_size": 2,
|
||||
"vocab_size": 32768
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
{
|
||||
"tokenizer_class": "BertJapaneseTokenizer",
|
||||
"model_max_length": 512,
|
||||
"do_lower_case": false,
|
||||
"word_tokenizer_type": "mecab",
|
||||
"subword_tokenizer_type": "wordpiece",
|
||||
"mecab_kwargs": {
|
||||
"mecab_dic": "unidic_lite"
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
34
bert/bert-large-japanese-v2/.gitattributes
vendored
34
bert/bert-large-japanese-v2/.gitattributes
vendored
@@ -1,34 +0,0 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
@@ -1,53 +0,0 @@
|
||||
---
|
||||
license: apache-2.0
|
||||
datasets:
|
||||
- cc100
|
||||
- wikipedia
|
||||
language:
|
||||
- ja
|
||||
widget:
|
||||
- text: 東北大学で[MASK]の研究をしています。
|
||||
---
|
||||
|
||||
# BERT large Japanese (unidic-lite with whole word masking, CC-100 and jawiki-20230102)
|
||||
|
||||
This is a [BERT](https://github.com/google-research/bert) model pretrained on texts in the Japanese language.
|
||||
|
||||
This version of the model processes input texts with word-level tokenization based on the Unidic 2.1.2 dictionary (available in [unidic-lite](https://pypi.org/project/unidic-lite/) package), followed by the WordPiece subword tokenization.
|
||||
Additionally, the model is trained with the whole word masking enabled for the masked language modeling (MLM) objective.
|
||||
|
||||
The codes for the pretraining are available at [cl-tohoku/bert-japanese](https://github.com/cl-tohoku/bert-japanese/).
|
||||
|
||||
## Model architecture
|
||||
|
||||
The model architecture is the same as the original BERT large model; 24 layers, 1024 dimensions of hidden states, and 16 attention heads.
|
||||
|
||||
## Training Data
|
||||
|
||||
The model is trained on the Japanese portion of [CC-100 dataset](https://data.statmt.org/cc-100/) and the Japanese version of Wikipedia.
|
||||
For Wikipedia, we generated a text corpus from the [Wikipedia Cirrussearch dump file](https://dumps.wikimedia.org/other/cirrussearch/) as of January 2, 2023.
|
||||
The corpus files generated from CC-100 and Wikipedia are 74.3GB and 4.9GB in size and consist of approximately 392M and 34M sentences, respectively.
|
||||
|
||||
For the purpose of splitting texts into sentences, we used [fugashi](https://github.com/polm/fugashi) with [mecab-ipadic-NEologd](https://github.com/neologd/mecab-ipadic-neologd) dictionary (v0.0.7).
|
||||
|
||||
## Tokenization
|
||||
|
||||
The texts are first tokenized by MeCab with the Unidic 2.1.2 dictionary and then split into subwords by the WordPiece algorithm.
|
||||
The vocabulary size is 32768.
|
||||
|
||||
We used [fugashi](https://github.com/polm/fugashi) and [unidic-lite](https://github.com/polm/unidic-lite) packages for the tokenization.
|
||||
|
||||
## Training
|
||||
|
||||
We trained the model first on the CC-100 corpus for 1M steps and then on the Wikipedia corpus for another 1M steps.
|
||||
For training of the MLM (masked language modeling) objective, we introduced whole word masking in which all of the subword tokens corresponding to a single word (tokenized by MeCab) are masked at once.
|
||||
|
||||
For training of each model, we used a v3-8 instance of Cloud TPUs provided by [TPU Research Cloud](https://sites.research.google/trc/about/).
|
||||
|
||||
## Licenses
|
||||
|
||||
The pretrained models are distributed under the Apache License 2.0.
|
||||
|
||||
## Acknowledgments
|
||||
|
||||
This model is trained with Cloud TPUs provided by [TPU Research Cloud](https://sites.research.google/trc/about/) program.
|
||||
@@ -1,19 +0,0 @@
|
||||
{
|
||||
"architectures": [
|
||||
"BertForPreTraining"
|
||||
],
|
||||
"attention_probs_dropout_prob": 0.1,
|
||||
"hidden_act": "gelu",
|
||||
"hidden_dropout_prob": 0.1,
|
||||
"hidden_size": 1024,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 4096,
|
||||
"layer_norm_eps": 1e-12,
|
||||
"max_position_embeddings": 512,
|
||||
"model_type": "bert",
|
||||
"num_attention_heads": 16,
|
||||
"num_hidden_layers": 24,
|
||||
"pad_token_id": 0,
|
||||
"type_vocab_size": 2,
|
||||
"vocab_size": 32768
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
{
|
||||
"tokenizer_class": "BertJapaneseTokenizer",
|
||||
"model_max_length": 512,
|
||||
"do_lower_case": false,
|
||||
"word_tokenizer_type": "mecab",
|
||||
"subword_tokenizer_type": "wordpiece",
|
||||
"mecab_kwargs": {
|
||||
"mecab_dic": "unidic_lite"
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
34
bert/deberta-v2-large-japanese/.gitattributes
vendored
34
bert/deberta-v2-large-japanese/.gitattributes
vendored
@@ -1,34 +0,0 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
@@ -1,111 +0,0 @@
|
||||
---
|
||||
language: ja
|
||||
license: cc-by-sa-4.0
|
||||
library_name: transformers
|
||||
tags:
|
||||
- deberta
|
||||
- deberta-v2
|
||||
- fill-mask
|
||||
datasets:
|
||||
- wikipedia
|
||||
- cc100
|
||||
- oscar
|
||||
metrics:
|
||||
- accuracy
|
||||
mask_token: "[MASK]"
|
||||
widget:
|
||||
- text: "京都 大学 で 自然 言語 処理 を [MASK] する 。"
|
||||
---
|
||||
|
||||
# Model Card for Japanese DeBERTa V2 large
|
||||
|
||||
## Model description
|
||||
|
||||
This is a Japanese DeBERTa V2 large model pre-trained on Japanese Wikipedia, the Japanese portion of CC-100, and the
|
||||
Japanese portion of OSCAR.
|
||||
|
||||
## How to use
|
||||
|
||||
You can use this model for masked language modeling as follows:
|
||||
|
||||
```python
|
||||
from transformers import AutoTokenizer, AutoModelForMaskedLM
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained('ku-nlp/deberta-v2-large-japanese')
|
||||
model = AutoModelForMaskedLM.from_pretrained('ku-nlp/deberta-v2-large-japanese')
|
||||
|
||||
sentence = '京都 大学 で 自然 言語 処理 を [MASK] する 。' # input should be segmented into words by Juman++ in advance
|
||||
encoding = tokenizer(sentence, return_tensors='pt')
|
||||
...
|
||||
```
|
||||
|
||||
You can also fine-tune this model on downstream tasks.
|
||||
|
||||
## Tokenization
|
||||
|
||||
The input text should be segmented into words by [Juman++](https://github.com/ku-nlp/jumanpp) in
|
||||
advance. [Juman++ 2.0.0-rc3](https://github.com/ku-nlp/jumanpp/releases/tag/v2.0.0-rc3) was used for pre-training. Each
|
||||
word is tokenized into subwords by [sentencepiece](https://github.com/google/sentencepiece).
|
||||
|
||||
## Training data
|
||||
|
||||
We used the following corpora for pre-training:
|
||||
|
||||
- Japanese Wikipedia (as of 20221020, 3.2GB, 27M sentences, 1.3M documents)
|
||||
- Japanese portion of CC-100 (85GB, 619M sentences, 66M documents)
|
||||
- Japanese portion of OSCAR (54GB, 326M sentences, 25M documents)
|
||||
|
||||
Note that we filtered out documents annotated with "header", "footer", or "noisy" tags in OSCAR.
|
||||
Also note that Japanese Wikipedia was duplicated 10 times to make the total size of the corpus comparable to that of
|
||||
CC-100 and OSCAR. As a result, the total size of the training data is 171GB.
|
||||
|
||||
## Training procedure
|
||||
|
||||
We first segmented texts in the corpora into words using [Juman++](https://github.com/ku-nlp/jumanpp).
|
||||
Then, we built a sentencepiece model with 32000 tokens including words ([JumanDIC](https://github.com/ku-nlp/JumanDIC))
|
||||
and subwords induced by the unigram language model of [sentencepiece](https://github.com/google/sentencepiece).
|
||||
|
||||
We tokenized the segmented corpora into subwords using the sentencepiece model and trained the Japanese DeBERTa model
|
||||
using [transformers](https://github.com/huggingface/transformers) library.
|
||||
The training took 36 days using 8 NVIDIA A100-SXM4-40GB GPUs.
|
||||
|
||||
The following hyperparameters were used during pre-training:
|
||||
|
||||
- learning_rate: 1e-4
|
||||
- per_device_train_batch_size: 18
|
||||
- distributed_type: multi-GPU
|
||||
- num_devices: 8
|
||||
- gradient_accumulation_steps: 16
|
||||
- total_train_batch_size: 2,304
|
||||
- max_seq_length: 512
|
||||
- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-06
|
||||
- lr_scheduler_type: linear schedule with warmup
|
||||
- training_steps: 300,000
|
||||
- warmup_steps: 10,000
|
||||
|
||||
The accuracy of the trained model on the masked language modeling task was 0.799.
|
||||
The evaluation set consists of 5,000 randomly sampled documents from each of the training corpora.
|
||||
|
||||
## Fine-tuning on NLU tasks
|
||||
|
||||
We fine-tuned the following models and evaluated them on the dev set of JGLUE.
|
||||
We tuned learning rate and training epochs for each model and task
|
||||
following [the JGLUE paper](https://www.jstage.jst.go.jp/article/jnlp/30/1/30_63/_pdf/-char/ja).
|
||||
|
||||
| Model | MARC-ja/acc | JSTS/pearson | JSTS/spearman | JNLI/acc | JSQuAD/EM | JSQuAD/F1 | JComQA/acc |
|
||||
|-------------------------------|-------------|--------------|---------------|----------|-----------|-----------|------------|
|
||||
| Waseda RoBERTa base | 0.965 | 0.913 | 0.876 | 0.905 | 0.853 | 0.916 | 0.853 |
|
||||
| Waseda RoBERTa large (seq512) | 0.969 | 0.925 | 0.890 | 0.928 | 0.910 | 0.955 | 0.900 |
|
||||
| LUKE Japanese base* | 0.965 | 0.916 | 0.877 | 0.912 | - | - | 0.842 |
|
||||
| LUKE Japanese large* | 0.965 | 0.932 | 0.902 | 0.927 | - | - | 0.893 |
|
||||
| DeBERTaV2 base | 0.970 | 0.922 | 0.886 | 0.922 | 0.899 | 0.951 | 0.873 |
|
||||
| DeBERTaV2 large | 0.968 | 0.925 | 0.892 | 0.924 | 0.912 | 0.959 | 0.890 |
|
||||
|
||||
*The scores of LUKE are from [the official repository](https://github.com/studio-ousia/luke).
|
||||
|
||||
## Acknowledgments
|
||||
|
||||
This work was supported by Joint Usage/Research Center for Interdisciplinary Large-scale Information Infrastructures (
|
||||
JHPCN) through General Collaboration Project no. jh221004, "Developing a Platform for Constructing and Sharing of
|
||||
Large-Scale Japanese Language Models".
|
||||
For training models, we used the mdx: a platform for the data-driven future.
|
||||
@@ -1,38 +0,0 @@
|
||||
{
|
||||
"_name_or_path": "configs/deberta_v2_large.json",
|
||||
"architectures": [
|
||||
"DebertaV2ForMaskedLM"
|
||||
],
|
||||
"attention_head_size": 64,
|
||||
"attention_probs_dropout_prob": 0.1,
|
||||
"conv_act": "gelu",
|
||||
"conv_kernel_size": 3,
|
||||
"hidden_act": "gelu",
|
||||
"hidden_dropout_prob": 0.1,
|
||||
"hidden_size": 1024,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 4096,
|
||||
"layer_norm_eps": 1e-07,
|
||||
"max_position_embeddings": 512,
|
||||
"max_relative_positions": -1,
|
||||
"model_type": "deberta-v2",
|
||||
"norm_rel_ebd": "layer_norm",
|
||||
"num_attention_heads": 16,
|
||||
"num_hidden_layers": 24,
|
||||
"pad_token_id": 0,
|
||||
"pooler_dropout": 0,
|
||||
"pooler_hidden_act": "gelu",
|
||||
"pooler_hidden_size": 1024,
|
||||
"pos_att_type": [
|
||||
"p2c",
|
||||
"c2p"
|
||||
],
|
||||
"position_biased_input": false,
|
||||
"position_buckets": 256,
|
||||
"relative_attention": true,
|
||||
"share_att_key": true,
|
||||
"torch_dtype": "float32",
|
||||
"transformers_version": "4.23.1",
|
||||
"type_vocab_size": 0,
|
||||
"vocab_size": 32000
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
{
|
||||
"bos_token": "[CLS]",
|
||||
"cls_token": "[CLS]",
|
||||
"eos_token": "[SEP]",
|
||||
"mask_token": "[MASK]",
|
||||
"pad_token": "[PAD]",
|
||||
"sep_token": "[SEP]",
|
||||
"unk_token": "[UNK]"
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"bos_token": "[CLS]",
|
||||
"cls_token": "[CLS]",
|
||||
"do_lower_case": false,
|
||||
"eos_token": "[SEP]",
|
||||
"keep_accents": true,
|
||||
"mask_token": "[MASK]",
|
||||
"pad_token": "[PAD]",
|
||||
"sep_token": "[SEP]",
|
||||
"sp_model_kwargs": {},
|
||||
"special_tokens_map_file": null,
|
||||
"split_by_punct": false,
|
||||
"tokenizer_class": "DebertaV2Tokenizer",
|
||||
"unk_token": "[UNK]"
|
||||
}
|
||||
17
bert_gen.py
17
bert_gen.py
@@ -1,12 +1,15 @@
|
||||
import torch
|
||||
import argparse
|
||||
import sys
|
||||
from multiprocessing import Pool
|
||||
|
||||
import torch
|
||||
import torch.multiprocessing as mp
|
||||
from tqdm import tqdm
|
||||
|
||||
import commons
|
||||
import utils
|
||||
from tqdm import tqdm
|
||||
from text import check_bert_models, cleaned_text_to_sequence, get_bert
|
||||
import argparse
|
||||
import torch.multiprocessing as mp
|
||||
from config import config
|
||||
from text import cleaned_text_to_sequence, get_bert
|
||||
|
||||
|
||||
def process_line(x):
|
||||
@@ -59,7 +62,6 @@ if __name__ == "__main__":
|
||||
args, _ = parser.parse_known_args()
|
||||
config_path = args.config
|
||||
hps = utils.get_hparams_from_file(config_path)
|
||||
check_bert_models()
|
||||
lines = []
|
||||
with open(hps.data.training_files, encoding="utf-8") as f:
|
||||
lines.extend(f.readlines())
|
||||
@@ -74,8 +76,9 @@ if __name__ == "__main__":
|
||||
for _ in tqdm(
|
||||
pool.imap_unordered(process_line, zip(lines, add_blank)),
|
||||
total=len(lines),
|
||||
file=sys.stdout,
|
||||
):
|
||||
# 这里是缩进的代码块，表示循环体
|
||||
pass # 使用pass语句作为占位符
|
||||
|
||||
print(f"bert生成完毕!, 共有{len(lines)}个bert.pt生成!")
|
||||
print(f"bert.pt is generated! total: {len(lines)} bert.pt files.")
|
||||
|
||||
64
clap_gen.py
64
clap_gen.py
@@ -1,64 +0,0 @@
|
||||
import argparse
|
||||
from multiprocessing import Pool, cpu_count
|
||||
|
||||
import torch
|
||||
import torch.multiprocessing as mp
|
||||
from tqdm import tqdm
|
||||
|
||||
import utils
|
||||
from config import config
|
||||
from oldVersion.V220.clap_wrapper import get_clap_audio_feature
|
||||
import librosa
|
||||
import os
|
||||
|
||||
os.environ["OMP_NUM_THREADS"] = "1"
|
||||
os.environ["MKL_NUM_THREADS"] = "1"
|
||||
|
||||
|
||||
def process_line(line):
|
||||
device = config.emo_gen_config.device
|
||||
if config.emo_gen_config.use_multi_device:
|
||||
rank = mp.current_process()._identity
|
||||
rank = rank[0] if len(rank) > 0 else 0
|
||||
if torch.cuda.is_available():
|
||||
gpu_id = rank % torch.cuda.device_count()
|
||||
device = torch.device(f"cuda:{gpu_id}")
|
||||
else:
|
||||
device = torch.device("cpu")
|
||||
wav_path, _, language_str, text, phones, tone, word2ph = line.strip().split("|")
|
||||
|
||||
clap_path = wav_path.replace(".WAV", ".wav").replace(".wav", ".emo.npy")
|
||||
if os.path.isfile(clap_path):
|
||||
return
|
||||
|
||||
audio = librosa.load(wav_path, 48000)[0]
|
||||
# audio = librosa.resample(audio, 44100, 48000)
|
||||
|
||||
clap = get_clap_audio_feature(audio, device)
|
||||
torch.save(clap, clap_path)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"-c", "--config", type=str, default=config.emo_gen_config.config_path
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num_processes", type=int, default=config.emo_gen_config.num_processes
|
||||
)
|
||||
args, _ = parser.parse_known_args()
|
||||
config_path = args.config
|
||||
hps = utils.get_hparams_from_file(config_path)
|
||||
lines = []
|
||||
with open(hps.data.training_files, encoding="utf-8") as f:
|
||||
lines.extend(f.readlines())
|
||||
|
||||
with open(hps.data.validation_files, encoding="utf-8") as f:
|
||||
lines.extend(f.readlines())
|
||||
if len(lines) != 0:
|
||||
num_processes = min(args.num_processes, cpu_count())
|
||||
with Pool(processes=num_processes) as pool:
|
||||
for _ in tqdm(pool.imap_unordered(process_line, lines), total=len(lines)):
|
||||
pass
|
||||
|
||||
print(f"clap生成完毕!, 共有{len(lines)}个emo.pt生成!")
|
||||
337
clustering.ipynb
Normal file
337
clustering.ipynb
Normal file
File diff suppressed because one or more lines are too long
@@ -105,12 +105,6 @@ def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
|
||||
return acts
|
||||
|
||||
|
||||
def convert_pad_shape(pad_shape):
|
||||
layer = pad_shape[::-1]
|
||||
pad_shape = [item for sublist in layer for item in sublist]
|
||||
return pad_shape
|
||||
|
||||
|
||||
def shift_1d(x):
|
||||
x = F.pad(x, convert_pad_shape([[0, 0], [0, 0], [1, 0]]))[:, :, :-1]
|
||||
return x
|
||||
|
||||
@@ -1,89 +0,0 @@
|
||||
from collections import OrderedDict
|
||||
from text.symbols import symbols
|
||||
import torch
|
||||
|
||||
from tools.log import logger
|
||||
import utils
|
||||
from models import SynthesizerTrn
|
||||
import os
|
||||
|
||||
|
||||
def copyStateDict(state_dict):
|
||||
if list(state_dict.keys())[0].startswith("module"):
|
||||
start_idx = 1
|
||||
else:
|
||||
start_idx = 0
|
||||
new_state_dict = OrderedDict()
|
||||
for k, v in state_dict.items():
|
||||
name = ",".join(k.split(".")[start_idx:])
|
||||
new_state_dict[name] = v
|
||||
return new_state_dict
|
||||
|
||||
|
||||
def removeOptimizer(config: str, input_model: str, ishalf: bool, output_model: str):
|
||||
hps = utils.get_hparams_from_file(config)
|
||||
|
||||
net_g = SynthesizerTrn(
|
||||
len(symbols),
|
||||
hps.data.filter_length // 2 + 1,
|
||||
hps.train.segment_size // hps.data.hop_length,
|
||||
n_speakers=hps.data.n_speakers,
|
||||
**hps.model,
|
||||
)
|
||||
|
||||
optim_g = torch.optim.AdamW(
|
||||
net_g.parameters(),
|
||||
hps.train.learning_rate,
|
||||
betas=hps.train.betas,
|
||||
eps=hps.train.eps,
|
||||
)
|
||||
|
||||
state_dict_g = torch.load(input_model, map_location="cpu")
|
||||
new_dict_g = copyStateDict(state_dict_g)
|
||||
keys = []
|
||||
for k, v in new_dict_g["model"].items():
|
||||
if "enc_q" in k:
|
||||
continue # noqa: E701
|
||||
keys.append(k)
|
||||
|
||||
new_dict_g = (
|
||||
{k: new_dict_g["model"][k].half() for k in keys}
|
||||
if ishalf
|
||||
else {k: new_dict_g["model"][k] for k in keys}
|
||||
)
|
||||
|
||||
torch.save(
|
||||
{
|
||||
"model": new_dict_g,
|
||||
"iteration": 0,
|
||||
"optimizer": optim_g.state_dict(),
|
||||
"learning_rate": 0.0001,
|
||||
},
|
||||
output_model,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("-c", "--config", type=str, default="configs/config.json")
|
||||
parser.add_argument("-i", "--input", type=str)
|
||||
parser.add_argument("-o", "--output", type=str, default=None)
|
||||
parser.add_argument(
|
||||
"-hf", "--half", action="store_true", default=False, help="Save as FP16"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
output = args.output
|
||||
|
||||
if output is None:
|
||||
import os.path
|
||||
|
||||
filename, ext = os.path.splitext(args.input)
|
||||
half = "_half" if args.half else ""
|
||||
output = filename + "_release" + half + ext
|
||||
|
||||
removeOptimizer(args.config, args.input, args.half, output)
|
||||
logger.info(f"压缩模型成功, 输出模型: {os.path.abspath(output)}")
|
||||
40
config.py
40
config.py
@@ -91,20 +91,18 @@ class Bert_gen_config:
|
||||
return cls(**data)
|
||||
|
||||
|
||||
class Emo_gen_config:
|
||||
"""emo_gen 配置"""
|
||||
class Style_gen_config:
|
||||
"""style_gen 配置"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config_path: str,
|
||||
num_processes: int = 2,
|
||||
device: str = "cuda",
|
||||
use_multi_device: bool = False,
|
||||
):
|
||||
self.config_path = config_path
|
||||
self.num_processes = num_processes
|
||||
self.device = device
|
||||
self.use_multi_device = use_multi_device
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, dataset_path: str, data: Dict[str, any]):
|
||||
@@ -120,14 +118,14 @@ class Train_ms_config:
|
||||
self,
|
||||
config_path: str,
|
||||
env: Dict[str, any],
|
||||
base: Dict[str, any],
|
||||
# base: Dict[str, any],
|
||||
model: str,
|
||||
num_workers: int,
|
||||
spec_cache: bool,
|
||||
keep_ckpts: int,
|
||||
):
|
||||
self.env = env # 需要加载的环境变量
|
||||
self.base = base # 底模配置
|
||||
# self.base = base # 底模配置
|
||||
self.model = model # 训练模型存储目录，该路径为相对于dataset_path的路径，而非项目根目录
|
||||
self.config_path = config_path # 配置文件路径
|
||||
self.num_workers = num_workers # worker数量
|
||||
@@ -202,17 +200,25 @@ class Config:
|
||||
if not os.path.isfile(config_path) and os.path.isfile("default_config.yml"):
|
||||
shutil.copy(src="default_config.yml", dst=config_path)
|
||||
print(
|
||||
f"已根据默认配置文件default_config.yml生成配置文件{config_path}。请按该配置文件的说明进行配置后重新运行。"
|
||||
f"A configuration file {config_path} has been generated based on the default configuration file default_config.yml."
|
||||
)
|
||||
print(
|
||||
"If you have no special needs, please do not modify default_config.yml."
|
||||
)
|
||||
print("如无特殊需求，请勿修改default_config.yml或备份该文件。")
|
||||
sys.exit(0)
|
||||
with open(file=config_path, mode="r", encoding="utf-8") as file:
|
||||
yaml_config: Dict[str, any] = yaml.safe_load(file.read())
|
||||
dataset_path: str = yaml_config["dataset_path"]
|
||||
openi_token: str = yaml_config["openi_token"]
|
||||
model_name: str = yaml_config["model_name"]
|
||||
self.model_name: str = model_name
|
||||
if "dataset_path" in yaml_config:
|
||||
dataset_path = yaml_config["dataset_path"]
|
||||
else:
|
||||
dataset_path = f"Data/{model_name}"
|
||||
self.out_dir = yaml_config["out_dir"]
|
||||
# openi_token: str = yaml_config["openi_token"]
|
||||
self.dataset_path: str = dataset_path
|
||||
self.mirror: str = yaml_config["mirror"]
|
||||
self.openi_token: str = openi_token
|
||||
# self.mirror: str = yaml_config["mirror"]
|
||||
# self.openi_token: str = openi_token
|
||||
self.resample_config: Resample_config = Resample_config.from_dict(
|
||||
dataset_path, yaml_config["resample"]
|
||||
)
|
||||
@@ -224,8 +230,8 @@ class Config:
|
||||
self.bert_gen_config: Bert_gen_config = Bert_gen_config.from_dict(
|
||||
dataset_path, yaml_config["bert_gen"]
|
||||
)
|
||||
self.emo_gen_config: Emo_gen_config = Emo_gen_config.from_dict(
|
||||
dataset_path, yaml_config["emo_gen"]
|
||||
self.style_gen_config: Style_gen_config = Style_gen_config.from_dict(
|
||||
dataset_path, yaml_config["style_gen"]
|
||||
)
|
||||
self.train_ms_config: Train_ms_config = Train_ms_config.from_dict(
|
||||
dataset_path, yaml_config["train_ms"]
|
||||
@@ -236,9 +242,9 @@ class Config:
|
||||
self.server_config: Server_config = Server_config.from_dict(
|
||||
yaml_config["server"]
|
||||
)
|
||||
self.translate_config: Translate_config = Translate_config.from_dict(
|
||||
yaml_config["translate"]
|
||||
)
|
||||
# self.translate_config: Translate_config = Translate_config.from_dict(
|
||||
# yaml_config["translate"]
|
||||
# )
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
@@ -1,75 +0,0 @@
|
||||
{
|
||||
"train": {
|
||||
"log_interval": 200,
|
||||
"eval_interval": 1000,
|
||||
"save_compressed_models": true,
|
||||
"seed": 42,
|
||||
"epochs": 1000,
|
||||
"learning_rate": 0.0002,
|
||||
"betas": [0.8, 0.99],
|
||||
"eps": 1e-9,
|
||||
"batch_size": 16,
|
||||
"bf16_run": false,
|
||||
"lr_decay": 0.99995,
|
||||
"segment_size": 16384,
|
||||
"init_lr_ratio": 1,
|
||||
"warmup_epochs": 0,
|
||||
"c_mel": 45,
|
||||
"c_kl": 1.0,
|
||||
"c_commit": 100,
|
||||
"skip_optimizer": true,
|
||||
"freeze_ZH_bert": false,
|
||||
"freeze_JP_bert": false,
|
||||
"freeze_EN_bert": false,
|
||||
"freeze_emo": false
|
||||
},
|
||||
"data": {
|
||||
"training_files": "filelists/train.list",
|
||||
"validation_files": "filelists/val.list",
|
||||
"max_wav_value": 32768.0,
|
||||
"sampling_rate": 44100,
|
||||
"filter_length": 2048,
|
||||
"hop_length": 512,
|
||||
"win_length": 2048,
|
||||
"n_mel_channels": 128,
|
||||
"mel_fmin": 0.0,
|
||||
"mel_fmax": null,
|
||||
"add_blank": true,
|
||||
"n_speakers": 850,
|
||||
"cleaned_text": true
|
||||
},
|
||||
"model": {
|
||||
"use_spk_conditioned_encoder": true,
|
||||
"use_noise_scaled_mas": true,
|
||||
"use_mel_posterior_encoder": false,
|
||||
"use_duration_discriminator": true,
|
||||
"inter_channels": 192,
|
||||
"hidden_channels": 192,
|
||||
"filter_channels": 768,
|
||||
"n_heads": 2,
|
||||
"n_layers": 6,
|
||||
"kernel_size": 3,
|
||||
"p_dropout": 0.1,
|
||||
"resblock": "1",
|
||||
"resblock_kernel_sizes": [3, 7, 11],
|
||||
"resblock_dilation_sizes": [
|
||||
[1, 3, 5],
|
||||
[1, 3, 5],
|
||||
[1, 3, 5]
|
||||
],
|
||||
"upsample_rates": [8, 8, 2, 2, 2],
|
||||
"upsample_initial_channel": 512,
|
||||
"upsample_kernel_sizes": [16, 16, 8, 2, 2],
|
||||
"n_layers_q": 3,
|
||||
"use_spectral_norm": false,
|
||||
"gin_channels": 512,
|
||||
"slm": {
|
||||
"model": "./slm/wavlm-base-plus",
|
||||
"sr": 16000,
|
||||
"hidden": 768,
|
||||
"nlayers": 13,
|
||||
"initial_channel": 64
|
||||
}
|
||||
},
|
||||
"version": "2.3"
|
||||
}
|
||||
@@ -1,26 +1,29 @@
|
||||
{
|
||||
"model_name": "your_model_name",
|
||||
"train": {
|
||||
"log_interval": 200,
|
||||
"eval_interval": 1000,
|
||||
"save_compressed_models": true,
|
||||
"seed": 42,
|
||||
"epochs": 100,
|
||||
"epochs": 1000,
|
||||
"learning_rate": 0.0002,
|
||||
"betas": [0.8, 0.99],
|
||||
"eps": 1e-9,
|
||||
"batch_size": 4,
|
||||
"fp16_run": false,
|
||||
"bf16_run": true,
|
||||
"lr_decay": 0.99995,
|
||||
"segment_size": 16384,
|
||||
"init_lr_ratio": 1,
|
||||
"warmup_epochs": 0,
|
||||
"c_mel": 45,
|
||||
"c_kl": 1.0,
|
||||
"skip_optimizer": true
|
||||
"skip_optimizer": false,
|
||||
"freeze_ZH_bert": false,
|
||||
"freeze_JP_bert": false,
|
||||
"freeze_EN_bert": false
|
||||
},
|
||||
"data": {
|
||||
"training_files": "Data/yksi/filelists/train.list",
|
||||
"validation_files": "Data/yksi/filelists/val.list",
|
||||
"training_files": "Data/your_model_name/filelists/train.list",
|
||||
"validation_files": "Data/your_model_name/filelists/val.list",
|
||||
"max_wav_value": 32768.0,
|
||||
"sampling_rate": 44100,
|
||||
"filter_length": 2048,
|
||||
@@ -31,7 +34,11 @@
|
||||
"mel_fmax": null,
|
||||
"add_blank": true,
|
||||
"n_speakers": 1,
|
||||
"cleaned_text": true
|
||||
"cleaned_text": true,
|
||||
"num_styles": 1,
|
||||
"style2id": {
|
||||
"Neutral": 0
|
||||
}
|
||||
},
|
||||
"model": {
|
||||
"use_spk_conditioned_encoder": true,
|
||||
@@ -59,5 +66,5 @@
|
||||
"use_spectral_norm": false,
|
||||
"gin_channels": 256
|
||||
},
|
||||
"version": "2.1"
|
||||
"version": "1.0"
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
|
||||
#yml_code {
|
||||
height: 600px;
|
||||
flex-grow: inherit;
|
||||
overflow-y: auto;
|
||||
}
|
||||
|
||||
#json_code {
|
||||
height: 600px;
|
||||
flex-grow: inherit;
|
||||
overflow-y: auto;
|
||||
}
|
||||
|
||||
#gpu_code {
|
||||
height: 300px;
|
||||
flex-grow: inherit;
|
||||
overflow-y: auto;
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import random
|
||||
import torch
|
||||
import torch.utils.data
|
||||
from tqdm import tqdm
|
||||
import numpy as np
|
||||
from tools.log import logger
|
||||
import commons
|
||||
from mel_processing import spectrogram_torch, mel_spectrogram_torch
|
||||
@@ -92,8 +93,19 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
|
||||
|
||||
spec, wav = self.get_audio(audiopath)
|
||||
sid = torch.LongTensor([int(self.spk_map[sid])])
|
||||
|
||||
return (phones, spec, wav, sid, tone, language, bert, ja_bert, en_bert)
|
||||
style_vec = torch.FloatTensor(np.load(f"{audiopath}.npy"))
|
||||
return (
|
||||
phones,
|
||||
spec,
|
||||
wav,
|
||||
sid,
|
||||
tone,
|
||||
language,
|
||||
bert,
|
||||
ja_bert,
|
||||
en_bert,
|
||||
style_vec,
|
||||
)
|
||||
|
||||
def get_audio(self, filename):
|
||||
audio, sampling_rate = load_wav_to_torch(filename)
|
||||
@@ -156,15 +168,15 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
|
||||
|
||||
if language_str == "ZH":
|
||||
bert = bert_ori
|
||||
ja_bert = torch.randn(1024, len(phone))
|
||||
en_bert = torch.randn(1024, len(phone))
|
||||
ja_bert = torch.zeros(1024, len(phone))
|
||||
en_bert = torch.zeros(1024, len(phone))
|
||||
elif language_str == "JP":
|
||||
bert = torch.randn(1024, len(phone))
|
||||
bert = torch.zeros(1024, len(phone))
|
||||
ja_bert = bert_ori
|
||||
en_bert = torch.randn(1024, len(phone))
|
||||
en_bert = torch.zeros(1024, len(phone))
|
||||
elif language_str == "EN":
|
||||
bert = torch.randn(1024, len(phone))
|
||||
ja_bert = torch.randn(1024, len(phone))
|
||||
bert = torch.zeros(1024, len(phone))
|
||||
ja_bert = torch.zeros(1024, len(phone))
|
||||
en_bert = bert_ori
|
||||
phone = torch.LongTensor(phone)
|
||||
tone = torch.LongTensor(tone)
|
||||
@@ -214,6 +226,7 @@ class TextAudioSpeakerCollate:
|
||||
bert_padded = torch.FloatTensor(len(batch), 1024, max_text_len)
|
||||
ja_bert_padded = torch.FloatTensor(len(batch), 1024, max_text_len)
|
||||
en_bert_padded = torch.FloatTensor(len(batch), 1024, max_text_len)
|
||||
style_vec = torch.FloatTensor(len(batch), 256)
|
||||
|
||||
spec_padded = torch.FloatTensor(len(batch), batch[0][1].size(0), max_spec_len)
|
||||
wav_padded = torch.FloatTensor(len(batch), 1, max_wav_len)
|
||||
@@ -225,6 +238,7 @@ class TextAudioSpeakerCollate:
|
||||
bert_padded.zero_()
|
||||
ja_bert_padded.zero_()
|
||||
en_bert_padded.zero_()
|
||||
style_vec.zero_()
|
||||
|
||||
for i in range(len(ids_sorted_decreasing)):
|
||||
row = batch[ids_sorted_decreasing[i]]
|
||||
@@ -258,6 +272,8 @@ class TextAudioSpeakerCollate:
|
||||
en_bert = row[8]
|
||||
en_bert_padded[i, :, : en_bert.size(1)] = en_bert
|
||||
|
||||
style_vec[i, :] = row[9]
|
||||
|
||||
return (
|
||||
text_padded,
|
||||
text_lengths,
|
||||
@@ -271,6 +287,7 @@ class TextAudioSpeakerCollate:
|
||||
bert_padded,
|
||||
ja_bert_padded,
|
||||
en_bert_padded,
|
||||
style_vec,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1,75 +1,40 @@
|
||||
# 全局配置
|
||||
# 对于希望在同一时间使用多个配置文件的情况，例如两个GPU同时跑两个训练集：通过环境变量指定配置文件，不指定则默认为./config.yml
|
||||
# Global configuration file for Bert-VITS2
|
||||
|
||||
# 拟提供通用路径配置，统一存放数据，避免数据放得很乱
|
||||
# 每个数据集与其对应的模型存放至统一路径下，后续所有的路径配置均为相对于datasetPath的路径
|
||||
# 不填或者填空则路径为相对于项目根目录的路径
|
||||
dataset_path: "Data/"
|
||||
model_name: "model_name"
|
||||
|
||||
# 模型镜像源，默认huggingface，使用openi镜像源需指定openi_token
|
||||
mirror: ""
|
||||
openi_token: "" # openi token
|
||||
out_dir: "model_assets"
|
||||
|
||||
# If you want to use a specific dataset path, uncomment the following line.
|
||||
# Otherwise, the dataset path is `Data/{model_name}`.
|
||||
|
||||
# dataset_path: "your/dataset/path"
|
||||
|
||||
# resample 音频重采样配置
|
||||
# 注意， “:” 后需要加空格
|
||||
resample:
|
||||
# 目标重采样率
|
||||
sampling_rate: 44100
|
||||
# 音频文件输入路径，重采样会将该路径下所有.wav音频文件重采样
|
||||
# 请填入相对于datasetPath的相对路径
|
||||
in_dir: "audios/raw" # 相对于根目录的路径为 /datasetPath/in_dir
|
||||
# 音频文件重采样后输出路径
|
||||
in_dir: "audios/raw"
|
||||
out_dir: "audios/wavs"
|
||||
|
||||
|
||||
# preprocess_text 数据集预处理相关配置
|
||||
# 注意， “:” 后需要加空格
|
||||
preprocess_text:
|
||||
# 原始文本文件路径，文本格式应为{wav_path}|{speaker_name}|{language}|{text}。
|
||||
transcription_path: "filelists/你的数据集文本.list"
|
||||
# 数据清洗后文本路径，可以不填。不填则将在原始文本目录生成
|
||||
transcription_path: "filelists/esd.list"
|
||||
cleaned_path: ""
|
||||
# 训练集路径
|
||||
train_path: "filelists/train.list"
|
||||
# 验证集路径
|
||||
val_path: "filelists/val.list"
|
||||
# 配置文件路径
|
||||
config_path: "config.json"
|
||||
# 每个语言的验证集条数
|
||||
val_per_lang: 4
|
||||
# 验证集最大条数，多于的会被截断并放到训练集中
|
||||
max_val_total: 12
|
||||
# 是否进行数据清洗
|
||||
clean: true
|
||||
|
||||
|
||||
# bert_gen 相关配置
|
||||
# 注意， “:” 后需要加空格
|
||||
bert_gen:
|
||||
# 训练数据集配置文件路径
|
||||
config_path: "config.json"
|
||||
# 并行数
|
||||
num_processes: 4
|
||||
# 使用设备：可选项 "cuda" 显卡推理，"cpu" cpu推理
|
||||
# 该选项同时决定了get_bert_feature的默认设备
|
||||
device: "cuda"
|
||||
# 使用多卡推理
|
||||
use_multi_device: false
|
||||
|
||||
# emo_gen 相关配置
|
||||
# 注意， “:” 后需要加空格
|
||||
emo_gen:
|
||||
# 训练数据集配置文件路径
|
||||
style_gen:
|
||||
config_path: "config.json"
|
||||
# 并行数
|
||||
num_processes: 4
|
||||
# 使用设备：可选项 "cuda" 显卡推理，"cpu" cpu推理
|
||||
device: "cuda"
|
||||
# 使用多卡推理
|
||||
use_multi_device: false
|
||||
|
||||
# train 训练配置
|
||||
# 注意， “:” 后需要加空格
|
||||
train_ms:
|
||||
env:
|
||||
MASTER_ADDR: "localhost"
|
||||
@@ -77,27 +42,12 @@ train_ms:
|
||||
WORLD_SIZE: 1
|
||||
LOCAL_RANK: 0
|
||||
RANK: 0
|
||||
# 可以填写任意名的环境变量
|
||||
# THE_ENV_VAR_YOU_NEED_TO_USE: "1234567"
|
||||
# 底模设置
|
||||
base:
|
||||
use_base_model: false
|
||||
repo_id: "Stardust_minus/Bert-VITS2"
|
||||
model_image: "Bert-VITS2_2.3底模" # openi网页的模型名
|
||||
# 训练模型存储目录：与旧版本的区别，原先数据集是存放在logs/model_name下的，现在改为统一存放在Data/你的数据集/models下
|
||||
model: "models"
|
||||
# 配置文件路径
|
||||
config_path: "config.json"
|
||||
# 训练使用的worker，不建议超过CPU核心数
|
||||
num_workers: 16
|
||||
# 关闭此项可以节约接近50%的磁盘空间，但是可能导致实际训练速度变慢和更高的CPU使用率。
|
||||
spec_cache: True
|
||||
# 保存的检查点数量，多于此数目的权重会被删除来节省空间。
|
||||
keep_ckpts: 8
|
||||
keep_ckpts: 1 # Set this to 0 to keep all checkpoints
|
||||
|
||||
|
||||
# webui webui配置
|
||||
# 注意， “:” 后需要加空格
|
||||
webui:
|
||||
# 推理设备
|
||||
device: "cuda"
|
||||
@@ -114,64 +64,18 @@ webui:
|
||||
# 语种识别库，可选langid, fastlid
|
||||
language_identification_library: "langid"
|
||||
|
||||
|
||||
# server-fastapi配置
|
||||
# 注意， “:” 后需要加空格
|
||||
# 注意，本配置下的所有配置均为相对于根目录的路径
|
||||
# server_fastapi's config
|
||||
# TODO: `server_fastapi.py` is not implemented yet for this version
|
||||
server:
|
||||
# 端口号
|
||||
port: 5000
|
||||
# 模型默认使用设备：但是当前并没有实现这个配置。
|
||||
device: "cuda"
|
||||
# 需要加载的所有模型的配置，可以填多个模型，也可以不填模型，等网页成功后手动加载模型
|
||||
# 不加载模型的配置格式：删除默认给的两个模型配置，给models赋值 [ ]，也就是空列表。参考模型2的speakers 即 models: [ ]
|
||||
# 注意，所有模型都必须正确配置model与config的路径，空路径会导致加载错误。
|
||||
# 也可以不填模型，等网页加载成功后手动填写models。
|
||||
models:
|
||||
- # 模型的路径
|
||||
model: ""
|
||||
# 模型config.json的路径
|
||||
- model: ""
|
||||
config: ""
|
||||
# 模型使用设备，若填写则会覆盖默认配置
|
||||
device: "cuda"
|
||||
# 模型默认使用的语言
|
||||
language: "ZH"
|
||||
# 模型人物默认参数
|
||||
# 不必填写所有人物，不填的使用默认值
|
||||
# 暂时不用填写，当前尚未实现按人区分配置
|
||||
speakers:
|
||||
- speaker: "科比"
|
||||
sdp_ratio: 0.2
|
||||
noise_scale: 0.6
|
||||
noise_scale_w: 0.8
|
||||
length_scale: 1
|
||||
- speaker: "五条悟"
|
||||
sdp_ratio: 0.3
|
||||
noise_scale: 0.7
|
||||
noise_scale_w: 0.8
|
||||
length_scale: 0.5
|
||||
- speaker: "安倍晋三"
|
||||
sdp_ratio: 0.2
|
||||
noise_scale: 0.6
|
||||
noise_scale_w: 0.8
|
||||
length_scale: 1.2
|
||||
- # 模型的路径
|
||||
model: ""
|
||||
# 模型config.json的路径
|
||||
- model: ""
|
||||
config: ""
|
||||
# 模型使用设备，若填写则会覆盖默认配置
|
||||
device: "cpu"
|
||||
# 模型默认使用的语言
|
||||
language: "JP"
|
||||
# 模型人物默认参数
|
||||
# 不必填写所有人物，不填的使用默认值
|
||||
speakers: [ ] # 也可以不填
|
||||
|
||||
# 百度翻译开放平台 api配置
|
||||
# api接入文档 https://api.fanyi.baidu.com/doc/21
|
||||
# 请不要在github等网站公开分享你的app id 与 key
|
||||
translate:
|
||||
# 你的APPID
|
||||
"app_key": ""
|
||||
# 你的密钥
|
||||
"secret_key": ""
|
||||
speakers: []
|
||||
|
||||
1
docs/tutorial.md
Normal file
1
docs/tutorial.md
Normal file
@@ -0,0 +1 @@
|
||||
To be written.
|
||||
155
emo_gen.py
155
emo_gen.py
@@ -1,155 +0,0 @@
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import librosa
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.utils.data import Dataset
|
||||
from torch.utils.data import DataLoader, Dataset
|
||||
from tqdm import tqdm
|
||||
from transformers import Wav2Vec2Processor
|
||||
from transformers.models.wav2vec2.modeling_wav2vec2 import (
|
||||
Wav2Vec2Model,
|
||||
Wav2Vec2PreTrainedModel,
|
||||
)
|
||||
|
||||
import utils
|
||||
from config import config
|
||||
|
||||
|
||||
class RegressionHead(nn.Module):
|
||||
r"""Classification head."""
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
|
||||
self.dense = nn.Linear(config.hidden_size, config.hidden_size)
|
||||
self.dropout = nn.Dropout(config.final_dropout)
|
||||
self.out_proj = nn.Linear(config.hidden_size, config.num_labels)
|
||||
|
||||
def forward(self, features, **kwargs):
|
||||
x = features
|
||||
x = self.dropout(x)
|
||||
x = self.dense(x)
|
||||
x = torch.tanh(x)
|
||||
x = self.dropout(x)
|
||||
x = self.out_proj(x)
|
||||
|
||||
return x
|
||||
|
||||
|
||||
class EmotionModel(Wav2Vec2PreTrainedModel):
|
||||
r"""Speech emotion classifier."""
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
|
||||
self.config = config
|
||||
self.wav2vec2 = Wav2Vec2Model(config)
|
||||
self.classifier = RegressionHead(config)
|
||||
self.init_weights()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_values,
|
||||
):
|
||||
outputs = self.wav2vec2(input_values)
|
||||
hidden_states = outputs[0]
|
||||
hidden_states = torch.mean(hidden_states, dim=1)
|
||||
logits = self.classifier(hidden_states)
|
||||
|
||||
return hidden_states, logits
|
||||
|
||||
|
||||
class AudioDataset(Dataset):
|
||||
def __init__(self, list_of_wav_files, sr, processor):
|
||||
self.list_of_wav_files = list_of_wav_files
|
||||
self.processor = processor
|
||||
self.sr = sr
|
||||
|
||||
def __len__(self):
|
||||
return len(self.list_of_wav_files)
|
||||
|
||||
def __getitem__(self, idx):
|
||||
wav_file = self.list_of_wav_files[idx]
|
||||
audio_data, _ = librosa.load(wav_file, sr=self.sr)
|
||||
processed_data = self.processor(audio_data, sampling_rate=self.sr)[
|
||||
"input_values"
|
||||
][0]
|
||||
return torch.from_numpy(processed_data)
|
||||
|
||||
|
||||
def process_func(
|
||||
x: np.ndarray,
|
||||
sampling_rate: int,
|
||||
model: EmotionModel,
|
||||
processor: Wav2Vec2Processor,
|
||||
device: str,
|
||||
embeddings: bool = False,
|
||||
) -> np.ndarray:
|
||||
r"""Predict emotions or extract embeddings from raw audio signal."""
|
||||
model = model.to(device)
|
||||
y = processor(x, sampling_rate=sampling_rate)
|
||||
y = y["input_values"][0]
|
||||
y = torch.from_numpy(y).unsqueeze(0).to(device)
|
||||
|
||||
# run through model
|
||||
with torch.no_grad():
|
||||
y = model(y)[0 if embeddings else 1]
|
||||
|
||||
# convert to numpy
|
||||
y = y.detach().cpu().numpy()
|
||||
|
||||
return y
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"-c", "--config", type=str, default=config.bert_gen_config.config_path
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num_processes", type=int, default=config.bert_gen_config.num_processes
|
||||
)
|
||||
args, _ = parser.parse_known_args()
|
||||
config_path = args.config
|
||||
hps = utils.get_hparams_from_file(config_path)
|
||||
|
||||
device = config.bert_gen_config.device
|
||||
|
||||
model_name = "./emotional/wav2vec2-large-robust-12-ft-emotion-msp-dim"
|
||||
REPO_ID = "audeering/wav2vec2-large-robust-12-ft-emotion-msp-dim"
|
||||
if not Path(model_name).joinpath("pytorch_model.bin").exists():
|
||||
utils.download_emo_models(config.mirror, REPO_ID, model_name)
|
||||
|
||||
processor = Wav2Vec2Processor.from_pretrained(model_name)
|
||||
model = EmotionModel.from_pretrained(model_name).to(device)
|
||||
|
||||
lines = []
|
||||
with open(hps.data.training_files, encoding="utf-8") as f:
|
||||
lines.extend(f.readlines())
|
||||
|
||||
with open(hps.data.validation_files, encoding="utf-8") as f:
|
||||
lines.extend(f.readlines())
|
||||
|
||||
wavnames = [line.split("|")[0] for line in lines]
|
||||
dataset = AudioDataset(wavnames, 16000, processor)
|
||||
data_loader = DataLoader(
|
||||
dataset,
|
||||
batch_size=1,
|
||||
shuffle=False,
|
||||
num_workers=min(args.num_processes, os.cpu_count() - 1),
|
||||
)
|
||||
|
||||
with torch.no_grad():
|
||||
for i, data in tqdm(enumerate(data_loader), total=len(data_loader)):
|
||||
wavname = wavnames[i]
|
||||
emo_path = wavname.replace(".wav", ".emo.npy")
|
||||
if os.path.exists(emo_path):
|
||||
continue
|
||||
emb = model(data.to(device))[0].detach().cpu().numpy()
|
||||
np.save(emo_path, emb)
|
||||
|
||||
print("Emo vec 生成完毕!")
|
||||
34
emotional/clap-htsat-fused/.gitattributes
vendored
34
emotional/clap-htsat-fused/.gitattributes
vendored
@@ -1,34 +0,0 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
@@ -1,107 +0,0 @@
|
||||
---
|
||||
license: apache-2.0
|
||||
---
|
||||
# Model card for CLAP
|
||||
|
||||
Model card for CLAP: Contrastive Language-Audio Pretraining
|
||||
|
||||
|
||||
|
||||
|
||||
# Table of Contents
|
||||
|
||||
0. [TL;DR](#TL;DR)
|
||||
1. [Model Details](#model-details)
|
||||
2. [Usage](#usage)
|
||||
3. [Uses](#uses)
|
||||
4. [Citation](#citation)
|
||||
|
||||
# TL;DR
|
||||
|
||||
The abstract of the paper states that:
|
||||
|
||||
> Contrastive learning has shown remarkable success in the field of multimodal representation learning. In this paper, we propose a pipeline of contrastive language-audio pretraining to develop an audio representation by combining audio data with natural language descriptions. To accomplish this target, we first release LAION-Audio-630K, a large collection of 633,526 audio-text pairs from different data sources. Second, we construct a contrastive language-audio pretraining model by considering different audio encoders and text encoders. We incorporate the feature fusion mechanism and keyword-to-caption augmentation into the model design to further enable the model to process audio inputs of variable lengths and enhance the performance. Third, we perform comprehensive experiments to evaluate our model across three tasks: text-to-audio retrieval, zero-shot audio classification, and supervised audio classification. The results demonstrate that our model achieves superior performance in text-to-audio retrieval task. In audio classification tasks, the model achieves state-of-the-art performance in the zero-shot setting and is able to obtain performance comparable to models' results in the non-zero-shot setting. LAION-Audio-630K and the proposed model are both available to the public.
|
||||
|
||||
|
||||
# Usage
|
||||
|
||||
You can use this model for zero shot audio classification or extracting audio and/or textual features.
|
||||
|
||||
# Uses
|
||||
|
||||
## Perform zero-shot audio classification
|
||||
|
||||
### Using `pipeline`
|
||||
|
||||
```python
|
||||
from datasets import load_dataset
|
||||
from transformers import pipeline
|
||||
|
||||
dataset = load_dataset("ashraq/esc50")
|
||||
audio = dataset["train"]["audio"][-1]["array"]
|
||||
|
||||
audio_classifier = pipeline(task="zero-shot-audio-classification", model="laion/clap-htsat-fused")
|
||||
output = audio_classifier(audio, candidate_labels=["Sound of a dog", "Sound of vaccum cleaner"])
|
||||
print(output)
|
||||
>>> [{"score": 0.999, "label": "Sound of a dog"}, {"score": 0.001, "label": "Sound of vaccum cleaner"}]
|
||||
```
|
||||
|
||||
## Run the model:
|
||||
|
||||
You can also get the audio and text embeddings using `ClapModel`
|
||||
|
||||
### Run the model on CPU:
|
||||
|
||||
```python
|
||||
from datasets import load_dataset
|
||||
from transformers import ClapModel, ClapProcessor
|
||||
|
||||
librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
|
||||
audio_sample = librispeech_dummy[0]
|
||||
|
||||
model = ClapModel.from_pretrained("laion/clap-htsat-fused")
|
||||
processor = ClapProcessor.from_pretrained("laion/clap-htsat-fused")
|
||||
|
||||
inputs = processor(audios=audio_sample["audio"]["array"], return_tensors="pt")
|
||||
audio_embed = model.get_audio_features(**inputs)
|
||||
```
|
||||
|
||||
### Run the model on GPU:
|
||||
|
||||
```python
|
||||
from datasets import load_dataset
|
||||
from transformers import ClapModel, ClapProcessor
|
||||
|
||||
librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
|
||||
audio_sample = librispeech_dummy[0]
|
||||
|
||||
model = ClapModel.from_pretrained("laion/clap-htsat-fused").to(0)
|
||||
processor = ClapProcessor.from_pretrained("laion/clap-htsat-fused")
|
||||
|
||||
inputs = processor(audios=audio_sample["audio"]["array"], return_tensors="pt").to(0)
|
||||
audio_embed = model.get_audio_features(**inputs)
|
||||
```
|
||||
|
||||
|
||||
# Citation
|
||||
|
||||
If you are using this model for your work, please consider citing the original paper:
|
||||
```
|
||||
@misc{https://doi.org/10.48550/arxiv.2211.06687,
|
||||
doi = {10.48550/ARXIV.2211.06687},
|
||||
|
||||
url = {https://arxiv.org/abs/2211.06687},
|
||||
|
||||
author = {Wu, Yusong and Chen, Ke and Zhang, Tianyu and Hui, Yuchen and Berg-Kirkpatrick, Taylor and Dubnov, Shlomo},
|
||||
|
||||
keywords = {Sound (cs.SD), Audio and Speech Processing (eess.AS), FOS: Computer and information sciences, FOS: Computer and information sciences, FOS: Electrical engineering, electronic engineering, information engineering, FOS: Electrical engineering, electronic engineering, information engineering},
|
||||
|
||||
title = {Large-scale Contrastive Language-Audio Pretraining with Feature Fusion and Keyword-to-Caption Augmentation},
|
||||
|
||||
publisher = {arXiv},
|
||||
|
||||
year = {2022},
|
||||
|
||||
copyright = {Creative Commons Attribution 4.0 International}
|
||||
}
|
||||
```
|
||||
@@ -1,207 +0,0 @@
|
||||
{
|
||||
"_commit_hash": null,
|
||||
"architectures": [
|
||||
"ClapModel"
|
||||
],
|
||||
"audio_config": {
|
||||
"_name_or_path": "",
|
||||
"add_cross_attention": false,
|
||||
"aff_block_r": 4,
|
||||
"architectures": null,
|
||||
"attention_probs_dropout_prob": 0.0,
|
||||
"bad_words_ids": null,
|
||||
"begin_suppress_tokens": null,
|
||||
"bos_token_id": null,
|
||||
"chunk_size_feed_forward": 0,
|
||||
"cross_attention_hidden_size": null,
|
||||
"decoder_start_token_id": null,
|
||||
"depths": [
|
||||
2,
|
||||
2,
|
||||
6,
|
||||
2
|
||||
],
|
||||
"diversity_penalty": 0.0,
|
||||
"do_sample": false,
|
||||
"drop_path_rate": 0.0,
|
||||
"early_stopping": false,
|
||||
"enable_fusion": true,
|
||||
"enable_patch_fusion": true,
|
||||
"enable_patch_layer_norm": true,
|
||||
"encoder_no_repeat_ngram_size": 0,
|
||||
"eos_token_id": null,
|
||||
"exponential_decay_length_penalty": null,
|
||||
"finetuning_task": null,
|
||||
"flatten_patch_embeds": true,
|
||||
"forced_bos_token_id": null,
|
||||
"forced_eos_token_id": null,
|
||||
"fusion_num_hidden_layers": 2,
|
||||
"fusion_type": null,
|
||||
"hidden_act": "gelu",
|
||||
"hidden_dropout_prob": 0.1,
|
||||
"hidden_size": 768,
|
||||
"id2label": {
|
||||
"0": "LABEL_0",
|
||||
"1": "LABEL_1"
|
||||
},
|
||||
"initializer_factor": 1.0,
|
||||
"is_decoder": false,
|
||||
"is_encoder_decoder": false,
|
||||
"label2id": {
|
||||
"LABEL_0": 0,
|
||||
"LABEL_1": 1
|
||||
},
|
||||
"layer_norm_eps": 1e-05,
|
||||
"length_penalty": 1.0,
|
||||
"max_length": 20,
|
||||
"min_length": 0,
|
||||
"mlp_ratio": 4.0,
|
||||
"model_type": "clap_audio_model",
|
||||
"no_repeat_ngram_size": 0,
|
||||
"num_attention_heads": [
|
||||
4,
|
||||
8,
|
||||
16,
|
||||
32
|
||||
],
|
||||
"num_beam_groups": 1,
|
||||
"num_beams": 1,
|
||||
"num_classes": 527,
|
||||
"num_hidden_layers": 4,
|
||||
"num_mel_bins": 64,
|
||||
"num_return_sequences": 1,
|
||||
"output_attentions": false,
|
||||
"output_hidden_states": false,
|
||||
"output_scores": false,
|
||||
"pad_token_id": null,
|
||||
"patch_embed_input_channels": 1,
|
||||
"patch_embeds_hidden_size": 96,
|
||||
"patch_size": 4,
|
||||
"patch_stride": [
|
||||
4,
|
||||
4
|
||||
],
|
||||
"prefix": null,
|
||||
"problem_type": null,
|
||||
"projection_dim": 512,
|
||||
"projection_hidden_act": "relu",
|
||||
"projection_hidden_size": 768,
|
||||
"pruned_heads": {},
|
||||
"qkv_bias": true,
|
||||
"remove_invalid_values": false,
|
||||
"repetition_penalty": 1.0,
|
||||
"return_dict": true,
|
||||
"return_dict_in_generate": false,
|
||||
"sep_token_id": null,
|
||||
"spec_size": 256,
|
||||
"suppress_tokens": null,
|
||||
"task_specific_params": null,
|
||||
"temperature": 1.0,
|
||||
"tf_legacy_loss": false,
|
||||
"tie_encoder_decoder": false,
|
||||
"tie_word_embeddings": true,
|
||||
"tokenizer_class": null,
|
||||
"top_k": 50,
|
||||
"top_p": 1.0,
|
||||
"torch_dtype": null,
|
||||
"torchscript": false,
|
||||
"transformers_version": "4.27.0.dev0",
|
||||
"typical_p": 1.0,
|
||||
"use_bfloat16": false,
|
||||
"window_size": 8
|
||||
},
|
||||
"hidden_size": 768,
|
||||
"initializer_factor": 1.0,
|
||||
"logit_scale_init_value": 14.285714285714285,
|
||||
"model_type": "clap",
|
||||
"num_hidden_layers": 16,
|
||||
"projection_dim": 512,
|
||||
"projection_hidden_act": "relu",
|
||||
"text_config": {
|
||||
"_name_or_path": "",
|
||||
"add_cross_attention": false,
|
||||
"architectures": null,
|
||||
"attention_probs_dropout_prob": 0.1,
|
||||
"bad_words_ids": null,
|
||||
"begin_suppress_tokens": null,
|
||||
"bos_token_id": 0,
|
||||
"chunk_size_feed_forward": 0,
|
||||
"classifier_dropout": null,
|
||||
"cross_attention_hidden_size": null,
|
||||
"decoder_start_token_id": null,
|
||||
"diversity_penalty": 0.0,
|
||||
"do_sample": false,
|
||||
"early_stopping": false,
|
||||
"encoder_no_repeat_ngram_size": 0,
|
||||
"eos_token_id": 2,
|
||||
"exponential_decay_length_penalty": null,
|
||||
"finetuning_task": null,
|
||||
"forced_bos_token_id": null,
|
||||
"forced_eos_token_id": null,
|
||||
"fusion_hidden_size": 768,
|
||||
"fusion_num_hidden_layers": 2,
|
||||
"hidden_act": "gelu",
|
||||
"hidden_dropout_prob": 0.1,
|
||||
"hidden_size": 768,
|
||||
"id2label": {
|
||||
"0": "LABEL_0",
|
||||
"1": "LABEL_1"
|
||||
},
|
||||
"initializer_factor": 1.0,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 3072,
|
||||
"is_decoder": false,
|
||||
"is_encoder_decoder": false,
|
||||
"label2id": {
|
||||
"LABEL_0": 0,
|
||||
"LABEL_1": 1
|
||||
},
|
||||
"layer_norm_eps": 1e-12,
|
||||
"length_penalty": 1.0,
|
||||
"max_length": 20,
|
||||
"max_position_embeddings": 514,
|
||||
"min_length": 0,
|
||||
"model_type": "clap_text_model",
|
||||
"no_repeat_ngram_size": 0,
|
||||
"num_attention_heads": 12,
|
||||
"num_beam_groups": 1,
|
||||
"num_beams": 1,
|
||||
"num_hidden_layers": 12,
|
||||
"num_return_sequences": 1,
|
||||
"output_attentions": false,
|
||||
"output_hidden_states": false,
|
||||
"output_scores": false,
|
||||
"pad_token_id": 1,
|
||||
"position_embedding_type": "absolute",
|
||||
"prefix": null,
|
||||
"problem_type": null,
|
||||
"projection_dim": 512,
|
||||
"projection_hidden_act": "relu",
|
||||
"projection_hidden_size": 768,
|
||||
"pruned_heads": {},
|
||||
"remove_invalid_values": false,
|
||||
"repetition_penalty": 1.0,
|
||||
"return_dict": true,
|
||||
"return_dict_in_generate": false,
|
||||
"sep_token_id": null,
|
||||
"suppress_tokens": null,
|
||||
"task_specific_params": null,
|
||||
"temperature": 1.0,
|
||||
"tf_legacy_loss": false,
|
||||
"tie_encoder_decoder": false,
|
||||
"tie_word_embeddings": true,
|
||||
"tokenizer_class": null,
|
||||
"top_k": 50,
|
||||
"top_p": 1.0,
|
||||
"torch_dtype": null,
|
||||
"torchscript": false,
|
||||
"transformers_version": "4.27.0.dev0",
|
||||
"type_vocab_size": 1,
|
||||
"typical_p": 1.0,
|
||||
"use_bfloat16": false,
|
||||
"use_cache": true,
|
||||
"vocab_size": 50265
|
||||
},
|
||||
"torch_dtype": "float32",
|
||||
"transformers_version": null
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,22 +0,0 @@
|
||||
{
|
||||
"chunk_length_s": 10,
|
||||
"feature_extractor_type": "ClapFeatureExtractor",
|
||||
"feature_size": 64,
|
||||
"fft_window_size": 1024,
|
||||
"frequency_max": 14000,
|
||||
"frequency_min": 50,
|
||||
"hop_length": 480,
|
||||
"max_length_s": 10,
|
||||
"n_fft": 1024,
|
||||
"nb_frequency_bins": 513,
|
||||
"nb_max_frames": 1000,
|
||||
"nb_max_samples": 480000,
|
||||
"padding": "repeatpad",
|
||||
"padding_side": "right",
|
||||
"padding_value": 0.0,
|
||||
"processor_class": "ClapProcessor",
|
||||
"return_attention_mask": false,
|
||||
"sampling_rate": 48000,
|
||||
"top_db": null,
|
||||
"truncation": "fusion"
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"bos_token": "<s>",
|
||||
"cls_token": "<s>",
|
||||
"eos_token": "</s>",
|
||||
"mask_token": {
|
||||
"content": "<mask>",
|
||||
"lstrip": true,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": "<pad>",
|
||||
"sep_token": "</s>",
|
||||
"unk_token": "<unk>"
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,16 +0,0 @@
|
||||
{
|
||||
"add_prefix_space": false,
|
||||
"bos_token": "<s>",
|
||||
"cls_token": "<s>",
|
||||
"eos_token": "</s>",
|
||||
"errors": "replace",
|
||||
"mask_token": "<mask>",
|
||||
"model_max_length": 512,
|
||||
"pad_token": "<pad>",
|
||||
"processor_class": "ClapProcessor",
|
||||
"sep_token": "</s>",
|
||||
"special_tokens_map_file": null,
|
||||
"tokenizer_class": "RobertaTokenizer",
|
||||
"trim_offsets": true,
|
||||
"unk_token": "<unk>"
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -1,28 +0,0 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zstandard filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
@@ -1,437 +0,0 @@
|
||||
Attribution-NonCommercial-ShareAlike 4.0 International
|
||||
|
||||
=======================================================================
|
||||
|
||||
Creative Commons Corporation ("Creative Commons") is not a law firm and
|
||||
does not provide legal services or legal advice. Distribution of
|
||||
Creative Commons public licenses does not create a lawyer-client or
|
||||
other relationship. Creative Commons makes its licenses and related
|
||||
information available on an "as-is" basis. Creative Commons gives no
|
||||
warranties regarding its licenses, any material licensed under their
|
||||
terms and conditions, or any related information. Creative Commons
|
||||
disclaims all liability for damages resulting from their use to the
|
||||
fullest extent possible.
|
||||
|
||||
Using Creative Commons Public Licenses
|
||||
|
||||
Creative Commons public licenses provide a standard set of terms and
|
||||
conditions that creators and other rights holders may use to share
|
||||
original works of authorship and other material subject to copyright
|
||||
and certain other rights specified in the public license below. The
|
||||
following considerations are for informational purposes only, are not
|
||||
exhaustive, and do not form part of our licenses.
|
||||
|
||||
Considerations for licensors: Our public licenses are
|
||||
intended for use by those authorized to give the public
|
||||
permission to use material in ways otherwise restricted by
|
||||
copyright and certain other rights. Our licenses are
|
||||
irrevocable. Licensors should read and understand the terms
|
||||
and conditions of the license they choose before applying it.
|
||||
Licensors should also secure all rights necessary before
|
||||
applying our licenses so that the public can reuse the
|
||||
material as expected. Licensors should clearly mark any
|
||||
material not subject to the license. This includes other CC-
|
||||
licensed material, or material used under an exception or
|
||||
limitation to copyright. More considerations for licensors:
|
||||
wiki.creativecommons.org/Considerations_for_licensors
|
||||
|
||||
Considerations for the public: By using one of our public
|
||||
licenses, a licensor grants the public permission to use the
|
||||
licensed material under specified terms and conditions. If
|
||||
the licensor's permission is not necessary for any reason--for
|
||||
example, because of any applicable exception or limitation to
|
||||
copyright--then that use is not regulated by the license. Our
|
||||
licenses grant only permissions under copyright and certain
|
||||
other rights that a licensor has authority to grant. Use of
|
||||
the licensed material may still be restricted for other
|
||||
reasons, including because others have copyright or other
|
||||
rights in the material. A licensor may make special requests,
|
||||
such as asking that all changes be marked or described.
|
||||
Although not required by our licenses, you are encouraged to
|
||||
respect those requests where reasonable. More considerations
|
||||
for the public:
|
||||
wiki.creativecommons.org/Considerations_for_licensees
|
||||
|
||||
=======================================================================
|
||||
|
||||
Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International
|
||||
Public License
|
||||
|
||||
By exercising the Licensed Rights (defined below), You accept and agree
|
||||
to be bound by the terms and conditions of this Creative Commons
|
||||
Attribution-NonCommercial-ShareAlike 4.0 International Public License
|
||||
("Public License"). To the extent this Public License may be
|
||||
interpreted as a contract, You are granted the Licensed Rights in
|
||||
consideration of Your acceptance of these terms and conditions, and the
|
||||
Licensor grants You such rights in consideration of benefits the
|
||||
Licensor receives from making the Licensed Material available under
|
||||
these terms and conditions.
|
||||
|
||||
|
||||
Section 1 -- Definitions.
|
||||
|
||||
a. Adapted Material means material subject to Copyright and Similar
|
||||
Rights that is derived from or based upon the Licensed Material
|
||||
and in which the Licensed Material is translated, altered,
|
||||
arranged, transformed, or otherwise modified in a manner requiring
|
||||
permission under the Copyright and Similar Rights held by the
|
||||
Licensor. For purposes of this Public License, where the Licensed
|
||||
Material is a musical work, performance, or sound recording,
|
||||
Adapted Material is always produced where the Licensed Material is
|
||||
synched in timed relation with a moving image.
|
||||
|
||||
b. Adapter's License means the license You apply to Your Copyright
|
||||
and Similar Rights in Your contributions to Adapted Material in
|
||||
accordance with the terms and conditions of this Public License.
|
||||
|
||||
c. BY-NC-SA Compatible License means a license listed at
|
||||
creativecommons.org/compatiblelicenses, approved by Creative
|
||||
Commons as essentially the equivalent of this Public License.
|
||||
|
||||
d. Copyright and Similar Rights means copyright and/or similar rights
|
||||
closely related to copyright including, without limitation,
|
||||
performance, broadcast, sound recording, and Sui Generis Database
|
||||
Rights, without regard to how the rights are labeled or
|
||||
categorized. For purposes of this Public License, the rights
|
||||
specified in Section 2(b)(1)-(2) are not Copyright and Similar
|
||||
Rights.
|
||||
|
||||
e. Effective Technological Measures means those measures that, in the
|
||||
absence of proper authority, may not be circumvented under laws
|
||||
fulfilling obligations under Article 11 of the WIPO Copyright
|
||||
Treaty adopted on December 20, 1996, and/or similar international
|
||||
agreements.
|
||||
|
||||
f. Exceptions and Limitations means fair use, fair dealing, and/or
|
||||
any other exception or limitation to Copyright and Similar Rights
|
||||
that applies to Your use of the Licensed Material.
|
||||
|
||||
g. License Elements means the license attributes listed in the name
|
||||
of a Creative Commons Public License. The License Elements of this
|
||||
Public License are Attribution, NonCommercial, and ShareAlike.
|
||||
|
||||
h. Licensed Material means the artistic or literary work, database,
|
||||
or other material to which the Licensor applied this Public
|
||||
License.
|
||||
|
||||
i. Licensed Rights means the rights granted to You subject to the
|
||||
terms and conditions of this Public License, which are limited to
|
||||
all Copyright and Similar Rights that apply to Your use of the
|
||||
Licensed Material and that the Licensor has authority to license.
|
||||
|
||||
j. Licensor means the individual(s) or entity(ies) granting rights
|
||||
under this Public License.
|
||||
|
||||
k. NonCommercial means not primarily intended for or directed towards
|
||||
commercial advantage or monetary compensation. For purposes of
|
||||
this Public License, the exchange of the Licensed Material for
|
||||
other material subject to Copyright and Similar Rights by digital
|
||||
file-sharing or similar means is NonCommercial provided there is
|
||||
no payment of monetary compensation in connection with the
|
||||
exchange.
|
||||
|
||||
l. Share means to provide material to the public by any means or
|
||||
process that requires permission under the Licensed Rights, such
|
||||
as reproduction, public display, public performance, distribution,
|
||||
dissemination, communication, or importation, and to make material
|
||||
available to the public including in ways that members of the
|
||||
public may access the material from a place and at a time
|
||||
individually chosen by them.
|
||||
|
||||
m. Sui Generis Database Rights means rights other than copyright
|
||||
resulting from Directive 96/9/EC of the European Parliament and of
|
||||
the Council of 11 March 1996 on the legal protection of databases,
|
||||
as amended and/or succeeded, as well as other essentially
|
||||
equivalent rights anywhere in the world.
|
||||
|
||||
n. You means the individual or entity exercising the Licensed Rights
|
||||
under this Public License. Your has a corresponding meaning.
|
||||
|
||||
|
||||
Section 2 -- Scope.
|
||||
|
||||
a. License grant.
|
||||
|
||||
1. Subject to the terms and conditions of this Public License,
|
||||
the Licensor hereby grants You a worldwide, royalty-free,
|
||||
non-sublicensable, non-exclusive, irrevocable license to
|
||||
exercise the Licensed Rights in the Licensed Material to:
|
||||
|
||||
a. reproduce and Share the Licensed Material, in whole or
|
||||
in part, for NonCommercial purposes only; and
|
||||
|
||||
b. produce, reproduce, and Share Adapted Material for
|
||||
NonCommercial purposes only.
|
||||
|
||||
2. Exceptions and Limitations. For the avoidance of doubt, where
|
||||
Exceptions and Limitations apply to Your use, this Public
|
||||
License does not apply, and You do not need to comply with
|
||||
its terms and conditions.
|
||||
|
||||
3. Term. The term of this Public License is specified in Section
|
||||
6(a).
|
||||
|
||||
4. Media and formats; technical modifications allowed. The
|
||||
Licensor authorizes You to exercise the Licensed Rights in
|
||||
all media and formats whether now known or hereafter created,
|
||||
and to make technical modifications necessary to do so. The
|
||||
Licensor waives and/or agrees not to assert any right or
|
||||
authority to forbid You from making technical modifications
|
||||
necessary to exercise the Licensed Rights, including
|
||||
technical modifications necessary to circumvent Effective
|
||||
Technological Measures. For purposes of this Public License,
|
||||
simply making modifications authorized by this Section 2(a)
|
||||
(4) never produces Adapted Material.
|
||||
|
||||
5. Downstream recipients.
|
||||
|
||||
a. Offer from the Licensor -- Licensed Material. Every
|
||||
recipient of the Licensed Material automatically
|
||||
receives an offer from the Licensor to exercise the
|
||||
Licensed Rights under the terms and conditions of this
|
||||
Public License.
|
||||
|
||||
b. Additional offer from the Licensor -- Adapted Material.
|
||||
Every recipient of Adapted Material from You
|
||||
automatically receives an offer from the Licensor to
|
||||
exercise the Licensed Rights in the Adapted Material
|
||||
under the conditions of the Adapter's License You apply.
|
||||
|
||||
c. No downstream restrictions. You may not offer or impose
|
||||
any additional or different terms or conditions on, or
|
||||
apply any Effective Technological Measures to, the
|
||||
Licensed Material if doing so restricts exercise of the
|
||||
Licensed Rights by any recipient of the Licensed
|
||||
Material.
|
||||
|
||||
6. No endorsement. Nothing in this Public License constitutes or
|
||||
may be construed as permission to assert or imply that You
|
||||
are, or that Your use of the Licensed Material is, connected
|
||||
with, or sponsored, endorsed, or granted official status by,
|
||||
the Licensor or others designated to receive attribution as
|
||||
provided in Section 3(a)(1)(A)(i).
|
||||
|
||||
b. Other rights.
|
||||
|
||||
1. Moral rights, such as the right of integrity, are not
|
||||
licensed under this Public License, nor are publicity,
|
||||
privacy, and/or other similar personality rights; however, to
|
||||
the extent possible, the Licensor waives and/or agrees not to
|
||||
assert any such rights held by the Licensor to the limited
|
||||
extent necessary to allow You to exercise the Licensed
|
||||
Rights, but not otherwise.
|
||||
|
||||
2. Patent and trademark rights are not licensed under this
|
||||
Public License.
|
||||
|
||||
3. To the extent possible, the Licensor waives any right to
|
||||
collect royalties from You for the exercise of the Licensed
|
||||
Rights, whether directly or through a collecting society
|
||||
under any voluntary or waivable statutory or compulsory
|
||||
licensing scheme. In all other cases the Licensor expressly
|
||||
reserves any right to collect such royalties, including when
|
||||
the Licensed Material is used other than for NonCommercial
|
||||
purposes.
|
||||
|
||||
|
||||
Section 3 -- License Conditions.
|
||||
|
||||
Your exercise of the Licensed Rights is expressly made subject to the
|
||||
following conditions.
|
||||
|
||||
a. Attribution.
|
||||
|
||||
1. If You Share the Licensed Material (including in modified
|
||||
form), You must:
|
||||
|
||||
a. retain the following if it is supplied by the Licensor
|
||||
with the Licensed Material:
|
||||
|
||||
i. identification of the creator(s) of the Licensed
|
||||
Material and any others designated to receive
|
||||
attribution, in any reasonable manner requested by
|
||||
the Licensor (including by pseudonym if
|
||||
designated);
|
||||
|
||||
ii. a copyright notice;
|
||||
|
||||
iii. a notice that refers to this Public License;
|
||||
|
||||
iv. a notice that refers to the disclaimer of
|
||||
warranties;
|
||||
|
||||
v. a URI or hyperlink to the Licensed Material to the
|
||||
extent reasonably practicable;
|
||||
|
||||
b. indicate if You modified the Licensed Material and
|
||||
retain an indication of any previous modifications; and
|
||||
|
||||
c. indicate the Licensed Material is licensed under this
|
||||
Public License, and include the text of, or the URI or
|
||||
hyperlink to, this Public License.
|
||||
|
||||
2. You may satisfy the conditions in Section 3(a)(1) in any
|
||||
reasonable manner based on the medium, means, and context in
|
||||
which You Share the Licensed Material. For example, it may be
|
||||
reasonable to satisfy the conditions by providing a URI or
|
||||
hyperlink to a resource that includes the required
|
||||
information.
|
||||
3. If requested by the Licensor, You must remove any of the
|
||||
information required by Section 3(a)(1)(A) to the extent
|
||||
reasonably practicable.
|
||||
|
||||
b. ShareAlike.
|
||||
|
||||
In addition to the conditions in Section 3(a), if You Share
|
||||
Adapted Material You produce, the following conditions also apply.
|
||||
|
||||
1. The Adapter's License You apply must be a Creative Commons
|
||||
license with the same License Elements, this version or
|
||||
later, or a BY-NC-SA Compatible License.
|
||||
|
||||
2. You must include the text of, or the URI or hyperlink to, the
|
||||
Adapter's License You apply. You may satisfy this condition
|
||||
in any reasonable manner based on the medium, means, and
|
||||
context in which You Share Adapted Material.
|
||||
|
||||
3. You may not offer or impose any additional or different terms
|
||||
or conditions on, or apply any Effective Technological
|
||||
Measures to, Adapted Material that restrict exercise of the
|
||||
rights granted under the Adapter's License You apply.
|
||||
|
||||
|
||||
Section 4 -- Sui Generis Database Rights.
|
||||
|
||||
Where the Licensed Rights include Sui Generis Database Rights that
|
||||
apply to Your use of the Licensed Material:
|
||||
|
||||
a. for the avoidance of doubt, Section 2(a)(1) grants You the right
|
||||
to extract, reuse, reproduce, and Share all or a substantial
|
||||
portion of the contents of the database for NonCommercial purposes
|
||||
only;
|
||||
|
||||
b. if You include all or a substantial portion of the database
|
||||
contents in a database in which You have Sui Generis Database
|
||||
Rights, then the database in which You have Sui Generis Database
|
||||
Rights (but not its individual contents) is Adapted Material,
|
||||
including for purposes of Section 3(b); and
|
||||
|
||||
c. You must comply with the conditions in Section 3(a) if You Share
|
||||
all or a substantial portion of the contents of the database.
|
||||
|
||||
For the avoidance of doubt, this Section 4 supplements and does not
|
||||
replace Your obligations under this Public License where the Licensed
|
||||
Rights include other Copyright and Similar Rights.
|
||||
|
||||
|
||||
Section 5 -- Disclaimer of Warranties and Limitation of Liability.
|
||||
|
||||
a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
|
||||
EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
|
||||
AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
|
||||
ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
|
||||
IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
|
||||
WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
|
||||
PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
|
||||
ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
|
||||
KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
|
||||
ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
|
||||
|
||||
b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
|
||||
TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
|
||||
NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
|
||||
INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
|
||||
COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
|
||||
USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
|
||||
ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
|
||||
DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
|
||||
IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
|
||||
|
||||
c. The disclaimer of warranties and limitation of liability provided
|
||||
above shall be interpreted in a manner that, to the extent
|
||||
possible, most closely approximates an absolute disclaimer and
|
||||
waiver of all liability.
|
||||
|
||||
|
||||
Section 6 -- Term and Termination.
|
||||
|
||||
a. This Public License applies for the term of the Copyright and
|
||||
Similar Rights licensed here. However, if You fail to comply with
|
||||
this Public License, then Your rights under this Public License
|
||||
terminate automatically.
|
||||
|
||||
b. Where Your right to use the Licensed Material has terminated under
|
||||
Section 6(a), it reinstates:
|
||||
|
||||
1. automatically as of the date the violation is cured, provided
|
||||
it is cured within 30 days of Your discovery of the
|
||||
violation; or
|
||||
|
||||
2. upon express reinstatement by the Licensor.
|
||||
|
||||
For the avoidance of doubt, this Section 6(b) does not affect any
|
||||
right the Licensor may have to seek remedies for Your violations
|
||||
of this Public License.
|
||||
|
||||
c. For the avoidance of doubt, the Licensor may also offer the
|
||||
Licensed Material under separate terms or conditions or stop
|
||||
distributing the Licensed Material at any time; however, doing so
|
||||
will not terminate this Public License.
|
||||
|
||||
d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
|
||||
License.
|
||||
|
||||
|
||||
Section 7 -- Other Terms and Conditions.
|
||||
|
||||
a. The Licensor shall not be bound by any additional or different
|
||||
terms or conditions communicated by You unless expressly agreed.
|
||||
|
||||
b. Any arrangements, understandings, or agreements regarding the
|
||||
Licensed Material not stated herein are separate from and
|
||||
independent of the terms and conditions of this Public License.
|
||||
|
||||
|
||||
Section 8 -- Interpretation.
|
||||
|
||||
a. For the avoidance of doubt, this Public License does not, and
|
||||
shall not be interpreted to, reduce, limit, restrict, or impose
|
||||
conditions on any use of the Licensed Material that could lawfully
|
||||
be made without permission under this Public License.
|
||||
|
||||
b. To the extent possible, if any provision of this Public License is
|
||||
deemed unenforceable, it shall be automatically reformed to the
|
||||
minimum extent necessary to make it enforceable. If the provision
|
||||
cannot be reformed, it shall be severed from this Public License
|
||||
without affecting the enforceability of the remaining terms and
|
||||
conditions.
|
||||
|
||||
c. No term or condition of this Public License will be waived and no
|
||||
failure to comply consented to unless expressly agreed to by the
|
||||
Licensor.
|
||||
|
||||
d. Nothing in this Public License constitutes or may be interpreted
|
||||
as a limitation upon, or waiver of, any privileges and immunities
|
||||
that apply to the Licensor or You, including from the legal
|
||||
processes of any jurisdiction or authority.
|
||||
|
||||
=======================================================================
|
||||
|
||||
Creative Commons is not a party to its public
|
||||
licenses. Notwithstanding, Creative Commons may elect to apply one of
|
||||
its public licenses to material it publishes and in those instances
|
||||
will be considered the “Licensor.” The text of the Creative Commons
|
||||
public licenses is dedicated to the public domain under the CC0 Public
|
||||
Domain Dedication. Except for the limited purpose of indicating that
|
||||
material is shared under a Creative Commons public license or as
|
||||
otherwise permitted by the Creative Commons policies published at
|
||||
creativecommons.org/policies, Creative Commons does not authorize the
|
||||
use of the trademark "Creative Commons" or any other trademark or logo
|
||||
of Creative Commons without its prior written consent including,
|
||||
without limitation, in connection with any unauthorized modifications
|
||||
to any of its public licenses or any other arrangements,
|
||||
understandings, or agreements concerning use of licensed material. For
|
||||
the avoidance of doubt, this paragraph does not form part of the
|
||||
public licenses.
|
||||
|
||||
Creative Commons may be contacted at creativecommons.org.
|
||||
@@ -1,127 +0,0 @@
|
||||
---
|
||||
language: en
|
||||
datasets:
|
||||
- msp-podcast
|
||||
inference: true
|
||||
tags:
|
||||
- speech
|
||||
- audio
|
||||
- wav2vec2
|
||||
- audio-classification
|
||||
- emotion-recognition
|
||||
license: cc-by-nc-sa-4.0
|
||||
pipeline_tag: audio-classification
|
||||
---
|
||||
|
||||
# Model for Dimensional Speech Emotion Recognition based on Wav2vec 2.0
|
||||
|
||||
The model expects a raw audio signal as input and outputs predictions for arousal, dominance and valence in a range of approximately 0...1. In addition, it also provides the pooled states of the last transformer layer. The model was created by fine-tuning [
|
||||
Wav2Vec2-Large-Robust](https://huggingface.co/facebook/wav2vec2-large-robust) on [MSP-Podcast](https://ecs.utdallas.edu/research/researchlabs/msp-lab/MSP-Podcast.html) (v1.7). The model was pruned from 24 to 12 transformer layers before fine-tuning. An [ONNX](https://onnx.ai/") export of the model is available from [doi:10.5281/zenodo.6221127](https://zenodo.org/record/6221127). Further details are given in the associated [paper](https://arxiv.org/abs/2203.07378) and [tutorial](https://github.com/audeering/w2v2-how-to).
|
||||
|
||||
# Usage
|
||||
|
||||
```python
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import Wav2Vec2Processor
|
||||
from transformers.models.wav2vec2.modeling_wav2vec2 import (
|
||||
Wav2Vec2Model,
|
||||
Wav2Vec2PreTrainedModel,
|
||||
)
|
||||
|
||||
|
||||
class RegressionHead(nn.Module):
|
||||
r"""Classification head."""
|
||||
|
||||
def __init__(self, config):
|
||||
|
||||
super().__init__()
|
||||
|
||||
self.dense = nn.Linear(config.hidden_size, config.hidden_size)
|
||||
self.dropout = nn.Dropout(config.final_dropout)
|
||||
self.out_proj = nn.Linear(config.hidden_size, config.num_labels)
|
||||
|
||||
def forward(self, features, **kwargs):
|
||||
|
||||
x = features
|
||||
x = self.dropout(x)
|
||||
x = self.dense(x)
|
||||
x = torch.tanh(x)
|
||||
x = self.dropout(x)
|
||||
x = self.out_proj(x)
|
||||
|
||||
return x
|
||||
|
||||
|
||||
class EmotionModel(Wav2Vec2PreTrainedModel):
|
||||
r"""Speech emotion classifier."""
|
||||
|
||||
def __init__(self, config):
|
||||
|
||||
super().__init__(config)
|
||||
|
||||
self.config = config
|
||||
self.wav2vec2 = Wav2Vec2Model(config)
|
||||
self.classifier = RegressionHead(config)
|
||||
self.init_weights()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_values,
|
||||
):
|
||||
|
||||
outputs = self.wav2vec2(input_values)
|
||||
hidden_states = outputs[0]
|
||||
hidden_states = torch.mean(hidden_states, dim=1)
|
||||
logits = self.classifier(hidden_states)
|
||||
|
||||
return hidden_states, logits
|
||||
|
||||
|
||||
|
||||
# load model from hub
|
||||
device = 'cpu'
|
||||
model_name = 'audeering/wav2vec2-large-robust-12-ft-emotion-msp-dim'
|
||||
processor = Wav2Vec2Processor.from_pretrained(model_name)
|
||||
model = EmotionModel.from_pretrained(model_name)
|
||||
|
||||
# dummy signal
|
||||
sampling_rate = 16000
|
||||
signal = np.zeros((1, sampling_rate), dtype=np.float32)
|
||||
|
||||
|
||||
def process_func(
|
||||
x: np.ndarray,
|
||||
sampling_rate: int,
|
||||
embeddings: bool = False,
|
||||
) -> np.ndarray:
|
||||
r"""Predict emotions or extract embeddings from raw audio signal."""
|
||||
|
||||
# run through processor to normalize signal
|
||||
# always returns a batch, so we just get the first entry
|
||||
# then we put it on the device
|
||||
y = processor(x, sampling_rate=sampling_rate)
|
||||
y = y['input_values'][0]
|
||||
y = y.reshape(1, -1)
|
||||
y = torch.from_numpy(y).to(device)
|
||||
|
||||
# run through model
|
||||
with torch.no_grad():
|
||||
y = model(y)[0 if embeddings else 1]
|
||||
|
||||
# convert to numpy
|
||||
y = y.detach().cpu().numpy()
|
||||
|
||||
return y
|
||||
|
||||
|
||||
print(process_func(signal, sampling_rate))
|
||||
# Arousal dominance valence
|
||||
# [[0.5460754 0.6062266 0.40431657]]
|
||||
|
||||
print(process_func(signal, sampling_rate, embeddings=True))
|
||||
# Pooled hidden states of last transformer layer
|
||||
# [[-0.00752167 0.0065819 -0.00746342 ... 0.00663632 0.00848748
|
||||
# 0.00599211]]
|
||||
```
|
||||
@@ -1,122 +0,0 @@
|
||||
{
|
||||
"_name_or_path": "torch",
|
||||
"activation_dropout": 0.1,
|
||||
"adapter_kernel_size": 3,
|
||||
"adapter_stride": 2,
|
||||
"add_adapter": false,
|
||||
"apply_spec_augment": true,
|
||||
"architectures": [
|
||||
"Wav2Vec2ForSpeechClassification"
|
||||
],
|
||||
"attention_dropout": 0.1,
|
||||
"bos_token_id": 1,
|
||||
"classifier_proj_size": 256,
|
||||
"codevector_dim": 768,
|
||||
"contrastive_logits_temperature": 0.1,
|
||||
"conv_bias": true,
|
||||
"conv_dim": [
|
||||
512,
|
||||
512,
|
||||
512,
|
||||
512,
|
||||
512,
|
||||
512,
|
||||
512
|
||||
],
|
||||
"conv_kernel": [
|
||||
10,
|
||||
3,
|
||||
3,
|
||||
3,
|
||||
3,
|
||||
2,
|
||||
2
|
||||
],
|
||||
"conv_stride": [
|
||||
5,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
2
|
||||
],
|
||||
"ctc_loss_reduction": "sum",
|
||||
"ctc_zero_infinity": false,
|
||||
"diversity_loss_weight": 0.1,
|
||||
"do_stable_layer_norm": true,
|
||||
"eos_token_id": 2,
|
||||
"feat_extract_activation": "gelu",
|
||||
"feat_extract_dropout": 0.0,
|
||||
"feat_extract_norm": "layer",
|
||||
"feat_proj_dropout": 0.1,
|
||||
"feat_quantizer_dropout": 0.0,
|
||||
"final_dropout": 0.1,
|
||||
"finetuning_task": "wav2vec2_reg",
|
||||
"gradient_checkpointing": false,
|
||||
"hidden_act": "gelu",
|
||||
"hidden_dropout": 0.1,
|
||||
"hidden_dropout_prob": 0.1,
|
||||
"hidden_size": 1024,
|
||||
"id2label": {
|
||||
"0": "arousal",
|
||||
"1": "dominance",
|
||||
"2": "valence"
|
||||
},
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 4096,
|
||||
"label2id": {
|
||||
"arousal": 0,
|
||||
"dominance": 1,
|
||||
"valence": 2
|
||||
},
|
||||
"layer_norm_eps": 1e-05,
|
||||
"layerdrop": 0.1,
|
||||
"mask_feature_length": 10,
|
||||
"mask_feature_min_masks": 0,
|
||||
"mask_feature_prob": 0.0,
|
||||
"mask_time_length": 10,
|
||||
"mask_time_min_masks": 2,
|
||||
"mask_time_prob": 0.05,
|
||||
"model_type": "wav2vec2",
|
||||
"num_adapter_layers": 3,
|
||||
"num_attention_heads": 16,
|
||||
"num_codevector_groups": 2,
|
||||
"num_codevectors_per_group": 320,
|
||||
"num_conv_pos_embedding_groups": 16,
|
||||
"num_conv_pos_embeddings": 128,
|
||||
"num_feat_extract_layers": 7,
|
||||
"num_hidden_layers": 12,
|
||||
"num_negatives": 100,
|
||||
"output_hidden_size": 1024,
|
||||
"pad_token_id": 0,
|
||||
"pooling_mode": "mean",
|
||||
"problem_type": "regression",
|
||||
"proj_codevector_dim": 768,
|
||||
"tdnn_dilation": [
|
||||
1,
|
||||
2,
|
||||
3,
|
||||
1,
|
||||
1
|
||||
],
|
||||
"tdnn_dim": [
|
||||
512,
|
||||
512,
|
||||
512,
|
||||
512,
|
||||
1500
|
||||
],
|
||||
"tdnn_kernel": [
|
||||
5,
|
||||
3,
|
||||
3,
|
||||
1,
|
||||
1
|
||||
],
|
||||
"torch_dtype": "float32",
|
||||
"transformers_version": "4.17.0.dev0",
|
||||
"use_weighted_layer_sum": false,
|
||||
"vocab_size": null,
|
||||
"xvector_output_dim": 512
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
{
|
||||
"do_normalize": true,
|
||||
"feature_extractor_type": "Wav2Vec2FeatureExtractor",
|
||||
"feature_size": 1,
|
||||
"padding_side": "right",
|
||||
"padding_value": 0.0,
|
||||
"return_attention_mask": true,
|
||||
"sampling_rate": 16000
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
{}
|
||||
BIN
empty_emo.npy
BIN
empty_emo.npy
Binary file not shown.
@@ -1,14 +0,0 @@
|
||||
from onnx_modules import export_onnx
|
||||
import os
|
||||
|
||||
if __name__ == "__main__":
|
||||
export_path = "BertVits2.2PT"
|
||||
model_path = "model\\G_0.pth"
|
||||
config_path = "model\\config.json"
|
||||
novq = False
|
||||
dev = False
|
||||
if not os.path.exists("onnx"):
|
||||
os.makedirs("onnx")
|
||||
if not os.path.exists(f"onnx/{export_path}"):
|
||||
os.makedirs(f"onnx/{export_path}")
|
||||
export_onnx(export_path, model_path, config_path, novq, dev)
|
||||
@@ -1,3 +0,0 @@
|
||||
Example:
|
||||
{wav_path}|{speaker_name}|{language}|{text}
|
||||
派蒙_1.wav|派蒙|ZH|前面的区域，以后再来探索吧！
|
||||
@@ -1,385 +0,0 @@
|
||||
"""
|
||||
版本管理、兼容推理及模型加载实现。
|
||||
版本说明：
|
||||
1. 版本号与github的release版本号对应，使用哪个release版本训练的模型即对应其版本号
|
||||
2. 请在模型的config.json中显示声明版本号，添加一个字段"version" : "你的版本号"
|
||||
特殊版本说明：
|
||||
1.1.1-fix： 1.1.1版本训练的模型，但是在推理时使用dev的日语修复
|
||||
2.2：当前版本
|
||||
"""
|
||||
import torch
|
||||
import commons
|
||||
from text import cleaned_text_to_sequence
|
||||
from text.cleaner import clean_text
|
||||
import utils
|
||||
import numpy as np
|
||||
|
||||
from models import SynthesizerTrn
|
||||
from text.symbols import symbols
|
||||
|
||||
from oldVersion.V210.models import SynthesizerTrn as V210SynthesizerTrn
|
||||
from oldVersion.V210.text import symbols as V210symbols
|
||||
from oldVersion.V200.models import SynthesizerTrn as V200SynthesizerTrn
|
||||
from oldVersion.V200.text import symbols as V200symbols
|
||||
from oldVersion.V111.models import SynthesizerTrn as V111SynthesizerTrn
|
||||
from oldVersion.V111.text import symbols as V111symbols
|
||||
from oldVersion.V110.models import SynthesizerTrn as V110SynthesizerTrn
|
||||
from oldVersion.V110.text import symbols as V110symbols
|
||||
from oldVersion.V101.models import SynthesizerTrn as V101SynthesizerTrn
|
||||
from oldVersion.V101.text import symbols as V101symbols
|
||||
|
||||
from oldVersion import V111, V110, V101, V200, V210
|
||||
|
||||
# 当前版本信息
|
||||
latest_version = "2.2"
|
||||
|
||||
# 版本兼容
|
||||
SynthesizerTrnMap = {
|
||||
"2.1": V210SynthesizerTrn,
|
||||
"2.0.2-fix": V200SynthesizerTrn,
|
||||
"2.0.1": V200SynthesizerTrn,
|
||||
"2.0": V200SynthesizerTrn,
|
||||
"1.1.1-fix": V111SynthesizerTrn,
|
||||
"1.1.1": V111SynthesizerTrn,
|
||||
"1.1": V110SynthesizerTrn,
|
||||
"1.1.0": V110SynthesizerTrn,
|
||||
"1.0.1": V101SynthesizerTrn,
|
||||
"1.0": V101SynthesizerTrn,
|
||||
"1.0.0": V101SynthesizerTrn,
|
||||
}
|
||||
|
||||
symbolsMap = {
|
||||
"2.1": V210symbols,
|
||||
"2.0.2-fix": V200symbols,
|
||||
"2.0.1": V200symbols,
|
||||
"2.0": V200symbols,
|
||||
"1.1.1-fix": V111symbols,
|
||||
"1.1.1": V111symbols,
|
||||
"1.1": V110symbols,
|
||||
"1.1.0": V110symbols,
|
||||
"1.0.1": V101symbols,
|
||||
"1.0": V101symbols,
|
||||
"1.0.0": V101symbols,
|
||||
}
|
||||
|
||||
|
||||
# def get_emo_(reference_audio, emotion, sid):
|
||||
# emo = (
|
||||
# torch.from_numpy(get_emo(reference_audio))
|
||||
# if reference_audio and emotion == -1
|
||||
# else torch.FloatTensor(
|
||||
# np.load(f"emo_clustering/{sid}/cluster_center_{emotion}.npy")
|
||||
# )
|
||||
# )
|
||||
# return emo
|
||||
|
||||
|
||||
def get_net_g(model_path: str, version: str, device: str, hps):
|
||||
if version != latest_version:
|
||||
net_g = SynthesizerTrnMap[version](
|
||||
len(symbolsMap[version]),
|
||||
hps.data.filter_length // 2 + 1,
|
||||
hps.train.segment_size // hps.data.hop_length,
|
||||
n_speakers=hps.data.n_speakers,
|
||||
**hps.model,
|
||||
).to(device)
|
||||
else:
|
||||
# 当前版本模型 net_g
|
||||
net_g = SynthesizerTrn(
|
||||
len(symbols),
|
||||
hps.data.filter_length // 2 + 1,
|
||||
hps.train.segment_size // hps.data.hop_length,
|
||||
n_speakers=hps.data.n_speakers,
|
||||
**hps.model,
|
||||
).to(device)
|
||||
_ = net_g.eval()
|
||||
_ = utils.load_checkpoint(model_path, net_g, None, skip_optimizer=True)
|
||||
return net_g
|
||||
|
||||
|
||||
def get_text(text, language_str, bert, hps, device):
|
||||
# 在此处实现当前版本的get_text
|
||||
norm_text, phone, tone, word2ph = clean_text(text, language_str)
|
||||
phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
|
||||
|
||||
if hps.data.add_blank:
|
||||
phone = commons.intersperse(phone, 0)
|
||||
tone = commons.intersperse(tone, 0)
|
||||
language = commons.intersperse(language, 0)
|
||||
for i in range(len(word2ph)):
|
||||
word2ph[i] = word2ph[i] * 2
|
||||
word2ph[0] += 1
|
||||
# bert_ori = get_bert(norm_text, word2ph, language_str, device)
|
||||
bert_ori = bert[language_str].get_bert_feature(norm_text, word2ph, device)
|
||||
del word2ph
|
||||
assert bert_ori.shape[-1] == len(phone), phone
|
||||
|
||||
if language_str == "ZH":
|
||||
bert = bert_ori
|
||||
ja_bert = torch.randn(1024, len(phone))
|
||||
en_bert = torch.randn(1024, len(phone))
|
||||
elif language_str == "JP":
|
||||
bert = torch.randn(1024, len(phone))
|
||||
ja_bert = bert_ori
|
||||
en_bert = torch.randn(1024, len(phone))
|
||||
elif language_str == "EN":
|
||||
bert = torch.randn(1024, len(phone))
|
||||
ja_bert = torch.randn(1024, len(phone))
|
||||
en_bert = bert_ori
|
||||
else:
|
||||
raise ValueError("language_str should be ZH, JP or EN")
|
||||
|
||||
assert bert.shape[-1] == len(
|
||||
phone
|
||||
), f"Bert seq len {bert.shape[-1]} != {len(phone)}"
|
||||
|
||||
phone = torch.LongTensor(phone)
|
||||
tone = torch.LongTensor(tone)
|
||||
language = torch.LongTensor(language)
|
||||
return bert, ja_bert, en_bert, phone, tone, language
|
||||
|
||||
|
||||
def infer(
|
||||
text,
|
||||
emotion,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
sid,
|
||||
language,
|
||||
hps,
|
||||
net_g,
|
||||
device,
|
||||
bert=None,
|
||||
clap=None,
|
||||
reference_audio=None,
|
||||
skip_start=False,
|
||||
skip_end=False,
|
||||
):
|
||||
# 2.2版本参数位置变了
|
||||
# 2.1 参数新增 emotion reference_audio skip_start skip_end
|
||||
inferMap_V3 = {
|
||||
"2.1": V210.infer,
|
||||
}
|
||||
# 支持中日英三语版本
|
||||
inferMap_V2 = {
|
||||
"2.0.2-fix": V200.infer,
|
||||
"2.0.1": V200.infer,
|
||||
"2.0": V200.infer,
|
||||
"1.1.1-fix": V111.infer_fix,
|
||||
"1.1.1": V111.infer,
|
||||
"1.1": V110.infer,
|
||||
"1.1.0": V110.infer,
|
||||
}
|
||||
# 仅支持中文版本
|
||||
# 在测试中，并未发现两个版本的模型不能互相通用
|
||||
inferMap_V1 = {
|
||||
"1.0.1": V101.infer,
|
||||
"1.0": V101.infer,
|
||||
"1.0.0": V101.infer,
|
||||
}
|
||||
version = hps.version if hasattr(hps, "version") else latest_version
|
||||
# 非当前版本，根据版本号选择合适的infer
|
||||
if version != latest_version:
|
||||
if version in inferMap_V3.keys():
|
||||
return inferMap_V3[version](
|
||||
text,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
sid,
|
||||
language,
|
||||
hps,
|
||||
net_g,
|
||||
device,
|
||||
reference_audio,
|
||||
emotion,
|
||||
skip_start,
|
||||
skip_end,
|
||||
)
|
||||
if version in inferMap_V2.keys():
|
||||
return inferMap_V2[version](
|
||||
text,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
sid,
|
||||
language,
|
||||
hps,
|
||||
net_g,
|
||||
device,
|
||||
)
|
||||
if version in inferMap_V1.keys():
|
||||
return inferMap_V1[version](
|
||||
text,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
sid,
|
||||
hps,
|
||||
net_g,
|
||||
device,
|
||||
)
|
||||
# 在此处实现当前版本的推理
|
||||
# emo = get_emo_(reference_audio, emotion, sid)
|
||||
if isinstance(reference_audio, np.ndarray):
|
||||
emo = clap.get_clap_audio_feature(reference_audio, device)
|
||||
else:
|
||||
emo = clap.get_clap_text_feature(emotion, device)
|
||||
emo = torch.squeeze(emo, dim=1)
|
||||
|
||||
bert, ja_bert, en_bert, phones, tones, lang_ids = get_text(
|
||||
text, language, bert, hps, device
|
||||
)
|
||||
if skip_start:
|
||||
phones = phones[3:]
|
||||
tones = tones[3:]
|
||||
lang_ids = lang_ids[3:]
|
||||
bert = bert[:, 3:]
|
||||
ja_bert = ja_bert[:, 3:]
|
||||
en_bert = en_bert[:, 3:]
|
||||
if skip_end:
|
||||
phones = phones[:-2]
|
||||
tones = tones[:-2]
|
||||
lang_ids = lang_ids[:-2]
|
||||
bert = bert[:, :-2]
|
||||
ja_bert = ja_bert[:, :-2]
|
||||
en_bert = en_bert[:, :-2]
|
||||
with torch.no_grad():
|
||||
x_tst = phones.to(device).unsqueeze(0)
|
||||
tones = tones.to(device).unsqueeze(0)
|
||||
lang_ids = lang_ids.to(device).unsqueeze(0)
|
||||
bert = bert.to(device).unsqueeze(0)
|
||||
ja_bert = ja_bert.to(device).unsqueeze(0)
|
||||
en_bert = en_bert.to(device).unsqueeze(0)
|
||||
x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
|
||||
emo = emo.to(device).unsqueeze(0)
|
||||
del phones
|
||||
speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
|
||||
audio = (
|
||||
net_g.infer(
|
||||
x_tst,
|
||||
x_tst_lengths,
|
||||
speakers,
|
||||
tones,
|
||||
lang_ids,
|
||||
bert,
|
||||
ja_bert,
|
||||
en_bert,
|
||||
emo,
|
||||
sdp_ratio=sdp_ratio,
|
||||
noise_scale=noise_scale,
|
||||
noise_scale_w=noise_scale_w,
|
||||
length_scale=length_scale,
|
||||
)[0][0, 0]
|
||||
.data.cpu()
|
||||
.float()
|
||||
.numpy()
|
||||
)
|
||||
del x_tst, tones, lang_ids, bert, x_tst_lengths, speakers, ja_bert, en_bert, emo
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
return audio
|
||||
|
||||
|
||||
def infer_multilang(
|
||||
text,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
sid,
|
||||
language,
|
||||
hps,
|
||||
net_g,
|
||||
device,
|
||||
bert=None,
|
||||
clap=None,
|
||||
reference_audio=None,
|
||||
emotion=None,
|
||||
skip_start=False,
|
||||
skip_end=False,
|
||||
):
|
||||
bert, ja_bert, en_bert, phones, tones, lang_ids = [], [], [], [], [], []
|
||||
# emo = get_emo_(reference_audio, emotion, sid)
|
||||
if isinstance(reference_audio, np.ndarray):
|
||||
emo = clap.get_clap_audio_feature(reference_audio, device)
|
||||
else:
|
||||
emo = clap.get_clap_text_feature(emotion, device)
|
||||
emo = torch.squeeze(emo, dim=1)
|
||||
for idx, (txt, lang) in enumerate(zip(text, language)):
|
||||
skip_start = (idx != 0) or (skip_start and idx == 0)
|
||||
skip_end = (idx != len(text) - 1) or (skip_end and idx == len(text) - 1)
|
||||
(
|
||||
temp_bert,
|
||||
temp_ja_bert,
|
||||
temp_en_bert,
|
||||
temp_phones,
|
||||
temp_tones,
|
||||
temp_lang_ids,
|
||||
) = get_text(txt, lang, bert, hps, device)
|
||||
if skip_start:
|
||||
temp_bert = temp_bert[:, 3:]
|
||||
temp_ja_bert = temp_ja_bert[:, 3:]
|
||||
temp_en_bert = temp_en_bert[:, 3:]
|
||||
temp_phones = temp_phones[3:]
|
||||
temp_tones = temp_tones[3:]
|
||||
temp_lang_ids = temp_lang_ids[3:]
|
||||
if skip_end:
|
||||
temp_bert = temp_bert[:, :-2]
|
||||
temp_ja_bert = temp_ja_bert[:, :-2]
|
||||
temp_en_bert = temp_en_bert[:, :-2]
|
||||
temp_phones = temp_phones[:-2]
|
||||
temp_tones = temp_tones[:-2]
|
||||
temp_lang_ids = temp_lang_ids[:-2]
|
||||
bert.append(temp_bert)
|
||||
ja_bert.append(temp_ja_bert)
|
||||
en_bert.append(temp_en_bert)
|
||||
phones.append(temp_phones)
|
||||
tones.append(temp_tones)
|
||||
lang_ids.append(temp_lang_ids)
|
||||
bert = torch.concatenate(bert, dim=1)
|
||||
ja_bert = torch.concatenate(ja_bert, dim=1)
|
||||
en_bert = torch.concatenate(en_bert, dim=1)
|
||||
phones = torch.concatenate(phones, dim=0)
|
||||
tones = torch.concatenate(tones, dim=0)
|
||||
lang_ids = torch.concatenate(lang_ids, dim=0)
|
||||
with torch.no_grad():
|
||||
x_tst = phones.to(device).unsqueeze(0)
|
||||
tones = tones.to(device).unsqueeze(0)
|
||||
lang_ids = lang_ids.to(device).unsqueeze(0)
|
||||
bert = bert.to(device).unsqueeze(0)
|
||||
ja_bert = ja_bert.to(device).unsqueeze(0)
|
||||
en_bert = en_bert.to(device).unsqueeze(0)
|
||||
emo = emo.to(device).unsqueeze(0)
|
||||
x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
|
||||
del phones
|
||||
speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
|
||||
audio = (
|
||||
net_g.infer(
|
||||
x_tst,
|
||||
x_tst_lengths,
|
||||
speakers,
|
||||
tones,
|
||||
lang_ids,
|
||||
bert,
|
||||
ja_bert,
|
||||
en_bert,
|
||||
emo,
|
||||
sdp_ratio=sdp_ratio,
|
||||
noise_scale=noise_scale,
|
||||
noise_scale_w=noise_scale_w,
|
||||
length_scale=length_scale,
|
||||
)[0][0, 0]
|
||||
.data.cpu()
|
||||
.float()
|
||||
.numpy()
|
||||
)
|
||||
del x_tst, tones, lang_ids, bert, x_tst_lengths, speakers, ja_bert, en_bert, emo
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
return audio
|
||||
@@ -1,111 +0,0 @@
|
||||
import sys
|
||||
|
||||
import torch
|
||||
from transformers import (
|
||||
AutoModelForMaskedLM,
|
||||
AutoTokenizer,
|
||||
DebertaV2Model,
|
||||
DebertaV2Tokenizer,
|
||||
ClapModel,
|
||||
ClapProcessor,
|
||||
)
|
||||
|
||||
from config import config
|
||||
from text.japanese import text2sep_kata
|
||||
|
||||
|
||||
class BertFeature:
|
||||
def __init__(self, model_path, language="ZH"):
|
||||
self.model_path = model_path
|
||||
self.language = language
|
||||
self.tokenizer = None
|
||||
self.model = None
|
||||
self.device = None
|
||||
|
||||
self._prepare()
|
||||
|
||||
def _get_device(self, device=config.bert_gen_config.device):
|
||||
if (
|
||||
sys.platform == "darwin"
|
||||
and torch.backends.mps.is_available()
|
||||
and device == "cpu"
|
||||
):
|
||||
device = "mps"
|
||||
if not device:
|
||||
device = "cuda"
|
||||
return device
|
||||
|
||||
def _prepare(self):
|
||||
self.device = self._get_device()
|
||||
|
||||
if self.language == "EN":
|
||||
self.tokenizer = DebertaV2Tokenizer.from_pretrained(self.model_path)
|
||||
self.model = DebertaV2Model.from_pretrained(self.model_path).to(self.device)
|
||||
else:
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(self.model_path)
|
||||
self.model = AutoModelForMaskedLM.from_pretrained(self.model_path).to(
|
||||
self.device
|
||||
)
|
||||
self.model.eval()
|
||||
|
||||
def get_bert_feature(self, text, word2ph):
|
||||
if self.language == "JP":
|
||||
text = "".join(text2sep_kata(text)[0])
|
||||
with torch.no_grad():
|
||||
inputs = self.tokenizer(text, return_tensors="pt")
|
||||
for i in inputs:
|
||||
inputs[i] = inputs[i].to(self.device)
|
||||
res = self.model(**inputs, output_hidden_states=True)
|
||||
res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
|
||||
|
||||
word2phone = word2ph
|
||||
phone_level_feature = []
|
||||
for i in range(len(word2phone)):
|
||||
repeat_feature = res[i].repeat(word2phone[i], 1)
|
||||
phone_level_feature.append(repeat_feature)
|
||||
|
||||
phone_level_feature = torch.cat(phone_level_feature, dim=0)
|
||||
|
||||
return phone_level_feature.T
|
||||
|
||||
|
||||
class ClapFeature:
|
||||
def __init__(self, model_path):
|
||||
self.model_path = model_path
|
||||
self.processor = None
|
||||
self.model = None
|
||||
self.device = None
|
||||
|
||||
self._prepare()
|
||||
|
||||
def _get_device(self, device=config.bert_gen_config.device):
|
||||
if (
|
||||
sys.platform == "darwin"
|
||||
and torch.backends.mps.is_available()
|
||||
and device == "cpu"
|
||||
):
|
||||
device = "mps"
|
||||
if not device:
|
||||
device = "cuda"
|
||||
return device
|
||||
|
||||
def _prepare(self):
|
||||
self.device = self._get_device()
|
||||
|
||||
self.processor = ClapProcessor.from_pretrained(self.model_path)
|
||||
self.model = ClapModel.from_pretrained(self.model_path).to(self.device)
|
||||
self.model.eval()
|
||||
|
||||
def get_clap_audio_feature(self, audio_data):
|
||||
with torch.no_grad():
|
||||
inputs = self.processor(
|
||||
audios=audio_data, return_tensors="pt", sampling_rate=48000
|
||||
).to(self.device)
|
||||
emb = self.model.get_audio_features(**inputs)
|
||||
return emb.T
|
||||
|
||||
def get_clap_text_feature(self, text):
|
||||
with torch.no_grad():
|
||||
inputs = self.processor(text=text, return_tensors="pt").to(self.device)
|
||||
emb = self.model.get_text_features(**inputs)
|
||||
return emb.T
|
||||
@@ -1,556 +0,0 @@
|
||||
# flake8: noqa: E402
|
||||
import os
|
||||
import logging
|
||||
import re_matching
|
||||
from tools.sentence import split_by_language
|
||||
|
||||
logging.getLogger("numba").setLevel(logging.WARNING)
|
||||
logging.getLogger("markdown_it").setLevel(logging.WARNING)
|
||||
logging.getLogger("urllib3").setLevel(logging.WARNING)
|
||||
logging.getLogger("matplotlib").setLevel(logging.WARNING)
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO, format="| %(name)s | %(levelname)s | %(message)s"
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
import torch
|
||||
import utils
|
||||
from infer import infer, latest_version, get_net_g, infer_multilang
|
||||
import gradio as gr
|
||||
import webbrowser
|
||||
import numpy as np
|
||||
from config import config
|
||||
from tools.translate import translate
|
||||
import librosa
|
||||
from infer_utils import BertFeature, ClapFeature
|
||||
|
||||
|
||||
net_g = None
|
||||
|
||||
device = config.webui_config.device
|
||||
if device == "mps":
|
||||
os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
|
||||
|
||||
os.environ["OMP_NUM_THREADS"] = "1"
|
||||
os.environ["MKL_NUM_THREADS"] = "1"
|
||||
|
||||
bert_feature_map = {
|
||||
"ZH": BertFeature(
|
||||
"./bert/chinese-roberta-wwm-ext-large",
|
||||
language="ZH",
|
||||
),
|
||||
"JP": BertFeature(
|
||||
"./bert/deberta-v2-large-japanese-char-wwm",
|
||||
language="JP",
|
||||
),
|
||||
"EN": BertFeature(
|
||||
"./bert/deberta-v3-large",
|
||||
language="EN",
|
||||
),
|
||||
}
|
||||
|
||||
clap_feature = ClapFeature("./emotional/clap-htsat-fused")
|
||||
|
||||
|
||||
def generate_audio(
|
||||
slices,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
speaker,
|
||||
language,
|
||||
reference_audio,
|
||||
emotion,
|
||||
skip_start=False,
|
||||
skip_end=False,
|
||||
):
|
||||
audio_list = []
|
||||
# silence = np.zeros(hps.data.sampling_rate // 2, dtype=np.int16)
|
||||
with torch.no_grad():
|
||||
for idx, piece in enumerate(slices):
|
||||
skip_start = (idx != 0) and skip_start
|
||||
skip_end = (idx != len(slices) - 1) and skip_end
|
||||
audio = infer(
|
||||
piece,
|
||||
reference_audio=reference_audio,
|
||||
emotion=emotion,
|
||||
sdp_ratio=sdp_ratio,
|
||||
noise_scale=noise_scale,
|
||||
noise_scale_w=noise_scale_w,
|
||||
length_scale=length_scale,
|
||||
sid=speaker,
|
||||
language=language,
|
||||
hps=hps,
|
||||
net_g=net_g,
|
||||
device=device,
|
||||
skip_start=skip_start,
|
||||
skip_end=skip_end,
|
||||
bert=bert_feature_map,
|
||||
clap=clap_feature,
|
||||
)
|
||||
audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
|
||||
audio_list.append(audio16bit)
|
||||
# audio_list.append(silence) # 将静音添加到列表中
|
||||
return audio_list
|
||||
|
||||
|
||||
def generate_audio_multilang(
|
||||
slices,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
speaker,
|
||||
language,
|
||||
reference_audio,
|
||||
emotion,
|
||||
skip_start=False,
|
||||
skip_end=False,
|
||||
):
|
||||
audio_list = []
|
||||
# silence = np.zeros(hps.data.sampling_rate // 2, dtype=np.int16)
|
||||
with torch.no_grad():
|
||||
for idx, piece in enumerate(slices):
|
||||
skip_start = (idx != 0) and skip_start
|
||||
skip_end = (idx != len(slices) - 1) and skip_end
|
||||
audio = infer_multilang(
|
||||
piece,
|
||||
reference_audio=reference_audio,
|
||||
emotion=emotion,
|
||||
sdp_ratio=sdp_ratio,
|
||||
noise_scale=noise_scale,
|
||||
noise_scale_w=noise_scale_w,
|
||||
length_scale=length_scale,
|
||||
sid=speaker,
|
||||
language=language[idx],
|
||||
hps=hps,
|
||||
net_g=net_g,
|
||||
device=device,
|
||||
skip_start=skip_start,
|
||||
skip_end=skip_end,
|
||||
)
|
||||
audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
|
||||
audio_list.append(audio16bit)
|
||||
# audio_list.append(silence) # 将静音添加到列表中
|
||||
return audio_list
|
||||
|
||||
|
||||
def tts_split(
|
||||
text: str,
|
||||
speaker,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
language,
|
||||
cut_by_sent,
|
||||
interval_between_para,
|
||||
interval_between_sent,
|
||||
reference_audio,
|
||||
emotion,
|
||||
):
|
||||
if language == "mix":
|
||||
return ("invalid", None)
|
||||
while text.find("\n\n") != -1:
|
||||
text = text.replace("\n\n", "\n")
|
||||
para_list = re_matching.cut_para(text)
|
||||
audio_list = []
|
||||
if not cut_by_sent:
|
||||
for idx, p in enumerate(para_list):
|
||||
skip_start = idx != 0
|
||||
skip_end = idx != len(para_list) - 1
|
||||
audio = infer(
|
||||
p,
|
||||
reference_audio=reference_audio,
|
||||
emotion=emotion,
|
||||
sdp_ratio=sdp_ratio,
|
||||
noise_scale=noise_scale,
|
||||
noise_scale_w=noise_scale_w,
|
||||
length_scale=length_scale,
|
||||
sid=speaker,
|
||||
language=language,
|
||||
hps=hps,
|
||||
net_g=net_g,
|
||||
device=device,
|
||||
skip_start=skip_start,
|
||||
skip_end=skip_end,
|
||||
)
|
||||
audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
|
||||
audio_list.append(audio16bit)
|
||||
silence = np.zeros((int)(44100 * interval_between_para), dtype=np.int16)
|
||||
audio_list.append(silence)
|
||||
else:
|
||||
for idx, p in enumerate(para_list):
|
||||
skip_start = idx != 0
|
||||
skip_end = idx != len(para_list) - 1
|
||||
audio_list_sent = []
|
||||
sent_list = re_matching.cut_sent(p)
|
||||
for idx, s in enumerate(sent_list):
|
||||
skip_start = (idx != 0) and skip_start
|
||||
skip_end = (idx != len(sent_list) - 1) and skip_end
|
||||
audio = infer(
|
||||
s,
|
||||
reference_audio=reference_audio,
|
||||
emotion=emotion,
|
||||
sdp_ratio=sdp_ratio,
|
||||
noise_scale=noise_scale,
|
||||
noise_scale_w=noise_scale_w,
|
||||
length_scale=length_scale,
|
||||
sid=speaker,
|
||||
language=language,
|
||||
hps=hps,
|
||||
net_g=net_g,
|
||||
device=device,
|
||||
skip_start=skip_start,
|
||||
skip_end=skip_end,
|
||||
)
|
||||
audio_list_sent.append(audio)
|
||||
silence = np.zeros((int)(44100 * interval_between_sent))
|
||||
audio_list_sent.append(silence)
|
||||
if (interval_between_para - interval_between_sent) > 0:
|
||||
silence = np.zeros(
|
||||
(int)(44100 * (interval_between_para - interval_between_sent))
|
||||
)
|
||||
audio_list_sent.append(silence)
|
||||
audio16bit = gr.processing_utils.convert_to_16_bit_wav(
|
||||
np.concatenate(audio_list_sent)
|
||||
) # 对完整句子做音量归一
|
||||
audio_list.append(audio16bit)
|
||||
audio_concat = np.concatenate(audio_list)
|
||||
return ("Success", (44100, audio_concat))
|
||||
|
||||
|
||||
def tts_fn(
|
||||
text: str,
|
||||
speaker,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
language,
|
||||
reference_audio,
|
||||
emotion,
|
||||
prompt_mode,
|
||||
):
|
||||
if prompt_mode == "Audio prompt":
|
||||
if reference_audio == None:
|
||||
return ("Invalid audio prompt", None)
|
||||
else:
|
||||
reference_audio = load_audio(reference_audio)[1]
|
||||
else:
|
||||
reference_audio = None
|
||||
audio_list = []
|
||||
if language == "mix":
|
||||
bool_valid, str_valid = re_matching.validate_text(text)
|
||||
if not bool_valid:
|
||||
return str_valid, (
|
||||
hps.data.sampling_rate,
|
||||
np.concatenate([np.zeros(hps.data.sampling_rate // 2)]),
|
||||
)
|
||||
result = []
|
||||
for slice in re_matching.text_matching(text):
|
||||
_speaker = slice.pop()
|
||||
temp_contant = []
|
||||
temp_lang = []
|
||||
for lang, content in slice:
|
||||
if "|" in content:
|
||||
temp = []
|
||||
temp_ = []
|
||||
for i in content.split("|"):
|
||||
if i != "":
|
||||
temp.append([i])
|
||||
temp_.append([lang])
|
||||
else:
|
||||
temp.append([])
|
||||
temp_.append([])
|
||||
temp_contant += temp
|
||||
temp_lang += temp_
|
||||
else:
|
||||
if len(temp_contant) == 0:
|
||||
temp_contant.append([])
|
||||
temp_lang.append([])
|
||||
temp_contant[-1].append(content)
|
||||
temp_lang[-1].append(lang)
|
||||
for i, j in zip(temp_lang, temp_contant):
|
||||
result.append([*zip(i, j), _speaker])
|
||||
for i, one in enumerate(result):
|
||||
skip_start = i != 0
|
||||
skip_end = i != len(result) - 1
|
||||
_speaker = one.pop()
|
||||
idx = 0
|
||||
while idx < len(one):
|
||||
text_to_generate = []
|
||||
lang_to_generate = []
|
||||
while True:
|
||||
lang, content = one[idx]
|
||||
temp_text = [content]
|
||||
if len(text_to_generate) > 0:
|
||||
text_to_generate[-1] += [temp_text.pop(0)]
|
||||
lang_to_generate[-1] += [lang]
|
||||
if len(temp_text) > 0:
|
||||
text_to_generate += [[i] for i in temp_text]
|
||||
lang_to_generate += [[lang]] * len(temp_text)
|
||||
if idx + 1 < len(one):
|
||||
idx += 1
|
||||
else:
|
||||
break
|
||||
skip_start = (idx != 0) and skip_start
|
||||
skip_end = (idx != len(one) - 1) and skip_end
|
||||
print(text_to_generate, lang_to_generate)
|
||||
audio_list.extend(
|
||||
generate_audio_multilang(
|
||||
text_to_generate,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
_speaker,
|
||||
lang_to_generate,
|
||||
reference_audio,
|
||||
emotion,
|
||||
skip_start,
|
||||
skip_end,
|
||||
)
|
||||
)
|
||||
idx += 1
|
||||
elif language.lower() == "auto":
|
||||
for idx, slice in enumerate(text.split("|")):
|
||||
if slice == "":
|
||||
continue
|
||||
skip_start = idx != 0
|
||||
skip_end = idx != len(text.split("|")) - 1
|
||||
sentences_list = split_by_language(
|
||||
slice, target_languages=["zh", "ja", "en"]
|
||||
)
|
||||
idx = 0
|
||||
while idx < len(sentences_list):
|
||||
text_to_generate = []
|
||||
lang_to_generate = []
|
||||
while True:
|
||||
content, lang = sentences_list[idx]
|
||||
temp_text = [content]
|
||||
lang = lang.upper()
|
||||
if lang == "JA":
|
||||
lang = "JP"
|
||||
if len(text_to_generate) > 0:
|
||||
text_to_generate[-1] += [temp_text.pop(0)]
|
||||
lang_to_generate[-1] += [lang]
|
||||
if len(temp_text) > 0:
|
||||
text_to_generate += [[i] for i in temp_text]
|
||||
lang_to_generate += [[lang]] * len(temp_text)
|
||||
if idx + 1 < len(sentences_list):
|
||||
idx += 1
|
||||
else:
|
||||
break
|
||||
skip_start = (idx != 0) and skip_start
|
||||
skip_end = (idx != len(sentences_list) - 1) and skip_end
|
||||
print(text_to_generate, lang_to_generate)
|
||||
audio_list.extend(
|
||||
generate_audio_multilang(
|
||||
text_to_generate,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
speaker,
|
||||
lang_to_generate,
|
||||
reference_audio,
|
||||
emotion,
|
||||
skip_start,
|
||||
skip_end,
|
||||
)
|
||||
)
|
||||
idx += 1
|
||||
else:
|
||||
audio_list.extend(
|
||||
generate_audio(
|
||||
text.split("|"),
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
speaker,
|
||||
language,
|
||||
reference_audio,
|
||||
emotion,
|
||||
)
|
||||
)
|
||||
|
||||
audio_concat = np.concatenate(audio_list)
|
||||
return "Success", (hps.data.sampling_rate, audio_concat)
|
||||
|
||||
|
||||
def load_audio(path):
|
||||
audio, sr = librosa.load(path, 48000)
|
||||
# audio = librosa.resample(audio, 44100, 48000)
|
||||
return sr, audio
|
||||
|
||||
|
||||
def gr_util(item):
|
||||
if item == "Text prompt":
|
||||
return {"visible": True, "__type__": "update"}, {
|
||||
"visible": False,
|
||||
"__type__": "update",
|
||||
}
|
||||
else:
|
||||
return {"visible": False, "__type__": "update"}, {
|
||||
"visible": True,
|
||||
"__type__": "update",
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if config.webui_config.debug:
|
||||
logger.info("Enable DEBUG-LEVEL log")
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
hps = utils.get_hparams_from_file(config.webui_config.config_path)
|
||||
# 若config.json中未指定版本则默认为最新版本
|
||||
version = hps.version if hasattr(hps, "version") else latest_version
|
||||
net_g = get_net_g(
|
||||
model_path=config.webui_config.model, version=version, device=device, hps=hps
|
||||
)
|
||||
speaker_ids = hps.data.spk2id
|
||||
speakers = list(speaker_ids.keys())
|
||||
languages = ["ZH", "JP", "EN", "mix", "auto"]
|
||||
with gr.Blocks() as app:
|
||||
with gr.Row():
|
||||
with gr.Column():
|
||||
text = gr.TextArea(
|
||||
label="输入文本内容",
|
||||
placeholder="""
|
||||
如果你选择语言为\'mix\'，必须按照格式输入，否则报错:
|
||||
格式举例(zh是中文，jp是日语，不区分大小写；说话人举例:gongzi):
|
||||
[说话人1]<zh>你好，こんにちは！ <jp>こんにちは，世界。
|
||||
[说话人2]<zh>你好吗？<jp>元気ですか？
|
||||
[说话人3]<zh>谢谢。<jp>どういたしまして。
|
||||
...
|
||||
另外，所有的语言选项都可以用'|'分割长段实现分句生成。
|
||||
""",
|
||||
)
|
||||
trans = gr.Button("中翻日", variant="primary")
|
||||
slicer = gr.Button("快速切分", variant="primary")
|
||||
speaker = gr.Dropdown(
|
||||
choices=speakers, value=speakers[0], label="Speaker"
|
||||
)
|
||||
_ = gr.Markdown(
|
||||
value="提示模式（Prompt mode）：可选文字提示或音频提示，用于生成文字或音频指定风格的声音。\n"
|
||||
)
|
||||
prompt_mode = gr.Radio(
|
||||
["Text prompt", "Audio prompt"],
|
||||
label="Prompt Mode",
|
||||
value="Text prompt",
|
||||
)
|
||||
text_prompt = gr.Textbox(
|
||||
label="Text prompt",
|
||||
placeholder="用文字描述生成风格。如：Happy",
|
||||
value="Happy",
|
||||
visible=True,
|
||||
)
|
||||
audio_prompt = gr.Audio(
|
||||
label="Audio prompt", type="filepath", visible=False
|
||||
)
|
||||
sdp_ratio = gr.Slider(
|
||||
minimum=0, maximum=1, value=0.2, step=0.1, label="SDP Ratio"
|
||||
)
|
||||
noise_scale = gr.Slider(
|
||||
minimum=0.1, maximum=2, value=0.6, step=0.1, label="Noise"
|
||||
)
|
||||
noise_scale_w = gr.Slider(
|
||||
minimum=0.1, maximum=2, value=0.8, step=0.1, label="Noise_W"
|
||||
)
|
||||
length_scale = gr.Slider(
|
||||
minimum=0.1, maximum=2, value=1.0, step=0.1, label="Length"
|
||||
)
|
||||
language = gr.Dropdown(
|
||||
choices=languages, value=languages[0], label="Language"
|
||||
)
|
||||
btn = gr.Button("生成音频！", variant="primary")
|
||||
with gr.Column():
|
||||
with gr.Row():
|
||||
with gr.Column():
|
||||
interval_between_sent = gr.Slider(
|
||||
minimum=0,
|
||||
maximum=5,
|
||||
value=0.2,
|
||||
step=0.1,
|
||||
label="句间停顿(秒)，勾选按句切分才生效",
|
||||
)
|
||||
interval_between_para = gr.Slider(
|
||||
minimum=0,
|
||||
maximum=10,
|
||||
value=1,
|
||||
step=0.1,
|
||||
label="段间停顿(秒)，需要大于句间停顿才有效",
|
||||
)
|
||||
opt_cut_by_sent = gr.Checkbox(
|
||||
label="按句切分 在按段落切分的基础上再按句子切分文本"
|
||||
)
|
||||
slicer = gr.Button("切分生成", variant="primary")
|
||||
text_output = gr.Textbox(label="状态信息")
|
||||
audio_output = gr.Audio(label="输出音频")
|
||||
# explain_image = gr.Image(
|
||||
# label="参数解释信息",
|
||||
# show_label=True,
|
||||
# show_share_button=False,
|
||||
# show_download_button=False,
|
||||
# value=os.path.abspath("./img/参数说明.png"),
|
||||
# )
|
||||
btn.click(
|
||||
tts_fn,
|
||||
inputs=[
|
||||
text,
|
||||
speaker,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
language,
|
||||
audio_prompt,
|
||||
text_prompt,
|
||||
prompt_mode,
|
||||
],
|
||||
outputs=[text_output, audio_output],
|
||||
)
|
||||
|
||||
trans.click(
|
||||
translate,
|
||||
inputs=[text],
|
||||
outputs=[text],
|
||||
)
|
||||
slicer.click(
|
||||
tts_split,
|
||||
inputs=[
|
||||
text,
|
||||
speaker,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
language,
|
||||
opt_cut_by_sent,
|
||||
interval_between_para,
|
||||
interval_between_sent,
|
||||
audio_prompt,
|
||||
text_prompt,
|
||||
],
|
||||
outputs=[text_output, audio_output],
|
||||
)
|
||||
|
||||
prompt_mode.change(
|
||||
lambda x: gr_util(x),
|
||||
inputs=[prompt_mode],
|
||||
outputs=[text_prompt, audio_prompt],
|
||||
)
|
||||
|
||||
audio_prompt.upload(
|
||||
lambda x: load_audio(x),
|
||||
inputs=[audio_prompt],
|
||||
outputs=[audio_prompt],
|
||||
)
|
||||
|
||||
print("推理页面已开启!")
|
||||
webbrowser.open(f"http://127.0.0.1:{config.webui_config.port}")
|
||||
app.launch(share=config.webui_config.share, server_port=config.webui_config.port)
|
||||
BIN
img/yuyu.png
BIN
img/yuyu.png
Binary file not shown.
|
Before Width: | Height: | Size: 241 KiB |
BIN
img/参数说明.png
BIN
img/参数说明.png
Binary file not shown.
|
Before Width: | Height: | Size: 202 KiB |
BIN
img/宵宫.png
BIN
img/宵宫.png
Binary file not shown.
|
Before Width: | Height: | Size: 337 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 232 KiB |
BIN
img/神里绫华.png
BIN
img/神里绫华.png
Binary file not shown.
|
Before Width: | Height: | Size: 280 KiB |
BIN
img/纳西妲.png
BIN
img/纳西妲.png
Binary file not shown.
|
Before Width: | Height: | Size: 310 KiB |
242
infer.py
242
infer.py
@@ -1,109 +1,35 @@
|
||||
"""
|
||||
版本管理、兼容推理及模型加载实现。
|
||||
版本说明：
|
||||
1. 版本号与github的release版本号对应，使用哪个release版本训练的模型即对应其版本号
|
||||
2. 请在模型的config.json中显示声明版本号，添加一个字段"version" : "你的版本号"
|
||||
特殊版本说明：
|
||||
1.1.1-fix： 1.1.1版本训练的模型，但是在推理时使用dev的日语修复
|
||||
2.3：当前版本
|
||||
"""
|
||||
import torch
|
||||
|
||||
import commons
|
||||
from text import cleaned_text_to_sequence, get_bert
|
||||
|
||||
# from clap_wrapper import get_clap_audio_feature, get_clap_text_feature
|
||||
from text.cleaner import clean_text
|
||||
import utils
|
||||
|
||||
from models import SynthesizerTrn
|
||||
from text import cleaned_text_to_sequence, get_bert
|
||||
from text.cleaner import clean_text
|
||||
from text.symbols import symbols
|
||||
|
||||
from oldVersion.V220.models import SynthesizerTrn as V220SynthesizerTrn
|
||||
from oldVersion.V220.text import symbols as V220symbols
|
||||
from oldVersion.V210.models import SynthesizerTrn as V210SynthesizerTrn
|
||||
from oldVersion.V210.text import symbols as V210symbols
|
||||
from oldVersion.V200.models import SynthesizerTrn as V200SynthesizerTrn
|
||||
from oldVersion.V200.text import symbols as V200symbols
|
||||
from oldVersion.V111.models import SynthesizerTrn as V111SynthesizerTrn
|
||||
from oldVersion.V111.text import symbols as V111symbols
|
||||
from oldVersion.V110.models import SynthesizerTrn as V110SynthesizerTrn
|
||||
from oldVersion.V110.text import symbols as V110symbols
|
||||
from oldVersion.V101.models import SynthesizerTrn as V101SynthesizerTrn
|
||||
from oldVersion.V101.text import symbols as V101symbols
|
||||
|
||||
from oldVersion import V111, V110, V101, V200, V210, V220
|
||||
|
||||
# 当前版本信息
|
||||
latest_version = "2.3"
|
||||
|
||||
# 版本兼容
|
||||
SynthesizerTrnMap = {
|
||||
"2.2": V220SynthesizerTrn,
|
||||
"2.1": V210SynthesizerTrn,
|
||||
"2.0.2-fix": V200SynthesizerTrn,
|
||||
"2.0.1": V200SynthesizerTrn,
|
||||
"2.0": V200SynthesizerTrn,
|
||||
"1.1.1-fix": V111SynthesizerTrn,
|
||||
"1.1.1": V111SynthesizerTrn,
|
||||
"1.1": V110SynthesizerTrn,
|
||||
"1.1.0": V110SynthesizerTrn,
|
||||
"1.0.1": V101SynthesizerTrn,
|
||||
"1.0": V101SynthesizerTrn,
|
||||
"1.0.0": V101SynthesizerTrn,
|
||||
}
|
||||
|
||||
symbolsMap = {
|
||||
"2.2": V220symbols,
|
||||
"2.1": V210symbols,
|
||||
"2.0.2-fix": V200symbols,
|
||||
"2.0.1": V200symbols,
|
||||
"2.0": V200symbols,
|
||||
"1.1.1-fix": V111symbols,
|
||||
"1.1.1": V111symbols,
|
||||
"1.1": V110symbols,
|
||||
"1.1.0": V110symbols,
|
||||
"1.0.1": V101symbols,
|
||||
"1.0": V101symbols,
|
||||
"1.0.0": V101symbols,
|
||||
}
|
||||
|
||||
|
||||
# def get_emo_(reference_audio, emotion, sid):
|
||||
# emo = (
|
||||
# torch.from_numpy(get_emo(reference_audio))
|
||||
# if reference_audio and emotion == -1
|
||||
# else torch.FloatTensor(
|
||||
# np.load(f"emo_clustering/{sid}/cluster_center_{emotion}.npy")
|
||||
# )
|
||||
# )
|
||||
# return emo
|
||||
# latest_version = "1.0"
|
||||
|
||||
|
||||
def get_net_g(model_path: str, version: str, device: str, hps):
|
||||
if version != latest_version:
|
||||
net_g = SynthesizerTrnMap[version](
|
||||
len(symbolsMap[version]),
|
||||
hps.data.filter_length // 2 + 1,
|
||||
hps.train.segment_size // hps.data.hop_length,
|
||||
n_speakers=hps.data.n_speakers,
|
||||
**hps.model,
|
||||
).to(device)
|
||||
else:
|
||||
# 当前版本模型 net_g
|
||||
net_g = SynthesizerTrn(
|
||||
len(symbols),
|
||||
hps.data.filter_length // 2 + 1,
|
||||
hps.train.segment_size // hps.data.hop_length,
|
||||
n_speakers=hps.data.n_speakers,
|
||||
**hps.model,
|
||||
).to(device)
|
||||
net_g = SynthesizerTrn(
|
||||
len(symbols),
|
||||
hps.data.filter_length // 2 + 1,
|
||||
hps.train.segment_size // hps.data.hop_length,
|
||||
n_speakers=hps.data.n_speakers,
|
||||
**hps.model,
|
||||
).to(device)
|
||||
net_g.state_dict()
|
||||
_ = net_g.eval()
|
||||
_ = utils.load_checkpoint(model_path, net_g, None, skip_optimizer=True)
|
||||
if model_path.endswith(".pth") or model_path.endswith(".pt"):
|
||||
_ = utils.load_checkpoint(model_path, net_g, None, skip_optimizer=True)
|
||||
elif model_path.endswith(".safetensors"):
|
||||
_ = utils.load_safetensors(model_path, net_g, device)
|
||||
else:
|
||||
raise ValueError(f"Unknown model format: {model_path}")
|
||||
return net_g
|
||||
|
||||
|
||||
def get_text(text, language_str, hps, device, style_text=None, style_weight=0.7):
|
||||
style_text = None if style_text == "" else style_text
|
||||
# 在此处实现当前版本的get_text
|
||||
norm_text, phone, tone, word2ph = clean_text(text, language_str)
|
||||
phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
|
||||
@@ -123,15 +49,15 @@ def get_text(text, language_str, hps, device, style_text=None, style_weight=0.7)
|
||||
|
||||
if language_str == "ZH":
|
||||
bert = bert_ori
|
||||
ja_bert = torch.randn(1024, len(phone))
|
||||
en_bert = torch.randn(1024, len(phone))
|
||||
ja_bert = torch.zeros(1024, len(phone))
|
||||
en_bert = torch.zeros(1024, len(phone))
|
||||
elif language_str == "JP":
|
||||
bert = torch.randn(1024, len(phone))
|
||||
bert = torch.zeros(1024, len(phone))
|
||||
ja_bert = bert_ori
|
||||
en_bert = torch.randn(1024, len(phone))
|
||||
en_bert = torch.zeros(1024, len(phone))
|
||||
elif language_str == "EN":
|
||||
bert = torch.randn(1024, len(phone))
|
||||
ja_bert = torch.randn(1024, len(phone))
|
||||
bert = torch.zeros(1024, len(phone))
|
||||
ja_bert = torch.zeros(1024, len(phone))
|
||||
en_bert = bert_ori
|
||||
else:
|
||||
raise ValueError("language_str should be ZH, JP or EN")
|
||||
@@ -148,123 +74,21 @@ def get_text(text, language_str, hps, device, style_text=None, style_weight=0.7)
|
||||
|
||||
def infer(
|
||||
text,
|
||||
emotion,
|
||||
style_vec,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
sid,
|
||||
sid: int, # In the original Bert-VITS2, its speaker_name: str, but here it's id
|
||||
language,
|
||||
hps,
|
||||
net_g,
|
||||
device,
|
||||
reference_audio=None,
|
||||
skip_start=False,
|
||||
skip_end=False,
|
||||
style_text=None,
|
||||
style_weight=0.7,
|
||||
):
|
||||
# 2.2版本参数位置变了
|
||||
inferMap_V4 = {
|
||||
"2.2": V220.infer,
|
||||
}
|
||||
# 2.1 参数新增 emotion reference_audio skip_start skip_end
|
||||
inferMap_V3 = {
|
||||
"2.1": V210.infer,
|
||||
}
|
||||
# 支持中日英三语版本
|
||||
inferMap_V2 = {
|
||||
"2.0.2-fix": V200.infer,
|
||||
"2.0.1": V200.infer,
|
||||
"2.0": V200.infer,
|
||||
"1.1.1-fix": V111.infer_fix,
|
||||
"1.1.1": V111.infer,
|
||||
"1.1": V110.infer,
|
||||
"1.1.0": V110.infer,
|
||||
}
|
||||
# 仅支持中文版本
|
||||
# 在测试中，并未发现两个版本的模型不能互相通用
|
||||
inferMap_V1 = {
|
||||
"1.0.1": V101.infer,
|
||||
"1.0": V101.infer,
|
||||
"1.0.0": V101.infer,
|
||||
}
|
||||
version = hps.version if hasattr(hps, "version") else latest_version
|
||||
# 非当前版本，根据版本号选择合适的infer
|
||||
if version != latest_version:
|
||||
if version in inferMap_V4.keys():
|
||||
emotion = "" # Use empty emotion prompt
|
||||
return inferMap_V4[version](
|
||||
text,
|
||||
emotion,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
sid,
|
||||
language,
|
||||
hps,
|
||||
net_g,
|
||||
device,
|
||||
reference_audio,
|
||||
skip_start,
|
||||
skip_end,
|
||||
style_text,
|
||||
style_weight,
|
||||
)
|
||||
if version in inferMap_V3.keys():
|
||||
emotion = 0
|
||||
return inferMap_V3[version](
|
||||
text,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
sid,
|
||||
language,
|
||||
hps,
|
||||
net_g,
|
||||
device,
|
||||
reference_audio,
|
||||
emotion,
|
||||
skip_start,
|
||||
skip_end,
|
||||
style_text,
|
||||
style_weight,
|
||||
)
|
||||
if version in inferMap_V2.keys():
|
||||
return inferMap_V2[version](
|
||||
text,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
sid,
|
||||
language,
|
||||
hps,
|
||||
net_g,
|
||||
device,
|
||||
)
|
||||
if version in inferMap_V1.keys():
|
||||
return inferMap_V1[version](
|
||||
text,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
sid,
|
||||
hps,
|
||||
net_g,
|
||||
device,
|
||||
)
|
||||
# 在此处实现当前版本的推理
|
||||
# emo = get_emo_(reference_audio, emotion, sid)
|
||||
# if isinstance(reference_audio, np.ndarray):
|
||||
# emo = get_clap_audio_feature(reference_audio, device)
|
||||
# else:
|
||||
# emo = get_clap_text_feature(emotion, device)
|
||||
# emo = torch.squeeze(emo, dim=1)
|
||||
|
||||
bert, ja_bert, en_bert, phones, tones, lang_ids = get_text(
|
||||
text,
|
||||
language,
|
||||
@@ -295,19 +119,20 @@ def infer(
|
||||
ja_bert = ja_bert.to(device).unsqueeze(0)
|
||||
en_bert = en_bert.to(device).unsqueeze(0)
|
||||
x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
|
||||
# emo = emo.to(device).unsqueeze(0)
|
||||
style_vec = torch.from_numpy(style_vec).to(device).unsqueeze(0)
|
||||
del phones
|
||||
speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
|
||||
sid_tensor = torch.LongTensor([sid]).to(device)
|
||||
audio = (
|
||||
net_g.infer(
|
||||
x_tst,
|
||||
x_tst_lengths,
|
||||
speakers,
|
||||
sid_tensor,
|
||||
tones,
|
||||
lang_ids,
|
||||
bert,
|
||||
ja_bert,
|
||||
en_bert,
|
||||
style_vec=style_vec,
|
||||
sdp_ratio=sdp_ratio,
|
||||
noise_scale=noise_scale,
|
||||
noise_scale_w=noise_scale_w,
|
||||
@@ -323,9 +148,10 @@ def infer(
|
||||
lang_ids,
|
||||
bert,
|
||||
x_tst_lengths,
|
||||
speakers,
|
||||
sid_tensor,
|
||||
ja_bert,
|
||||
en_bert,
|
||||
style_vec,
|
||||
) # , emo
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
@@ -334,6 +160,7 @@ def infer(
|
||||
|
||||
def infer_multilang(
|
||||
text,
|
||||
style_vec,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
@@ -343,8 +170,6 @@ def infer_multilang(
|
||||
hps,
|
||||
net_g,
|
||||
device,
|
||||
reference_audio=None,
|
||||
emotion=None,
|
||||
skip_start=False,
|
||||
skip_end=False,
|
||||
):
|
||||
@@ -413,6 +238,7 @@ def infer_multilang(
|
||||
bert,
|
||||
ja_bert,
|
||||
en_bert,
|
||||
style_vec=style_vec,
|
||||
sdp_ratio=sdp_ratio,
|
||||
noise_scale=noise_scale,
|
||||
noise_scale_w=noise_scale_w,
|
||||
|
||||
41
initialize.py
Normal file
41
initialize.py
Normal file
@@ -0,0 +1,41 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
from tools.log import logger
|
||||
|
||||
|
||||
def download_bert_models():
|
||||
with open("bert/bert_models.json", "r") as fp:
|
||||
models = json.load(fp)
|
||||
for k, v in models.items():
|
||||
local_path = Path("bert").joinpath(k)
|
||||
for file in v["files"]:
|
||||
if not Path(local_path).joinpath(file).exists():
|
||||
logger.info(f"Downloading {k} {file}")
|
||||
hf_hub_download(
|
||||
v["repo_id"],
|
||||
file,
|
||||
local_dir=local_path,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
|
||||
|
||||
def download_pretrained_models():
|
||||
files = ["G_0.safetensors", "D_0.safetensors", "DUR_0.safetensors"]
|
||||
local_path = Path("pretrained")
|
||||
for file in files:
|
||||
if not Path(local_path).joinpath(file).exists():
|
||||
logger.info(f"Downloading pretrained {file}")
|
||||
hf_hub_download(
|
||||
"litagin/Style-Bert-VITS2-1.0-base",
|
||||
file,
|
||||
local_dir=local_path,
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
|
||||
|
||||
download_bert_models()
|
||||
|
||||
download_pretrained_models()
|
||||
@@ -83,7 +83,9 @@ def spec_to_mel_torch(spec, n_fft, num_mels, sampling_rate, fmin, fmax):
|
||||
dtype_device = str(spec.dtype) + "_" + str(spec.device)
|
||||
fmax_dtype_device = str(fmax) + "_" + dtype_device
|
||||
if fmax_dtype_device not in mel_basis:
|
||||
mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax)
|
||||
mel = librosa_mel_fn(
|
||||
sr=sampling_rate, n_fft=n_fft, n_mels=num_mels, fmin=fmin, fmax=fmax
|
||||
)
|
||||
mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(
|
||||
dtype=spec.dtype, device=spec.device
|
||||
)
|
||||
@@ -105,7 +107,9 @@ def mel_spectrogram_torch(
|
||||
fmax_dtype_device = str(fmax) + "_" + dtype_device
|
||||
wnsize_dtype_device = str(win_size) + "_" + dtype_device
|
||||
if fmax_dtype_device not in mel_basis:
|
||||
mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax)
|
||||
mel = librosa_mel_fn(
|
||||
sr=sampling_rate, n_fft=n_fft, n_mels=num_mels, fmin=fmin, fmax=fmax
|
||||
)
|
||||
mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(
|
||||
dtype=y.dtype, device=y.device
|
||||
)
|
||||
|
||||
2
model_assets/.gitignore
vendored
Normal file
2
model_assets/.gitignore
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
*
|
||||
!.gitignore
|
||||
157
models.py
157
models.py
@@ -1,18 +1,21 @@
|
||||
import math
|
||||
import warnings
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.nn import Conv1d, Conv2d, ConvTranspose1d
|
||||
from torch.nn import functional as F
|
||||
|
||||
import attentions
|
||||
import commons
|
||||
import modules
|
||||
import attentions
|
||||
import monotonic_align
|
||||
from commons import get_padding, init_weights
|
||||
from text import num_languages, num_tones, symbols
|
||||
|
||||
from torch.nn import Conv1d, ConvTranspose1d, Conv2d
|
||||
from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm
|
||||
|
||||
from commons import init_weights, get_padding
|
||||
from text import symbols, num_tones, num_languages
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
from torch.nn.utils import remove_weight_norm, spectral_norm, weight_norm
|
||||
|
||||
|
||||
class DurationDiscriminator(nn.Module): # vits2
|
||||
@@ -38,22 +41,33 @@ class DurationDiscriminator(nn.Module): # vits2
|
||||
self.norm_2 = modules.LayerNorm(filter_channels)
|
||||
self.dur_proj = nn.Conv1d(1, filter_channels, 1)
|
||||
|
||||
self.LSTM = nn.LSTM(
|
||||
2 * filter_channels, filter_channels, batch_first=True, bidirectional=True
|
||||
self.pre_out_conv_1 = nn.Conv1d(
|
||||
2 * filter_channels, filter_channels, kernel_size, padding=kernel_size // 2
|
||||
)
|
||||
self.pre_out_norm_1 = modules.LayerNorm(filter_channels)
|
||||
self.pre_out_conv_2 = nn.Conv1d(
|
||||
filter_channels, filter_channels, kernel_size, padding=kernel_size // 2
|
||||
)
|
||||
self.pre_out_norm_2 = modules.LayerNorm(filter_channels)
|
||||
|
||||
if gin_channels != 0:
|
||||
self.cond = nn.Conv1d(gin_channels, in_channels, 1)
|
||||
|
||||
self.output_layer = nn.Sequential(
|
||||
nn.Linear(2 * filter_channels, 1), nn.Sigmoid()
|
||||
)
|
||||
self.output_layer = nn.Sequential(nn.Linear(filter_channels, 1), nn.Sigmoid())
|
||||
|
||||
def forward_probability(self, x, dur):
|
||||
def forward_probability(self, x, x_mask, dur, g=None):
|
||||
dur = self.dur_proj(dur)
|
||||
x = torch.cat([x, dur], dim=1)
|
||||
x = self.pre_out_conv_1(x * x_mask)
|
||||
x = torch.relu(x)
|
||||
x = self.pre_out_norm_1(x)
|
||||
x = self.drop(x)
|
||||
x = self.pre_out_conv_2(x * x_mask)
|
||||
x = torch.relu(x)
|
||||
x = self.pre_out_norm_2(x)
|
||||
x = self.drop(x)
|
||||
x = x * x_mask
|
||||
x = x.transpose(1, 2)
|
||||
x, _ = self.LSTM(x)
|
||||
output_prob = self.output_layer(x)
|
||||
return output_prob
|
||||
|
||||
@@ -73,7 +87,7 @@ class DurationDiscriminator(nn.Module): # vits2
|
||||
|
||||
output_probs = []
|
||||
for dur in [dur_r, dur_hat]:
|
||||
output_prob = self.forward_probability(x, dur)
|
||||
output_prob = self.forward_probability(x, x_mask, dur, g)
|
||||
output_probs.append(output_prob)
|
||||
|
||||
return output_probs
|
||||
@@ -299,37 +313,6 @@ class DurationPredictor(nn.Module):
|
||||
return x * x_mask
|
||||
|
||||
|
||||
class Bottleneck(nn.Sequential):
|
||||
def __init__(self, in_dim, hidden_dim):
|
||||
c_fc1 = nn.Linear(in_dim, hidden_dim, bias=False)
|
||||
c_fc2 = nn.Linear(in_dim, hidden_dim, bias=False)
|
||||
super().__init__(*[c_fc1, c_fc2])
|
||||
|
||||
|
||||
class Block(nn.Module):
|
||||
def __init__(self, in_dim, hidden_dim) -> None:
|
||||
super().__init__()
|
||||
self.norm = nn.LayerNorm(in_dim)
|
||||
self.mlp = MLP(in_dim, hidden_dim)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
x = x + self.mlp(self.norm(x))
|
||||
return x
|
||||
|
||||
|
||||
class MLP(nn.Module):
|
||||
def __init__(self, in_dim, hidden_dim):
|
||||
super().__init__()
|
||||
self.c_fc1 = nn.Linear(in_dim, hidden_dim, bias=False)
|
||||
self.c_fc2 = nn.Linear(in_dim, hidden_dim, bias=False)
|
||||
self.c_proj = nn.Linear(hidden_dim, in_dim, bias=False)
|
||||
|
||||
def forward(self, x: torch.Tensor):
|
||||
x = F.silu(self.c_fc1(x)) * self.c_fc2(x)
|
||||
x = self.c_proj(x)
|
||||
return x
|
||||
|
||||
|
||||
class TextEncoder(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -341,6 +324,7 @@ class TextEncoder(nn.Module):
|
||||
n_layers,
|
||||
kernel_size,
|
||||
p_dropout,
|
||||
n_speakers,
|
||||
gin_channels=0,
|
||||
):
|
||||
super().__init__()
|
||||
@@ -362,6 +346,7 @@ class TextEncoder(nn.Module):
|
||||
self.bert_proj = nn.Conv1d(1024, hidden_channels, 1)
|
||||
self.ja_bert_proj = nn.Conv1d(1024, hidden_channels, 1)
|
||||
self.en_bert_proj = nn.Conv1d(1024, hidden_channels, 1)
|
||||
self.style_proj = nn.Linear(256, hidden_channels)
|
||||
|
||||
self.encoder = attentions.Encoder(
|
||||
hidden_channels,
|
||||
@@ -374,10 +359,24 @@ class TextEncoder(nn.Module):
|
||||
)
|
||||
self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
|
||||
|
||||
def forward(self, x, x_lengths, tone, language, bert, ja_bert, en_bert, g=None):
|
||||
def forward(
|
||||
self,
|
||||
x,
|
||||
x_lengths,
|
||||
tone,
|
||||
language,
|
||||
bert,
|
||||
ja_bert,
|
||||
en_bert,
|
||||
style_vec,
|
||||
sid,
|
||||
g=None,
|
||||
):
|
||||
bert_emb = self.bert_proj(bert).transpose(1, 2)
|
||||
ja_bert_emb = self.ja_bert_proj(ja_bert).transpose(1, 2)
|
||||
en_bert_emb = self.en_bert_proj(en_bert).transpose(1, 2)
|
||||
style_emb = self.style_proj(style_vec.unsqueeze(1))
|
||||
|
||||
x = (
|
||||
self.emb(x)
|
||||
+ self.tone_emb(tone)
|
||||
@@ -385,6 +384,7 @@ class TextEncoder(nn.Module):
|
||||
+ bert_emb
|
||||
+ ja_bert_emb
|
||||
+ en_bert_emb
|
||||
+ style_emb
|
||||
) * math.sqrt(
|
||||
self.hidden_channels
|
||||
) # [b, t, h]
|
||||
@@ -700,55 +700,6 @@ class MultiPeriodDiscriminator(torch.nn.Module):
|
||||
return y_d_rs, y_d_gs, fmap_rs, fmap_gs
|
||||
|
||||
|
||||
class WavLMDiscriminator(nn.Module):
|
||||
"""docstring for Discriminator."""
|
||||
|
||||
def __init__(
|
||||
self, slm_hidden=768, slm_layers=13, initial_channel=64, use_spectral_norm=False
|
||||
):
|
||||
super(WavLMDiscriminator, self).__init__()
|
||||
norm_f = weight_norm if use_spectral_norm == False else spectral_norm
|
||||
self.pre = norm_f(
|
||||
Conv1d(slm_hidden * slm_layers, initial_channel, 1, 1, padding=0)
|
||||
)
|
||||
|
||||
self.convs = nn.ModuleList(
|
||||
[
|
||||
norm_f(
|
||||
nn.Conv1d(
|
||||
initial_channel, initial_channel * 2, kernel_size=5, padding=2
|
||||
)
|
||||
),
|
||||
norm_f(
|
||||
nn.Conv1d(
|
||||
initial_channel * 2,
|
||||
initial_channel * 4,
|
||||
kernel_size=5,
|
||||
padding=2,
|
||||
)
|
||||
),
|
||||
norm_f(
|
||||
nn.Conv1d(initial_channel * 4, initial_channel * 4, 5, 1, padding=2)
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
self.conv_post = norm_f(Conv1d(initial_channel * 4, 1, 3, 1, padding=1))
|
||||
|
||||
def forward(self, x):
|
||||
x = self.pre(x)
|
||||
|
||||
fmap = []
|
||||
for l in self.convs:
|
||||
x = l(x)
|
||||
x = F.leaky_relu(x, modules.LRELU_SLOPE)
|
||||
fmap.append(x)
|
||||
x = self.conv_post(x)
|
||||
x = torch.flatten(x, 1, -1)
|
||||
|
||||
return x
|
||||
|
||||
|
||||
class ReferenceEncoder(nn.Module):
|
||||
"""
|
||||
inputs --- [N, Ty/r, n_mels*r] mels
|
||||
@@ -838,7 +789,7 @@ class SynthesizerTrn(nn.Module):
|
||||
n_layers_trans_flow=4,
|
||||
flow_share_parameter=False,
|
||||
use_transformer_flow=True,
|
||||
**kwargs
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__()
|
||||
self.n_vocab = n_vocab
|
||||
@@ -879,6 +830,7 @@ class SynthesizerTrn(nn.Module):
|
||||
n_layers,
|
||||
kernel_size,
|
||||
p_dropout,
|
||||
self.n_speakers,
|
||||
gin_channels=self.enc_gin_channels,
|
||||
)
|
||||
self.dec = Generator(
|
||||
@@ -946,13 +898,14 @@ class SynthesizerTrn(nn.Module):
|
||||
bert,
|
||||
ja_bert,
|
||||
en_bert,
|
||||
style_vec,
|
||||
):
|
||||
if self.n_speakers > 0:
|
||||
g = self.emb_g(sid).unsqueeze(-1) # [b, h, 1]
|
||||
else:
|
||||
g = self.ref_enc(y.transpose(1, 2)).unsqueeze(-1)
|
||||
x, m_p, logs_p, x_mask = self.enc_p(
|
||||
x, x_lengths, tone, language, bert, ja_bert, en_bert, g=g
|
||||
x, x_lengths, tone, language, bert, ja_bert, en_bert, style_vec, sid, g=g
|
||||
)
|
||||
z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g)
|
||||
z_p = self.flow(z, y_mask, g=g)
|
||||
@@ -995,11 +948,11 @@ class SynthesizerTrn(nn.Module):
|
||||
|
||||
logw_ = torch.log(w + 1e-6) * x_mask
|
||||
logw = self.dp(x, x_mask, g=g)
|
||||
logw_sdp = self.sdp(x, x_mask, g=g, reverse=True, noise_scale=1.0)
|
||||
# logw_sdp = self.sdp(x, x_mask, g=g, reverse=True, noise_scale=1.0)
|
||||
l_length_dp = torch.sum((logw - logw_) ** 2, [1, 2]) / torch.sum(
|
||||
x_mask
|
||||
) # for averaging
|
||||
l_length_sdp += torch.sum((logw_sdp - logw_) ** 2, [1, 2]) / torch.sum(x_mask)
|
||||
# l_length_sdp += torch.sum((logw_sdp - logw_) ** 2, [1, 2]) / torch.sum(x_mask)
|
||||
|
||||
l_length = l_length_dp + l_length_sdp
|
||||
|
||||
@@ -1019,8 +972,7 @@ class SynthesizerTrn(nn.Module):
|
||||
x_mask,
|
||||
y_mask,
|
||||
(z, z_p, m_p, logs_p, m_q, logs_q),
|
||||
(x, logw, logw_, logw_sdp),
|
||||
g,
|
||||
(x, logw, logw_),
|
||||
)
|
||||
|
||||
def infer(
|
||||
@@ -1033,6 +985,7 @@ class SynthesizerTrn(nn.Module):
|
||||
bert,
|
||||
ja_bert,
|
||||
en_bert,
|
||||
style_vec,
|
||||
noise_scale=0.667,
|
||||
length_scale=1,
|
||||
noise_scale_w=0.8,
|
||||
@@ -1047,7 +1000,7 @@ class SynthesizerTrn(nn.Module):
|
||||
else:
|
||||
g = self.ref_enc(y.transpose(1, 2)).unsqueeze(-1)
|
||||
x, m_p, logs_p, x_mask = self.enc_p(
|
||||
x, x_lengths, tone, language, bert, ja_bert, en_bert, g=g
|
||||
x, x_lengths, tone, language, bert, ja_bert, en_bert, style_vec, sid, g=g
|
||||
)
|
||||
logw = self.sdp(x, x_mask, g=g, reverse=True, noise_scale=noise_scale_w) * (
|
||||
sdp_ratio
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
"""
|
||||
Original models in Bert-VITS2 ver 2.1.
|
||||
"""
|
||||
import math
|
||||
import torch
|
||||
from torch import nn
|
||||
31
modules.py
31
modules.py
@@ -1,15 +1,19 @@
|
||||
import math
|
||||
import warnings
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.nn import Conv1d
|
||||
from torch.nn import functional as F
|
||||
|
||||
from torch.nn import Conv1d
|
||||
from torch.nn.utils import weight_norm, remove_weight_norm
|
||||
|
||||
import commons
|
||||
from commons import init_weights, get_padding
|
||||
from transforms import piecewise_rational_quadratic_transform
|
||||
from attentions import Encoder
|
||||
from commons import get_padding, init_weights
|
||||
from transforms import piecewise_rational_quadratic_transform
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
from torch.nn.utils import remove_weight_norm, weight_norm
|
||||
|
||||
LRELU_SLOPE = 0.1
|
||||
|
||||
@@ -578,20 +582,3 @@ class TransformerCouplingLayer(nn.Module):
|
||||
x1 = (x1 - m) * torch.exp(-logs) * x_mask
|
||||
x = torch.cat([x0, x1], 1)
|
||||
return x
|
||||
|
||||
x1, logabsdet = piecewise_rational_quadratic_transform(
|
||||
x1,
|
||||
unnormalized_widths,
|
||||
unnormalized_heights,
|
||||
unnormalized_derivatives,
|
||||
inverse=reverse,
|
||||
tails="linear",
|
||||
tail_bound=self.tail_bound,
|
||||
)
|
||||
|
||||
x = torch.cat([x0, x1], 1) * x_mask
|
||||
logdet = torch.sum(logabsdet * x_mask, [1, 2])
|
||||
if not reverse:
|
||||
return x, logdet
|
||||
else:
|
||||
return x
|
||||
|
||||
@@ -1,75 +0,0 @@
|
||||
"""
|
||||
1.0.1 版本兼容
|
||||
https://github.com/fishaudio/Bert-VITS2/releases/tag/1.0.1
|
||||
"""
|
||||
import torch
|
||||
import commons
|
||||
from .text.cleaner import clean_text
|
||||
from .text import cleaned_text_to_sequence
|
||||
from oldVersion.V111.text import get_bert
|
||||
|
||||
|
||||
def get_text(text, language_str, hps, device):
|
||||
norm_text, phone, tone, word2ph = clean_text(text, language_str)
|
||||
phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
|
||||
|
||||
if hps.data.add_blank:
|
||||
phone = commons.intersperse(phone, 0)
|
||||
tone = commons.intersperse(tone, 0)
|
||||
language = commons.intersperse(language, 0)
|
||||
for i in range(len(word2ph)):
|
||||
word2ph[i] = word2ph[i] * 2
|
||||
word2ph[0] += 1
|
||||
bert = get_bert(norm_text, word2ph, language_str, device)
|
||||
del word2ph
|
||||
|
||||
assert bert.shape[-1] == len(phone)
|
||||
|
||||
phone = torch.LongTensor(phone)
|
||||
tone = torch.LongTensor(tone)
|
||||
language = torch.LongTensor(language)
|
||||
|
||||
return bert, phone, tone, language
|
||||
|
||||
|
||||
def infer(
|
||||
text,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
sid,
|
||||
hps,
|
||||
net_g,
|
||||
device,
|
||||
):
|
||||
bert, phones, tones, lang_ids = get_text(text, "ZH", hps, device)
|
||||
with torch.no_grad():
|
||||
x_tst = phones.to(device).unsqueeze(0)
|
||||
tones = tones.to(device).unsqueeze(0)
|
||||
lang_ids = lang_ids.to(device).unsqueeze(0)
|
||||
bert = bert.to(device).unsqueeze(0)
|
||||
x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
|
||||
del phones
|
||||
speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
|
||||
audio = (
|
||||
net_g.infer(
|
||||
x_tst,
|
||||
x_tst_lengths,
|
||||
speakers,
|
||||
tones,
|
||||
lang_ids,
|
||||
bert,
|
||||
sdp_ratio=sdp_ratio,
|
||||
noise_scale=noise_scale,
|
||||
noise_scale_w=noise_scale_w,
|
||||
length_scale=length_scale,
|
||||
)[0][0, 0]
|
||||
.data.cpu()
|
||||
.float()
|
||||
.numpy()
|
||||
)
|
||||
del x_tst, tones, lang_ids, bert, x_tst_lengths, speakers
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
return audio
|
||||
@@ -1,977 +0,0 @@
|
||||
import math
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.nn import functional as F
|
||||
|
||||
import commons
|
||||
import modules
|
||||
import attentions
|
||||
import monotonic_align
|
||||
|
||||
from torch.nn import Conv1d, ConvTranspose1d, Conv2d
|
||||
from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm
|
||||
|
||||
from commons import init_weights, get_padding
|
||||
from .text import symbols, num_tones, num_languages
|
||||
|
||||
|
||||
class DurationDiscriminator(nn.Module): # vits2
|
||||
def __init__(
|
||||
self, in_channels, filter_channels, kernel_size, p_dropout, gin_channels=0
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self.in_channels = in_channels
|
||||
self.filter_channels = filter_channels
|
||||
self.kernel_size = kernel_size
|
||||
self.p_dropout = p_dropout
|
||||
self.gin_channels = gin_channels
|
||||
|
||||
self.drop = nn.Dropout(p_dropout)
|
||||
self.conv_1 = nn.Conv1d(
|
||||
in_channels, filter_channels, kernel_size, padding=kernel_size // 2
|
||||
)
|
||||
self.norm_1 = modules.LayerNorm(filter_channels)
|
||||
self.conv_2 = nn.Conv1d(
|
||||
filter_channels, filter_channels, kernel_size, padding=kernel_size // 2
|
||||
)
|
||||
self.norm_2 = modules.LayerNorm(filter_channels)
|
||||
self.dur_proj = nn.Conv1d(1, filter_channels, 1)
|
||||
|
||||
self.pre_out_conv_1 = nn.Conv1d(
|
||||
2 * filter_channels, filter_channels, kernel_size, padding=kernel_size // 2
|
||||
)
|
||||
self.pre_out_norm_1 = modules.LayerNorm(filter_channels)
|
||||
self.pre_out_conv_2 = nn.Conv1d(
|
||||
filter_channels, filter_channels, kernel_size, padding=kernel_size // 2
|
||||
)
|
||||
self.pre_out_norm_2 = modules.LayerNorm(filter_channels)
|
||||
|
||||
if gin_channels != 0:
|
||||
self.cond = nn.Conv1d(gin_channels, in_channels, 1)
|
||||
|
||||
self.output_layer = nn.Sequential(nn.Linear(filter_channels, 1), nn.Sigmoid())
|
||||
|
||||
def forward_probability(self, x, x_mask, dur, g=None):
|
||||
dur = self.dur_proj(dur)
|
||||
x = torch.cat([x, dur], dim=1)
|
||||
x = self.pre_out_conv_1(x * x_mask)
|
||||
x = torch.relu(x)
|
||||
x = self.pre_out_norm_1(x)
|
||||
x = self.drop(x)
|
||||
x = self.pre_out_conv_2(x * x_mask)
|
||||
x = torch.relu(x)
|
||||
x = self.pre_out_norm_2(x)
|
||||
x = self.drop(x)
|
||||
x = x * x_mask
|
||||
x = x.transpose(1, 2)
|
||||
output_prob = self.output_layer(x)
|
||||
return output_prob
|
||||
|
||||
def forward(self, x, x_mask, dur_r, dur_hat, g=None):
|
||||
x = torch.detach(x)
|
||||
if g is not None:
|
||||
g = torch.detach(g)
|
||||
x = x + self.cond(g)
|
||||
x = self.conv_1(x * x_mask)
|
||||
x = torch.relu(x)
|
||||
x = self.norm_1(x)
|
||||
x = self.drop(x)
|
||||
x = self.conv_2(x * x_mask)
|
||||
x = torch.relu(x)
|
||||
x = self.norm_2(x)
|
||||
x = self.drop(x)
|
||||
|
||||
output_probs = []
|
||||
for dur in [dur_r, dur_hat]:
|
||||
output_prob = self.forward_probability(x, x_mask, dur, g)
|
||||
output_probs.append(output_prob)
|
||||
|
||||
return output_probs
|
||||
|
||||
|
||||
class TransformerCouplingBlock(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
channels,
|
||||
hidden_channels,
|
||||
filter_channels,
|
||||
n_heads,
|
||||
n_layers,
|
||||
kernel_size,
|
||||
p_dropout,
|
||||
n_flows=4,
|
||||
gin_channels=0,
|
||||
share_parameter=False,
|
||||
):
|
||||
super().__init__()
|
||||
self.channels = channels
|
||||
self.hidden_channels = hidden_channels
|
||||
self.kernel_size = kernel_size
|
||||
self.n_layers = n_layers
|
||||
self.n_flows = n_flows
|
||||
self.gin_channels = gin_channels
|
||||
|
||||
self.flows = nn.ModuleList()
|
||||
|
||||
self.wn = (
|
||||
attentions.FFT(
|
||||
hidden_channels,
|
||||
filter_channels,
|
||||
n_heads,
|
||||
n_layers,
|
||||
kernel_size,
|
||||
p_dropout,
|
||||
isflow=True,
|
||||
gin_channels=self.gin_channels,
|
||||
)
|
||||
if share_parameter
|
||||
else None
|
||||
)
|
||||
|
||||
for i in range(n_flows):
|
||||
self.flows.append(
|
||||
modules.TransformerCouplingLayer(
|
||||
channels,
|
||||
hidden_channels,
|
||||
kernel_size,
|
||||
n_layers,
|
||||
n_heads,
|
||||
p_dropout,
|
||||
filter_channels,
|
||||
mean_only=True,
|
||||
wn_sharing_parameter=self.wn,
|
||||
gin_channels=self.gin_channels,
|
||||
)
|
||||
)
|
||||
self.flows.append(modules.Flip())
|
||||
|
||||
def forward(self, x, x_mask, g=None, reverse=False):
|
||||
if not reverse:
|
||||
for flow in self.flows:
|
||||
x, _ = flow(x, x_mask, g=g, reverse=reverse)
|
||||
else:
|
||||
for flow in reversed(self.flows):
|
||||
x = flow(x, x_mask, g=g, reverse=reverse)
|
||||
return x
|
||||
|
||||
|
||||
class StochasticDurationPredictor(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
in_channels,
|
||||
filter_channels,
|
||||
kernel_size,
|
||||
p_dropout,
|
||||
n_flows=4,
|
||||
gin_channels=0,
|
||||
):
|
||||
super().__init__()
|
||||
filter_channels = in_channels # it needs to be removed from future version.
|
||||
self.in_channels = in_channels
|
||||
self.filter_channels = filter_channels
|
||||
self.kernel_size = kernel_size
|
||||
self.p_dropout = p_dropout
|
||||
self.n_flows = n_flows
|
||||
self.gin_channels = gin_channels
|
||||
|
||||
self.log_flow = modules.Log()
|
||||
self.flows = nn.ModuleList()
|
||||
self.flows.append(modules.ElementwiseAffine(2))
|
||||
for i in range(n_flows):
|
||||
self.flows.append(
|
||||
modules.ConvFlow(2, filter_channels, kernel_size, n_layers=3)
|
||||
)
|
||||
self.flows.append(modules.Flip())
|
||||
|
||||
self.post_pre = nn.Conv1d(1, filter_channels, 1)
|
||||
self.post_proj = nn.Conv1d(filter_channels, filter_channels, 1)
|
||||
self.post_convs = modules.DDSConv(
|
||||
filter_channels, kernel_size, n_layers=3, p_dropout=p_dropout
|
||||
)
|
||||
self.post_flows = nn.ModuleList()
|
||||
self.post_flows.append(modules.ElementwiseAffine(2))
|
||||
for i in range(4):
|
||||
self.post_flows.append(
|
||||
modules.ConvFlow(2, filter_channels, kernel_size, n_layers=3)
|
||||
)
|
||||
self.post_flows.append(modules.Flip())
|
||||
|
||||
self.pre = nn.Conv1d(in_channels, filter_channels, 1)
|
||||
self.proj = nn.Conv1d(filter_channels, filter_channels, 1)
|
||||
self.convs = modules.DDSConv(
|
||||
filter_channels, kernel_size, n_layers=3, p_dropout=p_dropout
|
||||
)
|
||||
if gin_channels != 0:
|
||||
self.cond = nn.Conv1d(gin_channels, filter_channels, 1)
|
||||
|
||||
def forward(self, x, x_mask, w=None, g=None, reverse=False, noise_scale=1.0):
|
||||
x = torch.detach(x)
|
||||
x = self.pre(x)
|
||||
if g is not None:
|
||||
g = torch.detach(g)
|
||||
x = x + self.cond(g)
|
||||
x = self.convs(x, x_mask)
|
||||
x = self.proj(x) * x_mask
|
||||
|
||||
if not reverse:
|
||||
flows = self.flows
|
||||
assert w is not None
|
||||
|
||||
logdet_tot_q = 0
|
||||
h_w = self.post_pre(w)
|
||||
h_w = self.post_convs(h_w, x_mask)
|
||||
h_w = self.post_proj(h_w) * x_mask
|
||||
e_q = (
|
||||
torch.randn(w.size(0), 2, w.size(2)).to(device=x.device, dtype=x.dtype)
|
||||
* x_mask
|
||||
)
|
||||
z_q = e_q
|
||||
for flow in self.post_flows:
|
||||
z_q, logdet_q = flow(z_q, x_mask, g=(x + h_w))
|
||||
logdet_tot_q += logdet_q
|
||||
z_u, z1 = torch.split(z_q, [1, 1], 1)
|
||||
u = torch.sigmoid(z_u) * x_mask
|
||||
z0 = (w - u) * x_mask
|
||||
logdet_tot_q += torch.sum(
|
||||
(F.logsigmoid(z_u) + F.logsigmoid(-z_u)) * x_mask, [1, 2]
|
||||
)
|
||||
logq = (
|
||||
torch.sum(-0.5 * (math.log(2 * math.pi) + (e_q**2)) * x_mask, [1, 2])
|
||||
- logdet_tot_q
|
||||
)
|
||||
|
||||
logdet_tot = 0
|
||||
z0, logdet = self.log_flow(z0, x_mask)
|
||||
logdet_tot += logdet
|
||||
z = torch.cat([z0, z1], 1)
|
||||
for flow in flows:
|
||||
z, logdet = flow(z, x_mask, g=x, reverse=reverse)
|
||||
logdet_tot = logdet_tot + logdet
|
||||
nll = (
|
||||
torch.sum(0.5 * (math.log(2 * math.pi) + (z**2)) * x_mask, [1, 2])
|
||||
- logdet_tot
|
||||
)
|
||||
return nll + logq # [b]
|
||||
else:
|
||||
flows = list(reversed(self.flows))
|
||||
flows = flows[:-2] + [flows[-1]] # remove a useless vflow
|
||||
z = (
|
||||
torch.randn(x.size(0), 2, x.size(2)).to(device=x.device, dtype=x.dtype)
|
||||
* noise_scale
|
||||
)
|
||||
for flow in flows:
|
||||
z = flow(z, x_mask, g=x, reverse=reverse)
|
||||
z0, z1 = torch.split(z, [1, 1], 1)
|
||||
logw = z0
|
||||
return logw
|
||||
|
||||
|
||||
class DurationPredictor(nn.Module):
|
||||
def __init__(
|
||||
self, in_channels, filter_channels, kernel_size, p_dropout, gin_channels=0
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self.in_channels = in_channels
|
||||
self.filter_channels = filter_channels
|
||||
self.kernel_size = kernel_size
|
||||
self.p_dropout = p_dropout
|
||||
self.gin_channels = gin_channels
|
||||
|
||||
self.drop = nn.Dropout(p_dropout)
|
||||
self.conv_1 = nn.Conv1d(
|
||||
in_channels, filter_channels, kernel_size, padding=kernel_size // 2
|
||||
)
|
||||
self.norm_1 = modules.LayerNorm(filter_channels)
|
||||
self.conv_2 = nn.Conv1d(
|
||||
filter_channels, filter_channels, kernel_size, padding=kernel_size // 2
|
||||
)
|
||||
self.norm_2 = modules.LayerNorm(filter_channels)
|
||||
self.proj = nn.Conv1d(filter_channels, 1, 1)
|
||||
|
||||
if gin_channels != 0:
|
||||
self.cond = nn.Conv1d(gin_channels, in_channels, 1)
|
||||
|
||||
def forward(self, x, x_mask, g=None):
|
||||
x = torch.detach(x)
|
||||
if g is not None:
|
||||
g = torch.detach(g)
|
||||
x = x + self.cond(g)
|
||||
x = self.conv_1(x * x_mask)
|
||||
x = torch.relu(x)
|
||||
x = self.norm_1(x)
|
||||
x = self.drop(x)
|
||||
x = self.conv_2(x * x_mask)
|
||||
x = torch.relu(x)
|
||||
x = self.norm_2(x)
|
||||
x = self.drop(x)
|
||||
x = self.proj(x * x_mask)
|
||||
return x * x_mask
|
||||
|
||||
|
||||
class TextEncoder(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
n_vocab,
|
||||
out_channels,
|
||||
hidden_channels,
|
||||
filter_channels,
|
||||
n_heads,
|
||||
n_layers,
|
||||
kernel_size,
|
||||
p_dropout,
|
||||
gin_channels=0,
|
||||
):
|
||||
super().__init__()
|
||||
self.n_vocab = n_vocab
|
||||
self.out_channels = out_channels
|
||||
self.hidden_channels = hidden_channels
|
||||
self.filter_channels = filter_channels
|
||||
self.n_heads = n_heads
|
||||
self.n_layers = n_layers
|
||||
self.kernel_size = kernel_size
|
||||
self.p_dropout = p_dropout
|
||||
self.gin_channels = gin_channels
|
||||
self.emb = nn.Embedding(len(symbols), hidden_channels)
|
||||
nn.init.normal_(self.emb.weight, 0.0, hidden_channels**-0.5)
|
||||
self.tone_emb = nn.Embedding(num_tones, hidden_channels)
|
||||
nn.init.normal_(self.tone_emb.weight, 0.0, hidden_channels**-0.5)
|
||||
self.language_emb = nn.Embedding(num_languages, hidden_channels)
|
||||
nn.init.normal_(self.language_emb.weight, 0.0, hidden_channels**-0.5)
|
||||
self.bert_proj = nn.Conv1d(1024, hidden_channels, 1)
|
||||
|
||||
self.encoder = attentions.Encoder(
|
||||
hidden_channels,
|
||||
filter_channels,
|
||||
n_heads,
|
||||
n_layers,
|
||||
kernel_size,
|
||||
p_dropout,
|
||||
gin_channels=self.gin_channels,
|
||||
)
|
||||
self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
|
||||
|
||||
def forward(self, x, x_lengths, tone, language, bert, g=None):
|
||||
x = (
|
||||
self.emb(x)
|
||||
+ self.tone_emb(tone)
|
||||
+ self.language_emb(language)
|
||||
+ self.bert_proj(bert).transpose(1, 2)
|
||||
) * math.sqrt(
|
||||
self.hidden_channels
|
||||
) # [b, t, h]
|
||||
x = torch.transpose(x, 1, -1) # [b, h, t]
|
||||
x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(
|
||||
x.dtype
|
||||
)
|
||||
|
||||
x = self.encoder(x * x_mask, x_mask, g=g)
|
||||
stats = self.proj(x) * x_mask
|
||||
|
||||
m, logs = torch.split(stats, self.out_channels, dim=1)
|
||||
return x, m, logs, x_mask
|
||||
|
||||
|
||||
class ResidualCouplingBlock(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
channels,
|
||||
hidden_channels,
|
||||
kernel_size,
|
||||
dilation_rate,
|
||||
n_layers,
|
||||
n_flows=4,
|
||||
gin_channels=0,
|
||||
):
|
||||
super().__init__()
|
||||
self.channels = channels
|
||||
self.hidden_channels = hidden_channels
|
||||
self.kernel_size = kernel_size
|
||||
self.dilation_rate = dilation_rate
|
||||
self.n_layers = n_layers
|
||||
self.n_flows = n_flows
|
||||
self.gin_channels = gin_channels
|
||||
|
||||
self.flows = nn.ModuleList()
|
||||
for i in range(n_flows):
|
||||
self.flows.append(
|
||||
modules.ResidualCouplingLayer(
|
||||
channels,
|
||||
hidden_channels,
|
||||
kernel_size,
|
||||
dilation_rate,
|
||||
n_layers,
|
||||
gin_channels=gin_channels,
|
||||
mean_only=True,
|
||||
)
|
||||
)
|
||||
self.flows.append(modules.Flip())
|
||||
|
||||
def forward(self, x, x_mask, g=None, reverse=False):
|
||||
if not reverse:
|
||||
for flow in self.flows:
|
||||
x, _ = flow(x, x_mask, g=g, reverse=reverse)
|
||||
else:
|
||||
for flow in reversed(self.flows):
|
||||
x = flow(x, x_mask, g=g, reverse=reverse)
|
||||
return x
|
||||
|
||||
|
||||
class PosteriorEncoder(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
in_channels,
|
||||
out_channels,
|
||||
hidden_channels,
|
||||
kernel_size,
|
||||
dilation_rate,
|
||||
n_layers,
|
||||
gin_channels=0,
|
||||
):
|
||||
super().__init__()
|
||||
self.in_channels = in_channels
|
||||
self.out_channels = out_channels
|
||||
self.hidden_channels = hidden_channels
|
||||
self.kernel_size = kernel_size
|
||||
self.dilation_rate = dilation_rate
|
||||
self.n_layers = n_layers
|
||||
self.gin_channels = gin_channels
|
||||
|
||||
self.pre = nn.Conv1d(in_channels, hidden_channels, 1)
|
||||
self.enc = modules.WN(
|
||||
hidden_channels,
|
||||
kernel_size,
|
||||
dilation_rate,
|
||||
n_layers,
|
||||
gin_channels=gin_channels,
|
||||
)
|
||||
self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
|
||||
|
||||
def forward(self, x, x_lengths, g=None):
|
||||
x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(
|
||||
x.dtype
|
||||
)
|
||||
x = self.pre(x) * x_mask
|
||||
x = self.enc(x, x_mask, g=g)
|
||||
stats = self.proj(x) * x_mask
|
||||
m, logs = torch.split(stats, self.out_channels, dim=1)
|
||||
z = (m + torch.randn_like(m) * torch.exp(logs)) * x_mask
|
||||
return z, m, logs, x_mask
|
||||
|
||||
|
||||
class Generator(torch.nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
initial_channel,
|
||||
resblock,
|
||||
resblock_kernel_sizes,
|
||||
resblock_dilation_sizes,
|
||||
upsample_rates,
|
||||
upsample_initial_channel,
|
||||
upsample_kernel_sizes,
|
||||
gin_channels=0,
|
||||
):
|
||||
super(Generator, self).__init__()
|
||||
self.num_kernels = len(resblock_kernel_sizes)
|
||||
self.num_upsamples = len(upsample_rates)
|
||||
self.conv_pre = Conv1d(
|
||||
initial_channel, upsample_initial_channel, 7, 1, padding=3
|
||||
)
|
||||
resblock = modules.ResBlock1 if resblock == "1" else modules.ResBlock2
|
||||
|
||||
self.ups = nn.ModuleList()
|
||||
for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):
|
||||
self.ups.append(
|
||||
weight_norm(
|
||||
ConvTranspose1d(
|
||||
upsample_initial_channel // (2**i),
|
||||
upsample_initial_channel // (2 ** (i + 1)),
|
||||
k,
|
||||
u,
|
||||
padding=(k - u) // 2,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
self.resblocks = nn.ModuleList()
|
||||
for i in range(len(self.ups)):
|
||||
ch = upsample_initial_channel // (2 ** (i + 1))
|
||||
for j, (k, d) in enumerate(
|
||||
zip(resblock_kernel_sizes, resblock_dilation_sizes)
|
||||
):
|
||||
self.resblocks.append(resblock(ch, k, d))
|
||||
|
||||
self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False)
|
||||
self.ups.apply(init_weights)
|
||||
|
||||
if gin_channels != 0:
|
||||
self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1)
|
||||
|
||||
def forward(self, x, g=None):
|
||||
x = self.conv_pre(x)
|
||||
if g is not None:
|
||||
x = x + self.cond(g)
|
||||
|
||||
for i in range(self.num_upsamples):
|
||||
x = F.leaky_relu(x, modules.LRELU_SLOPE)
|
||||
x = self.ups[i](x)
|
||||
xs = None
|
||||
for j in range(self.num_kernels):
|
||||
if xs is None:
|
||||
xs = self.resblocks[i * self.num_kernels + j](x)
|
||||
else:
|
||||
xs += self.resblocks[i * self.num_kernels + j](x)
|
||||
x = xs / self.num_kernels
|
||||
x = F.leaky_relu(x)
|
||||
x = self.conv_post(x)
|
||||
x = torch.tanh(x)
|
||||
|
||||
return x
|
||||
|
||||
def remove_weight_norm(self):
|
||||
print("Removing weight norm...")
|
||||
for l in self.ups:
|
||||
remove_weight_norm(l)
|
||||
for l in self.resblocks:
|
||||
l.remove_weight_norm()
|
||||
|
||||
|
||||
class DiscriminatorP(torch.nn.Module):
|
||||
def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False):
|
||||
super(DiscriminatorP, self).__init__()
|
||||
self.period = period
|
||||
self.use_spectral_norm = use_spectral_norm
|
||||
norm_f = weight_norm if use_spectral_norm is False else spectral_norm
|
||||
self.convs = nn.ModuleList(
|
||||
[
|
||||
norm_f(
|
||||
Conv2d(
|
||||
1,
|
||||
32,
|
||||
(kernel_size, 1),
|
||||
(stride, 1),
|
||||
padding=(get_padding(kernel_size, 1), 0),
|
||||
)
|
||||
),
|
||||
norm_f(
|
||||
Conv2d(
|
||||
32,
|
||||
128,
|
||||
(kernel_size, 1),
|
||||
(stride, 1),
|
||||
padding=(get_padding(kernel_size, 1), 0),
|
||||
)
|
||||
),
|
||||
norm_f(
|
||||
Conv2d(
|
||||
128,
|
||||
512,
|
||||
(kernel_size, 1),
|
||||
(stride, 1),
|
||||
padding=(get_padding(kernel_size, 1), 0),
|
||||
)
|
||||
),
|
||||
norm_f(
|
||||
Conv2d(
|
||||
512,
|
||||
1024,
|
||||
(kernel_size, 1),
|
||||
(stride, 1),
|
||||
padding=(get_padding(kernel_size, 1), 0),
|
||||
)
|
||||
),
|
||||
norm_f(
|
||||
Conv2d(
|
||||
1024,
|
||||
1024,
|
||||
(kernel_size, 1),
|
||||
1,
|
||||
padding=(get_padding(kernel_size, 1), 0),
|
||||
)
|
||||
),
|
||||
]
|
||||
)
|
||||
self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0)))
|
||||
|
||||
def forward(self, x):
|
||||
fmap = []
|
||||
|
||||
# 1d to 2d
|
||||
b, c, t = x.shape
|
||||
if t % self.period != 0: # pad first
|
||||
n_pad = self.period - (t % self.period)
|
||||
x = F.pad(x, (0, n_pad), "reflect")
|
||||
t = t + n_pad
|
||||
x = x.view(b, c, t // self.period, self.period)
|
||||
|
||||
for l in self.convs:
|
||||
x = l(x)
|
||||
x = F.leaky_relu(x, modules.LRELU_SLOPE)
|
||||
fmap.append(x)
|
||||
x = self.conv_post(x)
|
||||
fmap.append(x)
|
||||
x = torch.flatten(x, 1, -1)
|
||||
|
||||
return x, fmap
|
||||
|
||||
|
||||
class DiscriminatorS(torch.nn.Module):
|
||||
def __init__(self, use_spectral_norm=False):
|
||||
super(DiscriminatorS, self).__init__()
|
||||
norm_f = weight_norm if use_spectral_norm is False else spectral_norm
|
||||
self.convs = nn.ModuleList(
|
||||
[
|
||||
norm_f(Conv1d(1, 16, 15, 1, padding=7)),
|
||||
norm_f(Conv1d(16, 64, 41, 4, groups=4, padding=20)),
|
||||
norm_f(Conv1d(64, 256, 41, 4, groups=16, padding=20)),
|
||||
norm_f(Conv1d(256, 1024, 41, 4, groups=64, padding=20)),
|
||||
norm_f(Conv1d(1024, 1024, 41, 4, groups=256, padding=20)),
|
||||
norm_f(Conv1d(1024, 1024, 5, 1, padding=2)),
|
||||
]
|
||||
)
|
||||
self.conv_post = norm_f(Conv1d(1024, 1, 3, 1, padding=1))
|
||||
|
||||
def forward(self, x):
|
||||
fmap = []
|
||||
|
||||
for l in self.convs:
|
||||
x = l(x)
|
||||
x = F.leaky_relu(x, modules.LRELU_SLOPE)
|
||||
fmap.append(x)
|
||||
x = self.conv_post(x)
|
||||
fmap.append(x)
|
||||
x = torch.flatten(x, 1, -1)
|
||||
|
||||
return x, fmap
|
||||
|
||||
|
||||
class MultiPeriodDiscriminator(torch.nn.Module):
|
||||
def __init__(self, use_spectral_norm=False):
|
||||
super(MultiPeriodDiscriminator, self).__init__()
|
||||
periods = [2, 3, 5, 7, 11]
|
||||
|
||||
discs = [DiscriminatorS(use_spectral_norm=use_spectral_norm)]
|
||||
discs = discs + [
|
||||
DiscriminatorP(i, use_spectral_norm=use_spectral_norm) for i in periods
|
||||
]
|
||||
self.discriminators = nn.ModuleList(discs)
|
||||
|
||||
def forward(self, y, y_hat):
|
||||
y_d_rs = []
|
||||
y_d_gs = []
|
||||
fmap_rs = []
|
||||
fmap_gs = []
|
||||
for i, d in enumerate(self.discriminators):
|
||||
y_d_r, fmap_r = d(y)
|
||||
y_d_g, fmap_g = d(y_hat)
|
||||
y_d_rs.append(y_d_r)
|
||||
y_d_gs.append(y_d_g)
|
||||
fmap_rs.append(fmap_r)
|
||||
fmap_gs.append(fmap_g)
|
||||
|
||||
return y_d_rs, y_d_gs, fmap_rs, fmap_gs
|
||||
|
||||
|
||||
class ReferenceEncoder(nn.Module):
|
||||
"""
|
||||
inputs --- [N, Ty/r, n_mels*r] mels
|
||||
outputs --- [N, ref_enc_gru_size]
|
||||
"""
|
||||
|
||||
def __init__(self, spec_channels, gin_channels=0):
|
||||
super().__init__()
|
||||
self.spec_channels = spec_channels
|
||||
ref_enc_filters = [32, 32, 64, 64, 128, 128]
|
||||
K = len(ref_enc_filters)
|
||||
filters = [1] + ref_enc_filters
|
||||
convs = [
|
||||
weight_norm(
|
||||
nn.Conv2d(
|
||||
in_channels=filters[i],
|
||||
out_channels=filters[i + 1],
|
||||
kernel_size=(3, 3),
|
||||
stride=(2, 2),
|
||||
padding=(1, 1),
|
||||
)
|
||||
)
|
||||
for i in range(K)
|
||||
]
|
||||
self.convs = nn.ModuleList(convs)
|
||||
# self.wns = nn.ModuleList([weight_norm(num_features=ref_enc_filters[i]) for i in range(K)])
|
||||
|
||||
out_channels = self.calculate_channels(spec_channels, 3, 2, 1, K)
|
||||
self.gru = nn.GRU(
|
||||
input_size=ref_enc_filters[-1] * out_channels,
|
||||
hidden_size=256 // 2,
|
||||
batch_first=True,
|
||||
)
|
||||
self.proj = nn.Linear(128, gin_channels)
|
||||
|
||||
def forward(self, inputs, mask=None):
|
||||
N = inputs.size(0)
|
||||
out = inputs.view(N, 1, -1, self.spec_channels) # [N, 1, Ty, n_freqs]
|
||||
for conv in self.convs:
|
||||
out = conv(out)
|
||||
# out = wn(out)
|
||||
out = F.relu(out) # [N, 128, Ty//2^K, n_mels//2^K]
|
||||
|
||||
out = out.transpose(1, 2) # [N, Ty//2^K, 128, n_mels//2^K]
|
||||
T = out.size(1)
|
||||
N = out.size(0)
|
||||
out = out.contiguous().view(N, T, -1) # [N, Ty//2^K, 128*n_mels//2^K]
|
||||
|
||||
self.gru.flatten_parameters()
|
||||
memory, out = self.gru(out) # out --- [1, N, 128]
|
||||
|
||||
return self.proj(out.squeeze(0))
|
||||
|
||||
def calculate_channels(self, L, kernel_size, stride, pad, n_convs):
|
||||
for i in range(n_convs):
|
||||
L = (L - kernel_size + 2 * pad) // stride + 1
|
||||
return L
|
||||
|
||||
|
||||
class SynthesizerTrn(nn.Module):
|
||||
"""
|
||||
Synthesizer for Training
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
n_vocab,
|
||||
spec_channels,
|
||||
segment_size,
|
||||
inter_channels,
|
||||
hidden_channels,
|
||||
filter_channels,
|
||||
n_heads,
|
||||
n_layers,
|
||||
kernel_size,
|
||||
p_dropout,
|
||||
resblock,
|
||||
resblock_kernel_sizes,
|
||||
resblock_dilation_sizes,
|
||||
upsample_rates,
|
||||
upsample_initial_channel,
|
||||
upsample_kernel_sizes,
|
||||
n_speakers=256,
|
||||
gin_channels=256,
|
||||
use_sdp=True,
|
||||
n_flow_layer=4,
|
||||
n_layers_trans_flow=3,
|
||||
flow_share_parameter=False,
|
||||
use_transformer_flow=True,
|
||||
**kwargs
|
||||
):
|
||||
super().__init__()
|
||||
self.n_vocab = n_vocab
|
||||
self.spec_channels = spec_channels
|
||||
self.inter_channels = inter_channels
|
||||
self.hidden_channels = hidden_channels
|
||||
self.filter_channels = filter_channels
|
||||
self.n_heads = n_heads
|
||||
self.n_layers = n_layers
|
||||
self.kernel_size = kernel_size
|
||||
self.p_dropout = p_dropout
|
||||
self.resblock = resblock
|
||||
self.resblock_kernel_sizes = resblock_kernel_sizes
|
||||
self.resblock_dilation_sizes = resblock_dilation_sizes
|
||||
self.upsample_rates = upsample_rates
|
||||
self.upsample_initial_channel = upsample_initial_channel
|
||||
self.upsample_kernel_sizes = upsample_kernel_sizes
|
||||
self.segment_size = segment_size
|
||||
self.n_speakers = n_speakers
|
||||
self.gin_channels = gin_channels
|
||||
self.n_layers_trans_flow = n_layers_trans_flow
|
||||
self.use_spk_conditioned_encoder = kwargs.get(
|
||||
"use_spk_conditioned_encoder", True
|
||||
)
|
||||
self.use_sdp = use_sdp
|
||||
self.use_noise_scaled_mas = kwargs.get("use_noise_scaled_mas", False)
|
||||
self.mas_noise_scale_initial = kwargs.get("mas_noise_scale_initial", 0.01)
|
||||
self.noise_scale_delta = kwargs.get("noise_scale_delta", 2e-6)
|
||||
self.current_mas_noise_scale = self.mas_noise_scale_initial
|
||||
if self.use_spk_conditioned_encoder and gin_channels > 0:
|
||||
self.enc_gin_channels = gin_channels
|
||||
self.enc_p = TextEncoder(
|
||||
n_vocab,
|
||||
inter_channels,
|
||||
hidden_channels,
|
||||
filter_channels,
|
||||
n_heads,
|
||||
n_layers,
|
||||
kernel_size,
|
||||
p_dropout,
|
||||
gin_channels=self.enc_gin_channels,
|
||||
)
|
||||
self.dec = Generator(
|
||||
inter_channels,
|
||||
resblock,
|
||||
resblock_kernel_sizes,
|
||||
resblock_dilation_sizes,
|
||||
upsample_rates,
|
||||
upsample_initial_channel,
|
||||
upsample_kernel_sizes,
|
||||
gin_channels=gin_channels,
|
||||
)
|
||||
self.enc_q = PosteriorEncoder(
|
||||
spec_channels,
|
||||
inter_channels,
|
||||
hidden_channels,
|
||||
5,
|
||||
1,
|
||||
16,
|
||||
gin_channels=gin_channels,
|
||||
)
|
||||
if use_transformer_flow:
|
||||
self.flow = TransformerCouplingBlock(
|
||||
inter_channels,
|
||||
hidden_channels,
|
||||
filter_channels,
|
||||
n_heads,
|
||||
n_layers_trans_flow,
|
||||
5,
|
||||
p_dropout,
|
||||
n_flow_layer,
|
||||
gin_channels=gin_channels,
|
||||
share_parameter=flow_share_parameter,
|
||||
)
|
||||
else:
|
||||
self.flow = ResidualCouplingBlock(
|
||||
inter_channels,
|
||||
hidden_channels,
|
||||
5,
|
||||
1,
|
||||
n_flow_layer,
|
||||
gin_channels=gin_channels,
|
||||
)
|
||||
self.sdp = StochasticDurationPredictor(
|
||||
hidden_channels, 192, 3, 0.5, 4, gin_channels=gin_channels
|
||||
)
|
||||
self.dp = DurationPredictor(
|
||||
hidden_channels, 256, 3, 0.5, gin_channels=gin_channels
|
||||
)
|
||||
|
||||
if n_speakers > 0:
|
||||
self.emb_g = nn.Embedding(n_speakers, gin_channels)
|
||||
else:
|
||||
self.ref_enc = ReferenceEncoder(spec_channels, gin_channels)
|
||||
|
||||
def forward(self, x, x_lengths, y, y_lengths, sid, tone, language, bert):
|
||||
if self.n_speakers >= 0:
|
||||
g = self.emb_g(sid).unsqueeze(-1) # [b, h, 1]
|
||||
else:
|
||||
g = self.ref_enc(y.transpose(1, 2)).unsqueeze(-1)
|
||||
x, m_p, logs_p, x_mask = self.enc_p(x, x_lengths, tone, language, bert, g=g)
|
||||
z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g)
|
||||
z_p = self.flow(z, y_mask, g=g)
|
||||
|
||||
with torch.no_grad():
|
||||
# negative cross-entropy
|
||||
s_p_sq_r = torch.exp(-2 * logs_p) # [b, d, t]
|
||||
neg_cent1 = torch.sum(
|
||||
-0.5 * math.log(2 * math.pi) - logs_p, [1], keepdim=True
|
||||
) # [b, 1, t_s]
|
||||
neg_cent2 = torch.matmul(
|
||||
-0.5 * (z_p**2).transpose(1, 2), s_p_sq_r
|
||||
) # [b, t_t, d] x [b, d, t_s] = [b, t_t, t_s]
|
||||
neg_cent3 = torch.matmul(
|
||||
z_p.transpose(1, 2), (m_p * s_p_sq_r)
|
||||
) # [b, t_t, d] x [b, d, t_s] = [b, t_t, t_s]
|
||||
neg_cent4 = torch.sum(
|
||||
-0.5 * (m_p**2) * s_p_sq_r, [1], keepdim=True
|
||||
) # [b, 1, t_s]
|
||||
neg_cent = neg_cent1 + neg_cent2 + neg_cent3 + neg_cent4
|
||||
if self.use_noise_scaled_mas:
|
||||
epsilon = (
|
||||
torch.std(neg_cent)
|
||||
* torch.randn_like(neg_cent)
|
||||
* self.current_mas_noise_scale
|
||||
)
|
||||
neg_cent = neg_cent + epsilon
|
||||
|
||||
attn_mask = torch.unsqueeze(x_mask, 2) * torch.unsqueeze(y_mask, -1)
|
||||
attn = (
|
||||
monotonic_align.maximum_path(neg_cent, attn_mask.squeeze(1))
|
||||
.unsqueeze(1)
|
||||
.detach()
|
||||
)
|
||||
|
||||
w = attn.sum(2)
|
||||
|
||||
l_length_sdp = self.sdp(x, x_mask, w, g=g)
|
||||
l_length_sdp = l_length_sdp / torch.sum(x_mask)
|
||||
|
||||
logw_ = torch.log(w + 1e-6) * x_mask
|
||||
logw = self.dp(x, x_mask, g=g)
|
||||
l_length_dp = torch.sum((logw - logw_) ** 2, [1, 2]) / torch.sum(
|
||||
x_mask
|
||||
) # for averaging
|
||||
|
||||
l_length = l_length_dp + l_length_sdp
|
||||
|
||||
# expand prior
|
||||
m_p = torch.matmul(attn.squeeze(1), m_p.transpose(1, 2)).transpose(1, 2)
|
||||
logs_p = torch.matmul(attn.squeeze(1), logs_p.transpose(1, 2)).transpose(1, 2)
|
||||
|
||||
z_slice, ids_slice = commons.rand_slice_segments(
|
||||
z, y_lengths, self.segment_size
|
||||
)
|
||||
o = self.dec(z_slice, g=g)
|
||||
return (
|
||||
o,
|
||||
l_length,
|
||||
attn,
|
||||
ids_slice,
|
||||
x_mask,
|
||||
y_mask,
|
||||
(z, z_p, m_p, logs_p, m_q, logs_q),
|
||||
(x, logw, logw_),
|
||||
)
|
||||
|
||||
def infer(
|
||||
self,
|
||||
x,
|
||||
x_lengths,
|
||||
sid,
|
||||
tone,
|
||||
language,
|
||||
bert,
|
||||
noise_scale=0.667,
|
||||
length_scale=1,
|
||||
noise_scale_w=0.8,
|
||||
max_len=None,
|
||||
sdp_ratio=0,
|
||||
y=None,
|
||||
):
|
||||
# x, m_p, logs_p, x_mask = self.enc_p(x, x_lengths, tone, language, bert)
|
||||
# g = self.gst(y)
|
||||
if self.n_speakers > 0:
|
||||
g = self.emb_g(sid).unsqueeze(-1) # [b, h, 1]
|
||||
else:
|
||||
g = self.ref_enc(y.transpose(1, 2)).unsqueeze(-1)
|
||||
x, m_p, logs_p, x_mask = self.enc_p(x, x_lengths, tone, language, bert, g=g)
|
||||
logw = self.sdp(x, x_mask, g=g, reverse=True, noise_scale=noise_scale_w) * (
|
||||
sdp_ratio
|
||||
) + self.dp(x, x_mask, g=g) * (1 - sdp_ratio)
|
||||
w = torch.exp(logw) * x_mask * length_scale
|
||||
w_ceil = torch.ceil(w)
|
||||
y_lengths = torch.clamp_min(torch.sum(w_ceil, [1, 2]), 1).long()
|
||||
y_mask = torch.unsqueeze(commons.sequence_mask(y_lengths, None), 1).to(
|
||||
x_mask.dtype
|
||||
)
|
||||
attn_mask = torch.unsqueeze(x_mask, 2) * torch.unsqueeze(y_mask, -1)
|
||||
attn = commons.generate_path(w_ceil, attn_mask)
|
||||
|
||||
m_p = torch.matmul(attn.squeeze(1), m_p.transpose(1, 2)).transpose(
|
||||
1, 2
|
||||
) # [b, t', t], [b, t, d] -> [b, d, t']
|
||||
logs_p = torch.matmul(attn.squeeze(1), logs_p.transpose(1, 2)).transpose(
|
||||
1, 2
|
||||
) # [b, t', t], [b, t, d] -> [b, d, t']
|
||||
|
||||
z_p = m_p + torch.randn_like(m_p) * torch.exp(logs_p) * noise_scale
|
||||
z = self.flow(z_p, y_mask, g=g, reverse=True)
|
||||
o = self.dec((z * y_mask)[:, :, :max_len], g=g)
|
||||
return o, attn, y_mask, (z, z_p, m_p, logs_p)
|
||||
@@ -1,28 +0,0 @@
|
||||
from .symbols import *
|
||||
|
||||
|
||||
_symbol_to_id = {s: i for i, s in enumerate(symbols)}
|
||||
|
||||
|
||||
def cleaned_text_to_sequence(cleaned_text, tones, language):
|
||||
"""Converts a string of text to a sequence of IDs corresponding to the symbols in the text.
|
||||
Args:
|
||||
text: string to convert to a sequence
|
||||
Returns:
|
||||
List of integers corresponding to the symbols in the text
|
||||
"""
|
||||
phones = [_symbol_to_id[symbol] for symbol in cleaned_text]
|
||||
tone_start = language_tone_start_map[language]
|
||||
tones = [i + tone_start for i in tones]
|
||||
lang_id = language_id_map[language]
|
||||
lang_ids = [lang_id for i in phones]
|
||||
return phones, tones, lang_ids
|
||||
|
||||
|
||||
def get_bert(norm_text, word2ph, language):
|
||||
from .chinese_bert import get_bert_feature as zh_bert
|
||||
from .english_bert_mock import get_bert_feature as en_bert
|
||||
|
||||
lang_bert_func_map = {"ZH": zh_bert, "EN": en_bert}
|
||||
bert = lang_bert_func_map[language](norm_text, word2ph)
|
||||
return bert
|
||||
@@ -1,199 +0,0 @@
|
||||
import os
|
||||
import re
|
||||
|
||||
import cn2an
|
||||
from pypinyin import lazy_pinyin, Style
|
||||
|
||||
|
||||
from .symbols import punctuation
|
||||
from .tone_sandhi import ToneSandhi
|
||||
|
||||
current_file_path = os.path.dirname(__file__)
|
||||
pinyin_to_symbol_map = {
|
||||
line.split("\t")[0]: line.strip().split("\t")[1]
|
||||
for line in open(os.path.join(current_file_path, "opencpop-strict.txt")).readlines()
|
||||
}
|
||||
|
||||
import jieba.posseg as psg
|
||||
|
||||
|
||||
rep_map = {
|
||||
"：": ",",
|
||||
"；": ",",
|
||||
"，": ",",
|
||||
"。": ".",
|
||||
"！": "!",
|
||||
"？": "?",
|
||||
"\n": ".",
|
||||
"·": ",",
|
||||
"、": ",",
|
||||
"...": "…",
|
||||
"$": ".",
|
||||
"“": "'",
|
||||
"”": "'",
|
||||
"‘": "'",
|
||||
"’": "'",
|
||||
"（": "'",
|
||||
"）": "'",
|
||||
"(": "'",
|
||||
")": "'",
|
||||
"《": "'",
|
||||
"》": "'",
|
||||
"【": "'",
|
||||
"】": "'",
|
||||
"[": "'",
|
||||
"]": "'",
|
||||
"—": "-",
|
||||
"～": "-",
|
||||
"~": "-",
|
||||
"「": "'",
|
||||
"」": "'",
|
||||
}
|
||||
|
||||
tone_modifier = ToneSandhi()
|
||||
|
||||
|
||||
def replace_punctuation(text):
|
||||
text = text.replace("嗯", "恩").replace("呣", "母")
|
||||
pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys()))
|
||||
|
||||
replaced_text = pattern.sub(lambda x: rep_map[x.group()], text)
|
||||
|
||||
replaced_text = re.sub(
|
||||
r"[^\u4e00-\u9fa5" + "".join(punctuation) + r"]+", "", replaced_text
|
||||
)
|
||||
|
||||
return replaced_text
|
||||
|
||||
|
||||
def g2p(text):
|
||||
pattern = r"(?<=[{0}])\s*".format("".join(punctuation))
|
||||
sentences = [i for i in re.split(pattern, text) if i.strip() != ""]
|
||||
phones, tones, word2ph = _g2p(sentences)
|
||||
assert sum(word2ph) == len(phones)
|
||||
assert len(word2ph) == len(text) # Sometimes it will crash,you can add a try-catch.
|
||||
phones = ["_"] + phones + ["_"]
|
||||
tones = [0] + tones + [0]
|
||||
word2ph = [1] + word2ph + [1]
|
||||
return phones, tones, word2ph
|
||||
|
||||
|
||||
def _get_initials_finals(word):
|
||||
initials = []
|
||||
finals = []
|
||||
orig_initials = lazy_pinyin(word, neutral_tone_with_five=True, style=Style.INITIALS)
|
||||
orig_finals = lazy_pinyin(
|
||||
word, neutral_tone_with_five=True, style=Style.FINALS_TONE3
|
||||
)
|
||||
for c, v in zip(orig_initials, orig_finals):
|
||||
initials.append(c)
|
||||
finals.append(v)
|
||||
return initials, finals
|
||||
|
||||
|
||||
def _g2p(segments):
|
||||
phones_list = []
|
||||
tones_list = []
|
||||
word2ph = []
|
||||
for seg in segments:
|
||||
# Replace all English words in the sentence
|
||||
seg = re.sub("[a-zA-Z]+", "", seg)
|
||||
seg_cut = psg.lcut(seg)
|
||||
initials = []
|
||||
finals = []
|
||||
seg_cut = tone_modifier.pre_merge_for_modify(seg_cut)
|
||||
for word, pos in seg_cut:
|
||||
if pos == "eng":
|
||||
continue
|
||||
sub_initials, sub_finals = _get_initials_finals(word)
|
||||
sub_finals = tone_modifier.modified_tone(word, pos, sub_finals)
|
||||
initials.append(sub_initials)
|
||||
finals.append(sub_finals)
|
||||
|
||||
# assert len(sub_initials) == len(sub_finals) == len(word)
|
||||
initials = sum(initials, [])
|
||||
finals = sum(finals, [])
|
||||
#
|
||||
for c, v in zip(initials, finals):
|
||||
raw_pinyin = c + v
|
||||
# NOTE: post process for pypinyin outputs
|
||||
# we discriminate i, ii and iii
|
||||
if c == v:
|
||||
assert c in punctuation
|
||||
phone = [c]
|
||||
tone = "0"
|
||||
word2ph.append(1)
|
||||
else:
|
||||
v_without_tone = v[:-1]
|
||||
tone = v[-1]
|
||||
|
||||
pinyin = c + v_without_tone
|
||||
assert tone in "12345"
|
||||
|
||||
if c:
|
||||
# 多音节
|
||||
v_rep_map = {
|
||||
"uei": "ui",
|
||||
"iou": "iu",
|
||||
"uen": "un",
|
||||
}
|
||||
if v_without_tone in v_rep_map.keys():
|
||||
pinyin = c + v_rep_map[v_without_tone]
|
||||
else:
|
||||
# 单音节
|
||||
pinyin_rep_map = {
|
||||
"ing": "ying",
|
||||
"i": "yi",
|
||||
"in": "yin",
|
||||
"u": "wu",
|
||||
}
|
||||
if pinyin in pinyin_rep_map.keys():
|
||||
pinyin = pinyin_rep_map[pinyin]
|
||||
else:
|
||||
single_rep_map = {
|
||||
"v": "yu",
|
||||
"e": "e",
|
||||
"i": "y",
|
||||
"u": "w",
|
||||
}
|
||||
if pinyin[0] in single_rep_map.keys():
|
||||
pinyin = single_rep_map[pinyin[0]] + pinyin[1:]
|
||||
|
||||
assert pinyin in pinyin_to_symbol_map.keys(), (pinyin, seg, raw_pinyin)
|
||||
phone = pinyin_to_symbol_map[pinyin].split(" ")
|
||||
word2ph.append(len(phone))
|
||||
|
||||
phones_list += phone
|
||||
tones_list += [int(tone)] * len(phone)
|
||||
return phones_list, tones_list, word2ph
|
||||
|
||||
|
||||
def text_normalize(text):
|
||||
numbers = re.findall(r"\d+(?:\.?\d+)?", text)
|
||||
for number in numbers:
|
||||
text = text.replace(number, cn2an.an2cn(number), 1)
|
||||
text = replace_punctuation(text)
|
||||
return text
|
||||
|
||||
|
||||
def get_bert_feature(text, word2ph):
|
||||
from text import chinese_bert
|
||||
|
||||
return chinese_bert.get_bert_feature(text, word2ph)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
from text.chinese_bert import get_bert_feature
|
||||
|
||||
text = "啊！但是《原神》是由,米哈\游自主， [研发]的一款全.新开放世界.冒险游戏"
|
||||
text = text_normalize(text)
|
||||
print(text)
|
||||
phones, tones, word2ph = g2p(text)
|
||||
bert = get_bert_feature(text, word2ph)
|
||||
|
||||
print(phones, tones, word2ph, bert.shape)
|
||||
|
||||
|
||||
# # 示例用法
|
||||
# text = "这是一个示例文本：,你好！这是一个测试...."
|
||||
# print(g2p_paddle(text)) # 输出: 这是一个示例文本你好这是一个测试
|
||||
@@ -1,100 +0,0 @@
|
||||
import torch
|
||||
import sys
|
||||
from transformers import AutoTokenizer, AutoModelForMaskedLM
|
||||
|
||||
device = torch.device(
|
||||
"cuda"
|
||||
if torch.cuda.is_available()
|
||||
else (
|
||||
"mps"
|
||||
if sys.platform == "darwin" and torch.backends.mps.is_available()
|
||||
else "cpu"
|
||||
)
|
||||
)
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained("./bert/chinese-roberta-wwm-ext-large")
|
||||
model = AutoModelForMaskedLM.from_pretrained("./bert/chinese-roberta-wwm-ext-large").to(
|
||||
device
|
||||
)
|
||||
|
||||
|
||||
def get_bert_feature(text, word2ph):
|
||||
with torch.no_grad():
|
||||
inputs = tokenizer(text, return_tensors="pt")
|
||||
for i in inputs:
|
||||
inputs[i] = inputs[i].to(device)
|
||||
res = model(**inputs, output_hidden_states=True)
|
||||
res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
|
||||
|
||||
assert len(word2ph) == len(text) + 2
|
||||
word2phone = word2ph
|
||||
phone_level_feature = []
|
||||
for i in range(len(word2phone)):
|
||||
repeat_feature = res[i].repeat(word2phone[i], 1)
|
||||
phone_level_feature.append(repeat_feature)
|
||||
|
||||
phone_level_feature = torch.cat(phone_level_feature, dim=0)
|
||||
|
||||
return phone_level_feature.T
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# feature = get_bert_feature('你好,我是说的道理。')
|
||||
import torch
|
||||
|
||||
word_level_feature = torch.rand(38, 1024) # 12个词,每个词1024维特征
|
||||
word2phone = [
|
||||
1,
|
||||
2,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
1,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
1,
|
||||
]
|
||||
|
||||
# 计算总帧数
|
||||
total_frames = sum(word2phone)
|
||||
print(word_level_feature.shape)
|
||||
print(word2phone)
|
||||
phone_level_feature = []
|
||||
for i in range(len(word2phone)):
|
||||
print(word_level_feature[i].shape)
|
||||
|
||||
# 对每个词重复word2phone[i]次
|
||||
repeat_feature = word_level_feature[i].repeat(word2phone[i], 1)
|
||||
phone_level_feature.append(repeat_feature)
|
||||
|
||||
phone_level_feature = torch.cat(phone_level_feature, dim=0)
|
||||
print(phone_level_feature.shape) # torch.Size([36, 1024])
|
||||
@@ -1,28 +0,0 @@
|
||||
from . import chinese, cleaned_text_to_sequence
|
||||
|
||||
|
||||
language_module_map = {"ZH": chinese}
|
||||
|
||||
|
||||
def clean_text(text, language):
|
||||
language_module = language_module_map[language]
|
||||
norm_text = language_module.text_normalize(text)
|
||||
phones, tones, word2ph = language_module.g2p(norm_text)
|
||||
return norm_text, phones, tones, word2ph
|
||||
|
||||
|
||||
def clean_text_bert(text, language):
|
||||
language_module = language_module_map[language]
|
||||
norm_text = language_module.text_normalize(text)
|
||||
phones, tones, word2ph = language_module.g2p(norm_text)
|
||||
bert = language_module.get_bert_feature(norm_text, word2ph)
|
||||
return phones, tones, bert
|
||||
|
||||
|
||||
def text_to_sequence(text, language):
|
||||
norm_text, phones, tones, word2ph = clean_text(text, language)
|
||||
return cleaned_text_to_sequence(phones, tones, language)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pass
|
||||
@@ -1,214 +0,0 @@
|
||||
import pickle
|
||||
import os
|
||||
import re
|
||||
from g2p_en import G2p
|
||||
|
||||
from text import symbols
|
||||
|
||||
current_file_path = os.path.dirname(__file__)
|
||||
CMU_DICT_PATH = os.path.join(current_file_path, "cmudict.rep")
|
||||
CACHE_PATH = os.path.join(current_file_path, "cmudict_cache.pickle")
|
||||
_g2p = G2p()
|
||||
|
||||
arpa = {
|
||||
"AH0",
|
||||
"S",
|
||||
"AH1",
|
||||
"EY2",
|
||||
"AE2",
|
||||
"EH0",
|
||||
"OW2",
|
||||
"UH0",
|
||||
"NG",
|
||||
"B",
|
||||
"G",
|
||||
"AY0",
|
||||
"M",
|
||||
"AA0",
|
||||
"F",
|
||||
"AO0",
|
||||
"ER2",
|
||||
"UH1",
|
||||
"IY1",
|
||||
"AH2",
|
||||
"DH",
|
||||
"IY0",
|
||||
"EY1",
|
||||
"IH0",
|
||||
"K",
|
||||
"N",
|
||||
"W",
|
||||
"IY2",
|
||||
"T",
|
||||
"AA1",
|
||||
"ER1",
|
||||
"EH2",
|
||||
"OY0",
|
||||
"UH2",
|
||||
"UW1",
|
||||
"Z",
|
||||
"AW2",
|
||||
"AW1",
|
||||
"V",
|
||||
"UW2",
|
||||
"AA2",
|
||||
"ER",
|
||||
"AW0",
|
||||
"UW0",
|
||||
"R",
|
||||
"OW1",
|
||||
"EH1",
|
||||
"ZH",
|
||||
"AE0",
|
||||
"IH2",
|
||||
"IH",
|
||||
"Y",
|
||||
"JH",
|
||||
"P",
|
||||
"AY1",
|
||||
"EY0",
|
||||
"OY2",
|
||||
"TH",
|
||||
"HH",
|
||||
"D",
|
||||
"ER0",
|
||||
"CH",
|
||||
"AO1",
|
||||
"AE1",
|
||||
"AO2",
|
||||
"OY1",
|
||||
"AY2",
|
||||
"IH1",
|
||||
"OW0",
|
||||
"L",
|
||||
"SH",
|
||||
}
|
||||
|
||||
|
||||
def post_replace_ph(ph):
|
||||
rep_map = {
|
||||
"：": ",",
|
||||
"；": ",",
|
||||
"，": ",",
|
||||
"。": ".",
|
||||
"！": "!",
|
||||
"？": "?",
|
||||
"\n": ".",
|
||||
"·": ",",
|
||||
"、": ",",
|
||||
"...": "…",
|
||||
"v": "V",
|
||||
}
|
||||
if ph in rep_map.keys():
|
||||
ph = rep_map[ph]
|
||||
if ph in symbols:
|
||||
return ph
|
||||
if ph not in symbols:
|
||||
ph = "UNK"
|
||||
return ph
|
||||
|
||||
|
||||
def read_dict():
|
||||
g2p_dict = {}
|
||||
start_line = 49
|
||||
with open(CMU_DICT_PATH) as f:
|
||||
line = f.readline()
|
||||
line_index = 1
|
||||
while line:
|
||||
if line_index >= start_line:
|
||||
line = line.strip()
|
||||
word_split = line.split(" ")
|
||||
word = word_split[0]
|
||||
|
||||
syllable_split = word_split[1].split(" - ")
|
||||
g2p_dict[word] = []
|
||||
for syllable in syllable_split:
|
||||
phone_split = syllable.split(" ")
|
||||
g2p_dict[word].append(phone_split)
|
||||
|
||||
line_index = line_index + 1
|
||||
line = f.readline()
|
||||
|
||||
return g2p_dict
|
||||
|
||||
|
||||
def cache_dict(g2p_dict, file_path):
|
||||
with open(file_path, "wb") as pickle_file:
|
||||
pickle.dump(g2p_dict, pickle_file)
|
||||
|
||||
|
||||
def get_dict():
|
||||
if os.path.exists(CACHE_PATH):
|
||||
with open(CACHE_PATH, "rb") as pickle_file:
|
||||
g2p_dict = pickle.load(pickle_file)
|
||||
else:
|
||||
g2p_dict = read_dict()
|
||||
cache_dict(g2p_dict, CACHE_PATH)
|
||||
|
||||
return g2p_dict
|
||||
|
||||
|
||||
eng_dict = get_dict()
|
||||
|
||||
|
||||
def refine_ph(phn):
|
||||
tone = 0
|
||||
if re.search(r"\d$", phn):
|
||||
tone = int(phn[-1]) + 1
|
||||
phn = phn[:-1]
|
||||
return phn.lower(), tone
|
||||
|
||||
|
||||
def refine_syllables(syllables):
|
||||
tones = []
|
||||
phonemes = []
|
||||
for phn_list in syllables:
|
||||
for i in range(len(phn_list)):
|
||||
phn = phn_list[i]
|
||||
phn, tone = refine_ph(phn)
|
||||
phonemes.append(phn)
|
||||
tones.append(tone)
|
||||
return phonemes, tones
|
||||
|
||||
|
||||
def text_normalize(text):
|
||||
# todo: eng text normalize
|
||||
return text
|
||||
|
||||
|
||||
def g2p(text):
|
||||
phones = []
|
||||
tones = []
|
||||
words = re.split(r"([,;.\-\?\!\s+])", text)
|
||||
for w in words:
|
||||
if w.upper() in eng_dict:
|
||||
phns, tns = refine_syllables(eng_dict[w.upper()])
|
||||
phones += phns
|
||||
tones += tns
|
||||
else:
|
||||
phone_list = list(filter(lambda p: p != " ", _g2p(w)))
|
||||
for ph in phone_list:
|
||||
if ph in arpa:
|
||||
ph, tn = refine_ph(ph)
|
||||
phones.append(ph)
|
||||
tones.append(tn)
|
||||
else:
|
||||
phones.append(ph)
|
||||
tones.append(0)
|
||||
# todo: implement word2ph
|
||||
word2ph = [1 for i in phones]
|
||||
|
||||
phones = [post_replace_ph(i) for i in phones]
|
||||
return phones, tones, word2ph
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# print(get_dict())
|
||||
# print(eng_word_to_phoneme("hello"))
|
||||
print(g2p("In this paper, we propose 1 DSPGAN, a GAN-based universal vocoder."))
|
||||
# all_phones = set()
|
||||
# for k, syllables in eng_dict.items():
|
||||
# for group in syllables:
|
||||
# for ph in group:
|
||||
# all_phones.add(ph)
|
||||
# print(all_phones)
|
||||
@@ -1,5 +0,0 @@
|
||||
import torch
|
||||
|
||||
|
||||
def get_bert_feature(norm_text, word2ph):
|
||||
return torch.zeros(1024, sum(word2ph))
|
||||
@@ -1,112 +0,0 @@
|
||||
# modified from https://github.com/CjangCjengh/vits/blob/main/text/japanese.py
|
||||
import re
|
||||
import sys
|
||||
|
||||
import pyopenjtalk
|
||||
|
||||
from . import symbols
|
||||
|
||||
# Regular expression matching Japanese without punctuation marks:
|
||||
_japanese_characters = re.compile(
|
||||
r"[A-Za-z\d\u3005\u3040-\u30ff\u4e00-\u9fff\uff11-\uff19\uff21-\uff3a\uff41-\uff5a\uff66-\uff9d]"
|
||||
)
|
||||
|
||||
# Regular expression matching non-Japanese characters or punctuation marks:
|
||||
_japanese_marks = re.compile(
|
||||
r"[^A-Za-z\d\u3005\u3040-\u30ff\u4e00-\u9fff\uff11-\uff19\uff21-\uff3a\uff41-\uff5a\uff66-\uff9d]"
|
||||
)
|
||||
|
||||
# List of (symbol, Japanese) pairs for marks:
|
||||
_symbols_to_japanese = [(re.compile("%s" % x[0]), x[1]) for x in [("％", "パーセント")]]
|
||||
|
||||
|
||||
# List of (consonant, sokuon) pairs:
|
||||
_real_sokuon = [
|
||||
(re.compile("%s" % x[0]), x[1])
|
||||
for x in [
|
||||
(r"Q([↑↓]*[kg])", r"k#\1"),
|
||||
(r"Q([↑↓]*[tdjʧ])", r"t#\1"),
|
||||
(r"Q([↑↓]*[sʃ])", r"s\1"),
|
||||
(r"Q([↑↓]*[pb])", r"p#\1"),
|
||||
]
|
||||
]
|
||||
|
||||
# List of (consonant, hatsuon) pairs:
|
||||
_real_hatsuon = [
|
||||
(re.compile("%s" % x[0]), x[1])
|
||||
for x in [
|
||||
(r"N([↑↓]*[pbm])", r"m\1"),
|
||||
(r"N([↑↓]*[ʧʥj])", r"n^\1"),
|
||||
(r"N([↑↓]*[tdn])", r"n\1"),
|
||||
(r"N([↑↓]*[kg])", r"ŋ\1"),
|
||||
]
|
||||
]
|
||||
|
||||
|
||||
def post_replace_ph(ph):
|
||||
rep_map = {
|
||||
"：": ",",
|
||||
"；": ",",
|
||||
"，": ",",
|
||||
"。": ".",
|
||||
"！": "!",
|
||||
"？": "?",
|
||||
"\n": ".",
|
||||
"·": ",",
|
||||
"、": ",",
|
||||
"...": "…",
|
||||
"v": "V",
|
||||
}
|
||||
if ph in rep_map.keys():
|
||||
ph = rep_map[ph]
|
||||
if ph in symbols:
|
||||
return ph
|
||||
if ph not in symbols:
|
||||
ph = "UNK"
|
||||
return ph
|
||||
|
||||
|
||||
def symbols_to_japanese(text):
|
||||
for regex, replacement in _symbols_to_japanese:
|
||||
text = re.sub(regex, replacement, text)
|
||||
return text
|
||||
|
||||
|
||||
def preprocess_jap(text):
|
||||
"""Reference https://r9y9.github.io/ttslearn/latest/notebooks/ch10_Recipe-Tacotron.html"""
|
||||
text = symbols_to_japanese(text)
|
||||
sentences = re.split(_japanese_marks, text)
|
||||
marks = re.findall(_japanese_marks, text)
|
||||
text = []
|
||||
for i, sentence in enumerate(sentences):
|
||||
if re.match(_japanese_characters, sentence):
|
||||
p = pyopenjtalk.g2p(sentence)
|
||||
text += p.split(" ")
|
||||
|
||||
if i < len(marks):
|
||||
text += [marks[i].replace(" ", "")]
|
||||
return text
|
||||
|
||||
|
||||
def text_normalize(text):
|
||||
# todo: jap text normalize
|
||||
return text
|
||||
|
||||
|
||||
def g2p(norm_text):
|
||||
phones = preprocess_jap(norm_text)
|
||||
phones = [post_replace_ph(i) for i in phones]
|
||||
# todo: implement tones and word2ph
|
||||
tones = [0 for i in phones]
|
||||
word2ph = [1 for i in phones]
|
||||
return phones, tones, word2ph
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
for line in open("../../../Downloads/transcript_utf8.txt").readlines():
|
||||
text = line.split(":")[1]
|
||||
phones, tones, word2ph = g2p(text)
|
||||
for p in phones:
|
||||
if p == "z":
|
||||
print(text, phones)
|
||||
sys.exit(0)
|
||||
@@ -1,429 +0,0 @@
|
||||
a AA a
|
||||
ai AA ai
|
||||
an AA an
|
||||
ang AA ang
|
||||
ao AA ao
|
||||
ba b a
|
||||
bai b ai
|
||||
ban b an
|
||||
bang b ang
|
||||
bao b ao
|
||||
bei b ei
|
||||
ben b en
|
||||
beng b eng
|
||||
bi b i
|
||||
bian b ian
|
||||
biao b iao
|
||||
bie b ie
|
||||
bin b in
|
||||
bing b ing
|
||||
bo b o
|
||||
bu b u
|
||||
ca c a
|
||||
cai c ai
|
||||
can c an
|
||||
cang c ang
|
||||
cao c ao
|
||||
ce c e
|
||||
cei c ei
|
||||
cen c en
|
||||
ceng c eng
|
||||
cha ch a
|
||||
chai ch ai
|
||||
chan ch an
|
||||
chang ch ang
|
||||
chao ch ao
|
||||
che ch e
|
||||
chen ch en
|
||||
cheng ch eng
|
||||
chi ch ir
|
||||
chong ch ong
|
||||
chou ch ou
|
||||
chu ch u
|
||||
chua ch ua
|
||||
chuai ch uai
|
||||
chuan ch uan
|
||||
chuang ch uang
|
||||
chui ch ui
|
||||
chun ch un
|
||||
chuo ch uo
|
||||
ci c i0
|
||||
cong c ong
|
||||
cou c ou
|
||||
cu c u
|
||||
cuan c uan
|
||||
cui c ui
|
||||
cun c un
|
||||
cuo c uo
|
||||
da d a
|
||||
dai d ai
|
||||
dan d an
|
||||
dang d ang
|
||||
dao d ao
|
||||
de d e
|
||||
dei d ei
|
||||
den d en
|
||||
deng d eng
|
||||
di d i
|
||||
dia d ia
|
||||
dian d ian
|
||||
diao d iao
|
||||
die d ie
|
||||
ding d ing
|
||||
diu d iu
|
||||
dong d ong
|
||||
dou d ou
|
||||
du d u
|
||||
duan d uan
|
||||
dui d ui
|
||||
dun d un
|
||||
duo d uo
|
||||
e EE e
|
||||
ei EE ei
|
||||
en EE en
|
||||
eng EE eng
|
||||
er EE er
|
||||
fa f a
|
||||
fan f an
|
||||
fang f ang
|
||||
fei f ei
|
||||
fen f en
|
||||
feng f eng
|
||||
fo f o
|
||||
fou f ou
|
||||
fu f u
|
||||
ga g a
|
||||
gai g ai
|
||||
gan g an
|
||||
gang g ang
|
||||
gao g ao
|
||||
ge g e
|
||||
gei g ei
|
||||
gen g en
|
||||
geng g eng
|
||||
gong g ong
|
||||
gou g ou
|
||||
gu g u
|
||||
gua g ua
|
||||
guai g uai
|
||||
guan g uan
|
||||
guang g uang
|
||||
gui g ui
|
||||
gun g un
|
||||
guo g uo
|
||||
ha h a
|
||||
hai h ai
|
||||
han h an
|
||||
hang h ang
|
||||
hao h ao
|
||||
he h e
|
||||
hei h ei
|
||||
hen h en
|
||||
heng h eng
|
||||
hong h ong
|
||||
hou h ou
|
||||
hu h u
|
||||
hua h ua
|
||||
huai h uai
|
||||
huan h uan
|
||||
huang h uang
|
||||
hui h ui
|
||||
hun h un
|
||||
huo h uo
|
||||
ji j i
|
||||
jia j ia
|
||||
jian j ian
|
||||
jiang j iang
|
||||
jiao j iao
|
||||
jie j ie
|
||||
jin j in
|
||||
jing j ing
|
||||
jiong j iong
|
||||
jiu j iu
|
||||
ju j v
|
||||
jv j v
|
||||
juan j van
|
||||
jvan j van
|
||||
jue j ve
|
||||
jve j ve
|
||||
jun j vn
|
||||
jvn j vn
|
||||
ka k a
|
||||
kai k ai
|
||||
kan k an
|
||||
kang k ang
|
||||
kao k ao
|
||||
ke k e
|
||||
kei k ei
|
||||
ken k en
|
||||
keng k eng
|
||||
kong k ong
|
||||
kou k ou
|
||||
ku k u
|
||||
kua k ua
|
||||
kuai k uai
|
||||
kuan k uan
|
||||
kuang k uang
|
||||
kui k ui
|
||||
kun k un
|
||||
kuo k uo
|
||||
la l a
|
||||
lai l ai
|
||||
lan l an
|
||||
lang l ang
|
||||
lao l ao
|
||||
le l e
|
||||
lei l ei
|
||||
leng l eng
|
||||
li l i
|
||||
lia l ia
|
||||
lian l ian
|
||||
liang l iang
|
||||
liao l iao
|
||||
lie l ie
|
||||
lin l in
|
||||
ling l ing
|
||||
liu l iu
|
||||
lo l o
|
||||
long l ong
|
||||
lou l ou
|
||||
lu l u
|
||||
luan l uan
|
||||
lun l un
|
||||
luo l uo
|
||||
lv l v
|
||||
lve l ve
|
||||
ma m a
|
||||
mai m ai
|
||||
man m an
|
||||
mang m ang
|
||||
mao m ao
|
||||
me m e
|
||||
mei m ei
|
||||
men m en
|
||||
meng m eng
|
||||
mi m i
|
||||
mian m ian
|
||||
miao m iao
|
||||
mie m ie
|
||||
min m in
|
||||
ming m ing
|
||||
miu m iu
|
||||
mo m o
|
||||
mou m ou
|
||||
mu m u
|
||||
na n a
|
||||
nai n ai
|
||||
nan n an
|
||||
nang n ang
|
||||
nao n ao
|
||||
ne n e
|
||||
nei n ei
|
||||
nen n en
|
||||
neng n eng
|
||||
ni n i
|
||||
nian n ian
|
||||
niang n iang
|
||||
niao n iao
|
||||
nie n ie
|
||||
nin n in
|
||||
ning n ing
|
||||
niu n iu
|
||||
nong n ong
|
||||
nou n ou
|
||||
nu n u
|
||||
nuan n uan
|
||||
nun n un
|
||||
nuo n uo
|
||||
nv n v
|
||||
nve n ve
|
||||
o OO o
|
||||
ou OO ou
|
||||
pa p a
|
||||
pai p ai
|
||||
pan p an
|
||||
pang p ang
|
||||
pao p ao
|
||||
pei p ei
|
||||
pen p en
|
||||
peng p eng
|
||||
pi p i
|
||||
pian p ian
|
||||
piao p iao
|
||||
pie p ie
|
||||
pin p in
|
||||
ping p ing
|
||||
po p o
|
||||
pou p ou
|
||||
pu p u
|
||||
qi q i
|
||||
qia q ia
|
||||
qian q ian
|
||||
qiang q iang
|
||||
qiao q iao
|
||||
qie q ie
|
||||
qin q in
|
||||
qing q ing
|
||||
qiong q iong
|
||||
qiu q iu
|
||||
qu q v
|
||||
qv q v
|
||||
quan q van
|
||||
qvan q van
|
||||
que q ve
|
||||
qve q ve
|
||||
qun q vn
|
||||
qvn q vn
|
||||
ran r an
|
||||
rang r ang
|
||||
rao r ao
|
||||
re r e
|
||||
ren r en
|
||||
reng r eng
|
||||
ri r ir
|
||||
rong r ong
|
||||
rou r ou
|
||||
ru r u
|
||||
rua r ua
|
||||
ruan r uan
|
||||
rui r ui
|
||||
run r un
|
||||
ruo r uo
|
||||
sa s a
|
||||
sai s ai
|
||||
san s an
|
||||
sang s ang
|
||||
sao s ao
|
||||
se s e
|
||||
sen s en
|
||||
seng s eng
|
||||
sha sh a
|
||||
shai sh ai
|
||||
shan sh an
|
||||
shang sh ang
|
||||
shao sh ao
|
||||
she sh e
|
||||
shei sh ei
|
||||
shen sh en
|
||||
sheng sh eng
|
||||
shi sh ir
|
||||
shou sh ou
|
||||
shu sh u
|
||||
shua sh ua
|
||||
shuai sh uai
|
||||
shuan sh uan
|
||||
shuang sh uang
|
||||
shui sh ui
|
||||
shun sh un
|
||||
shuo sh uo
|
||||
si s i0
|
||||
song s ong
|
||||
sou s ou
|
||||
su s u
|
||||
suan s uan
|
||||
sui s ui
|
||||
sun s un
|
||||
suo s uo
|
||||
ta t a
|
||||
tai t ai
|
||||
tan t an
|
||||
tang t ang
|
||||
tao t ao
|
||||
te t e
|
||||
tei t ei
|
||||
teng t eng
|
||||
ti t i
|
||||
tian t ian
|
||||
tiao t iao
|
||||
tie t ie
|
||||
ting t ing
|
||||
tong t ong
|
||||
tou t ou
|
||||
tu t u
|
||||
tuan t uan
|
||||
tui t ui
|
||||
tun t un
|
||||
tuo t uo
|
||||
wa w a
|
||||
wai w ai
|
||||
wan w an
|
||||
wang w ang
|
||||
wei w ei
|
||||
wen w en
|
||||
weng w eng
|
||||
wo w o
|
||||
wu w u
|
||||
xi x i
|
||||
xia x ia
|
||||
xian x ian
|
||||
xiang x iang
|
||||
xiao x iao
|
||||
xie x ie
|
||||
xin x in
|
||||
xing x ing
|
||||
xiong x iong
|
||||
xiu x iu
|
||||
xu x v
|
||||
xv x v
|
||||
xuan x van
|
||||
xvan x van
|
||||
xue x ve
|
||||
xve x ve
|
||||
xun x vn
|
||||
xvn x vn
|
||||
ya y a
|
||||
yan y En
|
||||
yang y ang
|
||||
yao y ao
|
||||
ye y E
|
||||
yi y i
|
||||
yin y in
|
||||
ying y ing
|
||||
yo y o
|
||||
yong y ong
|
||||
you y ou
|
||||
yu y v
|
||||
yv y v
|
||||
yuan y van
|
||||
yvan y van
|
||||
yue y ve
|
||||
yve y ve
|
||||
yun y vn
|
||||
yvn y vn
|
||||
za z a
|
||||
zai z ai
|
||||
zan z an
|
||||
zang z ang
|
||||
zao z ao
|
||||
ze z e
|
||||
zei z ei
|
||||
zen z en
|
||||
zeng z eng
|
||||
zha zh a
|
||||
zhai zh ai
|
||||
zhan zh an
|
||||
zhang zh ang
|
||||
zhao zh ao
|
||||
zhe zh e
|
||||
zhei zh ei
|
||||
zhen zh en
|
||||
zheng zh eng
|
||||
zhi zh ir
|
||||
zhong zh ong
|
||||
zhou zh ou
|
||||
zhu zh u
|
||||
zhua zh ua
|
||||
zhuai zh uai
|
||||
zhuan zh uan
|
||||
zhuang zh uang
|
||||
zhui zh ui
|
||||
zhun zh un
|
||||
zhuo zh uo
|
||||
zi z i0
|
||||
zong z ong
|
||||
zou z ou
|
||||
zu z u
|
||||
zuan z uan
|
||||
zui z ui
|
||||
zun z un
|
||||
zuo z uo
|
||||
@@ -1,183 +0,0 @@
|
||||
punctuation = ["!", "?", "…", ",", ".", "'", "-"]
|
||||
pu_symbols = punctuation + ["SP", "UNK"]
|
||||
pad = "_"
|
||||
|
||||
# chinese
|
||||
zh_symbols = [
|
||||
"E",
|
||||
"En",
|
||||
"a",
|
||||
"ai",
|
||||
"an",
|
||||
"ang",
|
||||
"ao",
|
||||
"b",
|
||||
"c",
|
||||
"ch",
|
||||
"d",
|
||||
"e",
|
||||
"ei",
|
||||
"en",
|
||||
"eng",
|
||||
"er",
|
||||
"f",
|
||||
"g",
|
||||
"h",
|
||||
"i",
|
||||
"i0",
|
||||
"ia",
|
||||
"ian",
|
||||
"iang",
|
||||
"iao",
|
||||
"ie",
|
||||
"in",
|
||||
"ing",
|
||||
"iong",
|
||||
"ir",
|
||||
"iu",
|
||||
"j",
|
||||
"k",
|
||||
"l",
|
||||
"m",
|
||||
"n",
|
||||
"o",
|
||||
"ong",
|
||||
"ou",
|
||||
"p",
|
||||
"q",
|
||||
"r",
|
||||
"s",
|
||||
"sh",
|
||||
"t",
|
||||
"u",
|
||||
"ua",
|
||||
"uai",
|
||||
"uan",
|
||||
"uang",
|
||||
"ui",
|
||||
"un",
|
||||
"uo",
|
||||
"v",
|
||||
"van",
|
||||
"ve",
|
||||
"vn",
|
||||
"w",
|
||||
"x",
|
||||
"y",
|
||||
"z",
|
||||
"zh",
|
||||
"AA",
|
||||
"EE",
|
||||
"OO",
|
||||
]
|
||||
num_zh_tones = 6
|
||||
|
||||
# japanese
|
||||
ja_symbols = [
|
||||
"I",
|
||||
"N",
|
||||
"U",
|
||||
"a",
|
||||
"b",
|
||||
"by",
|
||||
"ch",
|
||||
"cl",
|
||||
"d",
|
||||
"dy",
|
||||
"e",
|
||||
"f",
|
||||
"g",
|
||||
"gy",
|
||||
"h",
|
||||
"hy",
|
||||
"i",
|
||||
"j",
|
||||
"k",
|
||||
"ky",
|
||||
"m",
|
||||
"my",
|
||||
"n",
|
||||
"ny",
|
||||
"o",
|
||||
"p",
|
||||
"py",
|
||||
"r",
|
||||
"ry",
|
||||
"s",
|
||||
"sh",
|
||||
"t",
|
||||
"ts",
|
||||
"u",
|
||||
"V",
|
||||
"w",
|
||||
"y",
|
||||
"z",
|
||||
]
|
||||
num_ja_tones = 1
|
||||
|
||||
# English
|
||||
en_symbols = [
|
||||
"aa",
|
||||
"ae",
|
||||
"ah",
|
||||
"ao",
|
||||
"aw",
|
||||
"ay",
|
||||
"b",
|
||||
"ch",
|
||||
"d",
|
||||
"dh",
|
||||
"eh",
|
||||
"er",
|
||||
"ey",
|
||||
"f",
|
||||
"g",
|
||||
"hh",
|
||||
"ih",
|
||||
"iy",
|
||||
"jh",
|
||||
"k",
|
||||
"l",
|
||||
"m",
|
||||
"n",
|
||||
"ng",
|
||||
"ow",
|
||||
"oy",
|
||||
"p",
|
||||
"r",
|
||||
"s",
|
||||
"sh",
|
||||
"t",
|
||||
"th",
|
||||
"uh",
|
||||
"uw",
|
||||
"V",
|
||||
"w",
|
||||
"y",
|
||||
"z",
|
||||
"zh",
|
||||
]
|
||||
num_en_tones = 4
|
||||
|
||||
# combine all symbols
|
||||
normal_symbols = sorted(set(zh_symbols + ja_symbols + en_symbols))
|
||||
symbols = [pad] + normal_symbols + pu_symbols
|
||||
sil_phonemes_ids = [symbols.index(i) for i in pu_symbols]
|
||||
|
||||
# combine all tones
|
||||
num_tones = num_zh_tones + num_ja_tones + num_en_tones
|
||||
|
||||
# language maps
|
||||
language_id_map = {"ZH": 0, "JA": 1, "EN": 2}
|
||||
num_languages = len(language_id_map.keys())
|
||||
|
||||
language_tone_start_map = {
|
||||
"ZH": 0,
|
||||
"JA": num_zh_tones,
|
||||
"EN": num_zh_tones + num_ja_tones,
|
||||
}
|
||||
|
||||
if __name__ == "__main__":
|
||||
a = set(zh_symbols)
|
||||
b = set(en_symbols)
|
||||
print(sorted(a & b))
|
||||
@@ -1,769 +0,0 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from typing import List
|
||||
from typing import Tuple
|
||||
|
||||
import jieba
|
||||
from pypinyin import lazy_pinyin
|
||||
from pypinyin import Style
|
||||
|
||||
|
||||
class ToneSandhi:
|
||||
def __init__(self):
|
||||
self.must_neural_tone_words = {
|
||||
"麻烦",
|
||||
"麻利",
|
||||
"鸳鸯",
|
||||
"高粱",
|
||||
"骨头",
|
||||
"骆驼",
|
||||
"马虎",
|
||||
"首饰",
|
||||
"馒头",
|
||||
"馄饨",
|
||||
"风筝",
|
||||
"难为",
|
||||
"队伍",
|
||||
"阔气",
|
||||
"闺女",
|
||||
"门道",
|
||||
"锄头",
|
||||
"铺盖",
|
||||
"铃铛",
|
||||
"铁匠",
|
||||
"钥匙",
|
||||
"里脊",
|
||||
"里头",
|
||||
"部分",
|
||||
"那么",
|
||||
"道士",
|
||||
"造化",
|
||||
"迷糊",
|
||||
"连累",
|
||||
"这么",
|
||||
"这个",
|
||||
"运气",
|
||||
"过去",
|
||||
"软和",
|
||||
"转悠",
|
||||
"踏实",
|
||||
"跳蚤",
|
||||
"跟头",
|
||||
"趔趄",
|
||||
"财主",
|
||||
"豆腐",
|
||||
"讲究",
|
||||
"记性",
|
||||
"记号",
|
||||
"认识",
|
||||
"规矩",
|
||||
"见识",
|
||||
"裁缝",
|
||||
"补丁",
|
||||
"衣裳",
|
||||
"衣服",
|
||||
"衙门",
|
||||
"街坊",
|
||||
"行李",
|
||||
"行当",
|
||||
"蛤蟆",
|
||||
"蘑菇",
|
||||
"薄荷",
|
||||
"葫芦",
|
||||
"葡萄",
|
||||
"萝卜",
|
||||
"荸荠",
|
||||
"苗条",
|
||||
"苗头",
|
||||
"苍蝇",
|
||||
"芝麻",
|
||||
"舒服",
|
||||
"舒坦",
|
||||
"舌头",
|
||||
"自在",
|
||||
"膏药",
|
||||
"脾气",
|
||||
"脑袋",
|
||||
"脊梁",
|
||||
"能耐",
|
||||
"胳膊",
|
||||
"胭脂",
|
||||
"胡萝",
|
||||
"胡琴",
|
||||
"胡同",
|
||||
"聪明",
|
||||
"耽误",
|
||||
"耽搁",
|
||||
"耷拉",
|
||||
"耳朵",
|
||||
"老爷",
|
||||
"老实",
|
||||
"老婆",
|
||||
"老头",
|
||||
"老太",
|
||||
"翻腾",
|
||||
"罗嗦",
|
||||
"罐头",
|
||||
"编辑",
|
||||
"结实",
|
||||
"红火",
|
||||
"累赘",
|
||||
"糨糊",
|
||||
"糊涂",
|
||||
"精神",
|
||||
"粮食",
|
||||
"簸箕",
|
||||
"篱笆",
|
||||
"算计",
|
||||
"算盘",
|
||||
"答应",
|
||||
"笤帚",
|
||||
"笑语",
|
||||
"笑话",
|
||||
"窟窿",
|
||||
"窝囊",
|
||||
"窗户",
|
||||
"稳当",
|
||||
"稀罕",
|
||||
"称呼",
|
||||
"秧歌",
|
||||
"秀气",
|
||||
"秀才",
|
||||
"福气",
|
||||
"祖宗",
|
||||
"砚台",
|
||||
"码头",
|
||||
"石榴",
|
||||
"石头",
|
||||
"石匠",
|
||||
"知识",
|
||||
"眼睛",
|
||||
"眯缝",
|
||||
"眨巴",
|
||||
"眉毛",
|
||||
"相声",
|
||||
"盘算",
|
||||
"白净",
|
||||
"痢疾",
|
||||
"痛快",
|
||||
"疟疾",
|
||||
"疙瘩",
|
||||
"疏忽",
|
||||
"畜生",
|
||||
"生意",
|
||||
"甘蔗",
|
||||
"琵琶",
|
||||
"琢磨",
|
||||
"琉璃",
|
||||
"玻璃",
|
||||
"玫瑰",
|
||||
"玄乎",
|
||||
"狐狸",
|
||||
"状元",
|
||||
"特务",
|
||||
"牲口",
|
||||
"牙碜",
|
||||
"牌楼",
|
||||
"爽快",
|
||||
"爱人",
|
||||
"热闹",
|
||||
"烧饼",
|
||||
"烟筒",
|
||||
"烂糊",
|
||||
"点心",
|
||||
"炊帚",
|
||||
"灯笼",
|
||||
"火候",
|
||||
"漂亮",
|
||||
"滑溜",
|
||||
"溜达",
|
||||
"温和",
|
||||
"清楚",
|
||||
"消息",
|
||||
"浪头",
|
||||
"活泼",
|
||||
"比方",
|
||||
"正经",
|
||||
"欺负",
|
||||
"模糊",
|
||||
"槟榔",
|
||||
"棺材",
|
||||
"棒槌",
|
||||
"棉花",
|
||||
"核桃",
|
||||
"栅栏",
|
||||
"柴火",
|
||||
"架势",
|
||||
"枕头",
|
||||
"枇杷",
|
||||
"机灵",
|
||||
"本事",
|
||||
"木头",
|
||||
"木匠",
|
||||
"朋友",
|
||||
"月饼",
|
||||
"月亮",
|
||||
"暖和",
|
||||
"明白",
|
||||
"时候",
|
||||
"新鲜",
|
||||
"故事",
|
||||
"收拾",
|
||||
"收成",
|
||||
"提防",
|
||||
"挖苦",
|
||||
"挑剔",
|
||||
"指甲",
|
||||
"指头",
|
||||
"拾掇",
|
||||
"拳头",
|
||||
"拨弄",
|
||||
"招牌",
|
||||
"招呼",
|
||||
"抬举",
|
||||
"护士",
|
||||
"折腾",
|
||||
"扫帚",
|
||||
"打量",
|
||||
"打算",
|
||||
"打点",
|
||||
"打扮",
|
||||
"打听",
|
||||
"打发",
|
||||
"扎实",
|
||||
"扁担",
|
||||
"戒指",
|
||||
"懒得",
|
||||
"意识",
|
||||
"意思",
|
||||
"情形",
|
||||
"悟性",
|
||||
"怪物",
|
||||
"思量",
|
||||
"怎么",
|
||||
"念头",
|
||||
"念叨",
|
||||
"快活",
|
||||
"忙活",
|
||||
"志气",
|
||||
"心思",
|
||||
"得罪",
|
||||
"张罗",
|
||||
"弟兄",
|
||||
"开通",
|
||||
"应酬",
|
||||
"庄稼",
|
||||
"干事",
|
||||
"帮手",
|
||||
"帐篷",
|
||||
"希罕",
|
||||
"师父",
|
||||
"师傅",
|
||||
"巴结",
|
||||
"巴掌",
|
||||
"差事",
|
||||
"工夫",
|
||||
"岁数",
|
||||
"屁股",
|
||||
"尾巴",
|
||||
"少爷",
|
||||
"小气",
|
||||
"小伙",
|
||||
"将就",
|
||||
"对头",
|
||||
"对付",
|
||||
"寡妇",
|
||||
"家伙",
|
||||
"客气",
|
||||
"实在",
|
||||
"官司",
|
||||
"学问",
|
||||
"学生",
|
||||
"字号",
|
||||
"嫁妆",
|
||||
"媳妇",
|
||||
"媒人",
|
||||
"婆家",
|
||||
"娘家",
|
||||
"委屈",
|
||||
"姑娘",
|
||||
"姐夫",
|
||||
"妯娌",
|
||||
"妥当",
|
||||
"妖精",
|
||||
"奴才",
|
||||
"女婿",
|
||||
"头发",
|
||||
"太阳",
|
||||
"大爷",
|
||||
"大方",
|
||||
"大意",
|
||||
"大夫",
|
||||
"多少",
|
||||
"多么",
|
||||
"外甥",
|
||||
"壮实",
|
||||
"地道",
|
||||
"地方",
|
||||
"在乎",
|
||||
"困难",
|
||||
"嘴巴",
|
||||
"嘱咐",
|
||||
"嘟囔",
|
||||
"嘀咕",
|
||||
"喜欢",
|
||||
"喇嘛",
|
||||
"喇叭",
|
||||
"商量",
|
||||
"唾沫",
|
||||
"哑巴",
|
||||
"哈欠",
|
||||
"哆嗦",
|
||||
"咳嗽",
|
||||
"和尚",
|
||||
"告诉",
|
||||
"告示",
|
||||
"含糊",
|
||||
"吓唬",
|
||||
"后头",
|
||||
"名字",
|
||||
"名堂",
|
||||
"合同",
|
||||
"吆喝",
|
||||
"叫唤",
|
||||
"口袋",
|
||||
"厚道",
|
||||
"厉害",
|
||||
"千斤",
|
||||
"包袱",
|
||||
"包涵",
|
||||
"匀称",
|
||||
"勤快",
|
||||
"动静",
|
||||
"动弹",
|
||||
"功夫",
|
||||
"力气",
|
||||
"前头",
|
||||
"刺猬",
|
||||
"刺激",
|
||||
"别扭",
|
||||
"利落",
|
||||
"利索",
|
||||
"利害",
|
||||
"分析",
|
||||
"出息",
|
||||
"凑合",
|
||||
"凉快",
|
||||
"冷战",
|
||||
"冤枉",
|
||||
"冒失",
|
||||
"养活",
|
||||
"关系",
|
||||
"先生",
|
||||
"兄弟",
|
||||
"便宜",
|
||||
"使唤",
|
||||
"佩服",
|
||||
"作坊",
|
||||
"体面",
|
||||
"位置",
|
||||
"似的",
|
||||
"伙计",
|
||||
"休息",
|
||||
"什么",
|
||||
"人家",
|
||||
"亲戚",
|
||||
"亲家",
|
||||
"交情",
|
||||
"云彩",
|
||||
"事情",
|
||||
"买卖",
|
||||
"主意",
|
||||
"丫头",
|
||||
"丧气",
|
||||
"两口",
|
||||
"东西",
|
||||
"东家",
|
||||
"世故",
|
||||
"不由",
|
||||
"不在",
|
||||
"下水",
|
||||
"下巴",
|
||||
"上头",
|
||||
"上司",
|
||||
"丈夫",
|
||||
"丈人",
|
||||
"一辈",
|
||||
"那个",
|
||||
"菩萨",
|
||||
"父亲",
|
||||
"母亲",
|
||||
"咕噜",
|
||||
"邋遢",
|
||||
"费用",
|
||||
"冤家",
|
||||
"甜头",
|
||||
"介绍",
|
||||
"荒唐",
|
||||
"大人",
|
||||
"泥鳅",
|
||||
"幸福",
|
||||
"熟悉",
|
||||
"计划",
|
||||
"扑腾",
|
||||
"蜡烛",
|
||||
"姥爷",
|
||||
"照顾",
|
||||
"喉咙",
|
||||
"吉他",
|
||||
"弄堂",
|
||||
"蚂蚱",
|
||||
"凤凰",
|
||||
"拖沓",
|
||||
"寒碜",
|
||||
"糟蹋",
|
||||
"倒腾",
|
||||
"报复",
|
||||
"逻辑",
|
||||
"盘缠",
|
||||
"喽啰",
|
||||
"牢骚",
|
||||
"咖喱",
|
||||
"扫把",
|
||||
"惦记",
|
||||
}
|
||||
self.must_not_neural_tone_words = {
|
||||
"男子",
|
||||
"女子",
|
||||
"分子",
|
||||
"原子",
|
||||
"量子",
|
||||
"莲子",
|
||||
"石子",
|
||||
"瓜子",
|
||||
"电子",
|
||||
"人人",
|
||||
"虎虎",
|
||||
}
|
||||
self.punc = "：，；。？！“”‘’':,;.?!"
|
||||
|
||||
# the meaning of jieba pos tag: https://blog.csdn.net/weixin_44174352/article/details/113731041
|
||||
# e.g.
|
||||
# word: "家里"
|
||||
# pos: "s"
|
||||
# finals: ['ia1', 'i3']
|
||||
def _neural_sandhi(self, word: str, pos: str, finals: List[str]) -> List[str]:
|
||||
# reduplication words for n. and v. e.g. 奶奶, 试试, 旺旺
|
||||
for j, item in enumerate(word):
|
||||
if (
|
||||
j - 1 >= 0
|
||||
and item == word[j - 1]
|
||||
and pos[0] in {"n", "v", "a"}
|
||||
and word not in self.must_not_neural_tone_words
|
||||
):
|
||||
finals[j] = finals[j][:-1] + "5"
|
||||
ge_idx = word.find("个")
|
||||
if len(word) >= 1 and word[-1] in "吧呢啊呐噻嘛吖嗨呐哦哒额滴哩哟喽啰耶喔诶":
|
||||
finals[-1] = finals[-1][:-1] + "5"
|
||||
elif len(word) >= 1 and word[-1] in "的地得":
|
||||
finals[-1] = finals[-1][:-1] + "5"
|
||||
# e.g. 走了, 看着, 去过
|
||||
# elif len(word) == 1 and word in "了着过" and pos in {"ul", "uz", "ug"}:
|
||||
# finals[-1] = finals[-1][:-1] + "5"
|
||||
elif (
|
||||
len(word) > 1
|
||||
and word[-1] in "们子"
|
||||
and pos in {"r", "n"}
|
||||
and word not in self.must_not_neural_tone_words
|
||||
):
|
||||
finals[-1] = finals[-1][:-1] + "5"
|
||||
# e.g. 桌上, 地下, 家里
|
||||
elif len(word) > 1 and word[-1] in "上下里" and pos in {"s", "l", "f"}:
|
||||
finals[-1] = finals[-1][:-1] + "5"
|
||||
# e.g. 上来, 下去
|
||||
elif len(word) > 1 and word[-1] in "来去" and word[-2] in "上下进出回过起开":
|
||||
finals[-1] = finals[-1][:-1] + "5"
|
||||
# 个做量词
|
||||
elif (
|
||||
ge_idx >= 1
|
||||
and (word[ge_idx - 1].isnumeric() or word[ge_idx - 1] in "几有两半多各整每做是")
|
||||
) or word == "个":
|
||||
finals[ge_idx] = finals[ge_idx][:-1] + "5"
|
||||
else:
|
||||
if (
|
||||
word in self.must_neural_tone_words
|
||||
or word[-2:] in self.must_neural_tone_words
|
||||
):
|
||||
finals[-1] = finals[-1][:-1] + "5"
|
||||
|
||||
word_list = self._split_word(word)
|
||||
finals_list = [finals[: len(word_list[0])], finals[len(word_list[0]) :]]
|
||||
for i, word in enumerate(word_list):
|
||||
# conventional neural in Chinese
|
||||
if (
|
||||
word in self.must_neural_tone_words
|
||||
or word[-2:] in self.must_neural_tone_words
|
||||
):
|
||||
finals_list[i][-1] = finals_list[i][-1][:-1] + "5"
|
||||
finals = sum(finals_list, [])
|
||||
return finals
|
||||
|
||||
def _bu_sandhi(self, word: str, finals: List[str]) -> List[str]:
|
||||
# e.g. 看不懂
|
||||
if len(word) == 3 and word[1] == "不":
|
||||
finals[1] = finals[1][:-1] + "5"
|
||||
else:
|
||||
for i, char in enumerate(word):
|
||||
# "不" before tone4 should be bu2, e.g. 不怕
|
||||
if char == "不" and i + 1 < len(word) and finals[i + 1][-1] == "4":
|
||||
finals[i] = finals[i][:-1] + "2"
|
||||
return finals
|
||||
|
||||
def _yi_sandhi(self, word: str, finals: List[str]) -> List[str]:
|
||||
# "一" in number sequences, e.g. 一零零, 二一零
|
||||
if word.find("一") != -1 and all(
|
||||
[item.isnumeric() for item in word if item != "一"]
|
||||
):
|
||||
return finals
|
||||
# "一" between reduplication words shold be yi5, e.g. 看一看
|
||||
elif len(word) == 3 and word[1] == "一" and word[0] == word[-1]:
|
||||
finals[1] = finals[1][:-1] + "5"
|
||||
# when "一" is ordinal word, it should be yi1
|
||||
elif word.startswith("第一"):
|
||||
finals[1] = finals[1][:-1] + "1"
|
||||
else:
|
||||
for i, char in enumerate(word):
|
||||
if char == "一" and i + 1 < len(word):
|
||||
# "一" before tone4 should be yi2, e.g. 一段
|
||||
if finals[i + 1][-1] == "4":
|
||||
finals[i] = finals[i][:-1] + "2"
|
||||
# "一" before non-tone4 should be yi4, e.g. 一天
|
||||
else:
|
||||
# "一" 后面如果是标点，还读一声
|
||||
if word[i + 1] not in self.punc:
|
||||
finals[i] = finals[i][:-1] + "4"
|
||||
return finals
|
||||
|
||||
def _split_word(self, word: str) -> List[str]:
|
||||
word_list = jieba.cut_for_search(word)
|
||||
word_list = sorted(word_list, key=lambda i: len(i), reverse=False)
|
||||
first_subword = word_list[0]
|
||||
first_begin_idx = word.find(first_subword)
|
||||
if first_begin_idx == 0:
|
||||
second_subword = word[len(first_subword) :]
|
||||
new_word_list = [first_subword, second_subword]
|
||||
else:
|
||||
second_subword = word[: -len(first_subword)]
|
||||
new_word_list = [second_subword, first_subword]
|
||||
return new_word_list
|
||||
|
||||
def _three_sandhi(self, word: str, finals: List[str]) -> List[str]:
|
||||
if len(word) == 2 and self._all_tone_three(finals):
|
||||
finals[0] = finals[0][:-1] + "2"
|
||||
elif len(word) == 3:
|
||||
word_list = self._split_word(word)
|
||||
if self._all_tone_three(finals):
|
||||
# disyllabic + monosyllabic, e.g. 蒙古/包
|
||||
if len(word_list[0]) == 2:
|
||||
finals[0] = finals[0][:-1] + "2"
|
||||
finals[1] = finals[1][:-1] + "2"
|
||||
# monosyllabic + disyllabic, e.g. 纸/老虎
|
||||
elif len(word_list[0]) == 1:
|
||||
finals[1] = finals[1][:-1] + "2"
|
||||
else:
|
||||
finals_list = [finals[: len(word_list[0])], finals[len(word_list[0]) :]]
|
||||
if len(finals_list) == 2:
|
||||
for i, sub in enumerate(finals_list):
|
||||
# e.g. 所有/人
|
||||
if self._all_tone_three(sub) and len(sub) == 2:
|
||||
finals_list[i][0] = finals_list[i][0][:-1] + "2"
|
||||
# e.g. 好/喜欢
|
||||
elif (
|
||||
i == 1
|
||||
and not self._all_tone_three(sub)
|
||||
and finals_list[i][0][-1] == "3"
|
||||
and finals_list[0][-1][-1] == "3"
|
||||
):
|
||||
finals_list[0][-1] = finals_list[0][-1][:-1] + "2"
|
||||
finals = sum(finals_list, [])
|
||||
# split idiom into two words who's length is 2
|
||||
elif len(word) == 4:
|
||||
finals_list = [finals[:2], finals[2:]]
|
||||
finals = []
|
||||
for sub in finals_list:
|
||||
if self._all_tone_three(sub):
|
||||
sub[0] = sub[0][:-1] + "2"
|
||||
finals += sub
|
||||
|
||||
return finals
|
||||
|
||||
def _all_tone_three(self, finals: List[str]) -> bool:
|
||||
return all(x[-1] == "3" for x in finals)
|
||||
|
||||
# merge "不" and the word behind it
|
||||
# if don't merge, "不" sometimes appears alone according to jieba, which may occur sandhi error
|
||||
def _merge_bu(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
|
||||
new_seg = []
|
||||
last_word = ""
|
||||
for word, pos in seg:
|
||||
if last_word == "不":
|
||||
word = last_word + word
|
||||
if word != "不":
|
||||
new_seg.append((word, pos))
|
||||
last_word = word[:]
|
||||
if last_word == "不":
|
||||
new_seg.append((last_word, "d"))
|
||||
last_word = ""
|
||||
return new_seg
|
||||
|
||||
# function 1: merge "一" and reduplication words in it's left and right, e.g. "听","一","听" ->"听一听"
|
||||
# function 2: merge single "一" and the word behind it
|
||||
# if don't merge, "一" sometimes appears alone according to jieba, which may occur sandhi error
|
||||
# e.g.
|
||||
# input seg: [('听', 'v'), ('一', 'm'), ('听', 'v')]
|
||||
# output seg: [['听一听', 'v']]
|
||||
def _merge_yi(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
|
||||
new_seg = []
|
||||
# function 1
|
||||
for i, (word, pos) in enumerate(seg):
|
||||
if (
|
||||
i - 1 >= 0
|
||||
and word == "一"
|
||||
and i + 1 < len(seg)
|
||||
and seg[i - 1][0] == seg[i + 1][0]
|
||||
and seg[i - 1][1] == "v"
|
||||
):
|
||||
new_seg[i - 1][0] = new_seg[i - 1][0] + "一" + new_seg[i - 1][0]
|
||||
else:
|
||||
if (
|
||||
i - 2 >= 0
|
||||
and seg[i - 1][0] == "一"
|
||||
and seg[i - 2][0] == word
|
||||
and pos == "v"
|
||||
):
|
||||
continue
|
||||
else:
|
||||
new_seg.append([word, pos])
|
||||
seg = new_seg
|
||||
new_seg = []
|
||||
# function 2
|
||||
for i, (word, pos) in enumerate(seg):
|
||||
if new_seg and new_seg[-1][0] == "一":
|
||||
new_seg[-1][0] = new_seg[-1][0] + word
|
||||
else:
|
||||
new_seg.append([word, pos])
|
||||
return new_seg
|
||||
|
||||
# the first and the second words are all_tone_three
|
||||
def _merge_continuous_three_tones(
|
||||
self, seg: List[Tuple[str, str]]
|
||||
) -> List[Tuple[str, str]]:
|
||||
new_seg = []
|
||||
sub_finals_list = [
|
||||
lazy_pinyin(word, neutral_tone_with_five=True, style=Style.FINALS_TONE3)
|
||||
for (word, pos) in seg
|
||||
]
|
||||
assert len(sub_finals_list) == len(seg)
|
||||
merge_last = [False] * len(seg)
|
||||
for i, (word, pos) in enumerate(seg):
|
||||
if (
|
||||
i - 1 >= 0
|
||||
and self._all_tone_three(sub_finals_list[i - 1])
|
||||
and self._all_tone_three(sub_finals_list[i])
|
||||
and not merge_last[i - 1]
|
||||
):
|
||||
# if the last word is reduplication, not merge, because reduplication need to be _neural_sandhi
|
||||
if (
|
||||
not self._is_reduplication(seg[i - 1][0])
|
||||
and len(seg[i - 1][0]) + len(seg[i][0]) <= 3
|
||||
):
|
||||
new_seg[-1][0] = new_seg[-1][0] + seg[i][0]
|
||||
merge_last[i] = True
|
||||
else:
|
||||
new_seg.append([word, pos])
|
||||
else:
|
||||
new_seg.append([word, pos])
|
||||
|
||||
return new_seg
|
||||
|
||||
def _is_reduplication(self, word: str) -> bool:
|
||||
return len(word) == 2 and word[0] == word[1]
|
||||
|
||||
# the last char of first word and the first char of second word is tone_three
|
||||
def _merge_continuous_three_tones_2(
|
||||
self, seg: List[Tuple[str, str]]
|
||||
) -> List[Tuple[str, str]]:
|
||||
new_seg = []
|
||||
sub_finals_list = [
|
||||
lazy_pinyin(word, neutral_tone_with_five=True, style=Style.FINALS_TONE3)
|
||||
for (word, pos) in seg
|
||||
]
|
||||
assert len(sub_finals_list) == len(seg)
|
||||
merge_last = [False] * len(seg)
|
||||
for i, (word, pos) in enumerate(seg):
|
||||
if (
|
||||
i - 1 >= 0
|
||||
and sub_finals_list[i - 1][-1][-1] == "3"
|
||||
and sub_finals_list[i][0][-1] == "3"
|
||||
and not merge_last[i - 1]
|
||||
):
|
||||
# if the last word is reduplication, not merge, because reduplication need to be _neural_sandhi
|
||||
if (
|
||||
not self._is_reduplication(seg[i - 1][0])
|
||||
and len(seg[i - 1][0]) + len(seg[i][0]) <= 3
|
||||
):
|
||||
new_seg[-1][0] = new_seg[-1][0] + seg[i][0]
|
||||
merge_last[i] = True
|
||||
else:
|
||||
new_seg.append([word, pos])
|
||||
else:
|
||||
new_seg.append([word, pos])
|
||||
return new_seg
|
||||
|
||||
def _merge_er(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
|
||||
new_seg = []
|
||||
for i, (word, pos) in enumerate(seg):
|
||||
if i - 1 >= 0 and word == "儿" and seg[i - 1][0] != "#":
|
||||
new_seg[-1][0] = new_seg[-1][0] + seg[i][0]
|
||||
else:
|
||||
new_seg.append([word, pos])
|
||||
return new_seg
|
||||
|
||||
def _merge_reduplication(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
|
||||
new_seg = []
|
||||
for i, (word, pos) in enumerate(seg):
|
||||
if new_seg and word == new_seg[-1][0]:
|
||||
new_seg[-1][0] = new_seg[-1][0] + seg[i][0]
|
||||
else:
|
||||
new_seg.append([word, pos])
|
||||
return new_seg
|
||||
|
||||
def pre_merge_for_modify(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
|
||||
seg = self._merge_bu(seg)
|
||||
try:
|
||||
seg = self._merge_yi(seg)
|
||||
except:
|
||||
print("_merge_yi failed")
|
||||
seg = self._merge_reduplication(seg)
|
||||
seg = self._merge_continuous_three_tones(seg)
|
||||
seg = self._merge_continuous_three_tones_2(seg)
|
||||
seg = self._merge_er(seg)
|
||||
return seg
|
||||
|
||||
def modified_tone(self, word: str, pos: str, finals: List[str]) -> List[str]:
|
||||
finals = self._bu_sandhi(word, finals)
|
||||
finals = self._yi_sandhi(word, finals)
|
||||
finals = self._neural_sandhi(word, pos, finals)
|
||||
finals = self._three_sandhi(word, finals)
|
||||
return finals
|
||||
@@ -1,90 +0,0 @@
|
||||
"""
|
||||
1.1 版本兼容
|
||||
https://github.com/fishaudio/Bert-VITS2/releases/tag/1.1
|
||||
"""
|
||||
import torch
|
||||
import commons
|
||||
from .text.cleaner import clean_text
|
||||
from .text import cleaned_text_to_sequence
|
||||
from oldVersion.V111.text import get_bert
|
||||
|
||||
|
||||
def get_text(text, language_str, hps, device):
|
||||
norm_text, phone, tone, word2ph = clean_text(text, language_str)
|
||||
phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
|
||||
|
||||
if hps.data.add_blank:
|
||||
phone = commons.intersperse(phone, 0)
|
||||
tone = commons.intersperse(tone, 0)
|
||||
language = commons.intersperse(language, 0)
|
||||
for i in range(len(word2ph)):
|
||||
word2ph[i] = word2ph[i] * 2
|
||||
word2ph[0] += 1
|
||||
bert = get_bert(norm_text, word2ph, language_str, device)
|
||||
del word2ph
|
||||
assert bert.shape[-1] == len(phone), phone
|
||||
|
||||
if language_str == "ZH":
|
||||
bert = bert
|
||||
ja_bert = torch.zeros(768, len(phone))
|
||||
elif language_str == "JP":
|
||||
ja_bert = bert
|
||||
bert = torch.zeros(1024, len(phone))
|
||||
else:
|
||||
bert = torch.zeros(1024, len(phone))
|
||||
ja_bert = torch.zeros(768, len(phone))
|
||||
|
||||
assert bert.shape[-1] == len(
|
||||
phone
|
||||
), f"Bert seq len {bert.shape[-1]} != {len(phone)}"
|
||||
|
||||
phone = torch.LongTensor(phone)
|
||||
tone = torch.LongTensor(tone)
|
||||
language = torch.LongTensor(language)
|
||||
return bert, ja_bert, phone, tone, language
|
||||
|
||||
|
||||
def infer(
|
||||
text,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
sid,
|
||||
language,
|
||||
hps,
|
||||
net_g,
|
||||
device,
|
||||
):
|
||||
bert, ja_bert, phones, tones, lang_ids = get_text(text, language, hps, device)
|
||||
with torch.no_grad():
|
||||
x_tst = phones.to(device).unsqueeze(0)
|
||||
tones = tones.to(device).unsqueeze(0)
|
||||
lang_ids = lang_ids.to(device).unsqueeze(0)
|
||||
bert = bert.to(device).unsqueeze(0)
|
||||
ja_bert = ja_bert.to(device).unsqueeze(0)
|
||||
x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
|
||||
del phones
|
||||
speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
|
||||
audio = (
|
||||
net_g.infer(
|
||||
x_tst,
|
||||
x_tst_lengths,
|
||||
speakers,
|
||||
tones,
|
||||
lang_ids,
|
||||
bert,
|
||||
ja_bert,
|
||||
sdp_ratio=sdp_ratio,
|
||||
noise_scale=noise_scale,
|
||||
noise_scale_w=noise_scale_w,
|
||||
length_scale=length_scale,
|
||||
)[0][0, 0]
|
||||
.data.cpu()
|
||||
.float()
|
||||
.numpy()
|
||||
)
|
||||
del x_tst, x_tst_lengths, speakers, tones, lang_ids, bert, ja_bert
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
return audio
|
||||
@@ -1,986 +0,0 @@
|
||||
import math
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.nn import functional as F
|
||||
|
||||
import commons
|
||||
import modules
|
||||
import attentions
|
||||
import monotonic_align
|
||||
|
||||
from torch.nn import Conv1d, ConvTranspose1d, Conv2d
|
||||
from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm
|
||||
|
||||
from commons import init_weights, get_padding
|
||||
from .text import symbols, num_tones, num_languages
|
||||
|
||||
|
||||
class DurationDiscriminator(nn.Module): # vits2
|
||||
def __init__(
|
||||
self, in_channels, filter_channels, kernel_size, p_dropout, gin_channels=0
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self.in_channels = in_channels
|
||||
self.filter_channels = filter_channels
|
||||
self.kernel_size = kernel_size
|
||||
self.p_dropout = p_dropout
|
||||
self.gin_channels = gin_channels
|
||||
|
||||
self.drop = nn.Dropout(p_dropout)
|
||||
self.conv_1 = nn.Conv1d(
|
||||
in_channels, filter_channels, kernel_size, padding=kernel_size // 2
|
||||
)
|
||||
self.norm_1 = modules.LayerNorm(filter_channels)
|
||||
self.conv_2 = nn.Conv1d(
|
||||
filter_channels, filter_channels, kernel_size, padding=kernel_size // 2
|
||||
)
|
||||
self.norm_2 = modules.LayerNorm(filter_channels)
|
||||
self.dur_proj = nn.Conv1d(1, filter_channels, 1)
|
||||
|
||||
self.pre_out_conv_1 = nn.Conv1d(
|
||||
2 * filter_channels, filter_channels, kernel_size, padding=kernel_size // 2
|
||||
)
|
||||
self.pre_out_norm_1 = modules.LayerNorm(filter_channels)
|
||||
self.pre_out_conv_2 = nn.Conv1d(
|
||||
filter_channels, filter_channels, kernel_size, padding=kernel_size // 2
|
||||
)
|
||||
self.pre_out_norm_2 = modules.LayerNorm(filter_channels)
|
||||
|
||||
if gin_channels != 0:
|
||||
self.cond = nn.Conv1d(gin_channels, in_channels, 1)
|
||||
|
||||
self.output_layer = nn.Sequential(nn.Linear(filter_channels, 1), nn.Sigmoid())
|
||||
|
||||
def forward_probability(self, x, x_mask, dur, g=None):
|
||||
dur = self.dur_proj(dur)
|
||||
x = torch.cat([x, dur], dim=1)
|
||||
x = self.pre_out_conv_1(x * x_mask)
|
||||
x = torch.relu(x)
|
||||
x = self.pre_out_norm_1(x)
|
||||
x = self.drop(x)
|
||||
x = self.pre_out_conv_2(x * x_mask)
|
||||
x = torch.relu(x)
|
||||
x = self.pre_out_norm_2(x)
|
||||
x = self.drop(x)
|
||||
x = x * x_mask
|
||||
x = x.transpose(1, 2)
|
||||
output_prob = self.output_layer(x)
|
||||
return output_prob
|
||||
|
||||
def forward(self, x, x_mask, dur_r, dur_hat, g=None):
|
||||
x = torch.detach(x)
|
||||
if g is not None:
|
||||
g = torch.detach(g)
|
||||
x = x + self.cond(g)
|
||||
x = self.conv_1(x * x_mask)
|
||||
x = torch.relu(x)
|
||||
x = self.norm_1(x)
|
||||
x = self.drop(x)
|
||||
x = self.conv_2(x * x_mask)
|
||||
x = torch.relu(x)
|
||||
x = self.norm_2(x)
|
||||
x = self.drop(x)
|
||||
|
||||
output_probs = []
|
||||
for dur in [dur_r, dur_hat]:
|
||||
output_prob = self.forward_probability(x, x_mask, dur, g)
|
||||
output_probs.append(output_prob)
|
||||
|
||||
return output_probs
|
||||
|
||||
|
||||
class TransformerCouplingBlock(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
channels,
|
||||
hidden_channels,
|
||||
filter_channels,
|
||||
n_heads,
|
||||
n_layers,
|
||||
kernel_size,
|
||||
p_dropout,
|
||||
n_flows=4,
|
||||
gin_channels=0,
|
||||
share_parameter=False,
|
||||
):
|
||||
super().__init__()
|
||||
self.channels = channels
|
||||
self.hidden_channels = hidden_channels
|
||||
self.kernel_size = kernel_size
|
||||
self.n_layers = n_layers
|
||||
self.n_flows = n_flows
|
||||
self.gin_channels = gin_channels
|
||||
|
||||
self.flows = nn.ModuleList()
|
||||
|
||||
self.wn = (
|
||||
attentions.FFT(
|
||||
hidden_channels,
|
||||
filter_channels,
|
||||
n_heads,
|
||||
n_layers,
|
||||
kernel_size,
|
||||
p_dropout,
|
||||
isflow=True,
|
||||
gin_channels=self.gin_channels,
|
||||
)
|
||||
if share_parameter
|
||||
else None
|
||||
)
|
||||
|
||||
for i in range(n_flows):
|
||||
self.flows.append(
|
||||
modules.TransformerCouplingLayer(
|
||||
channels,
|
||||
hidden_channels,
|
||||
kernel_size,
|
||||
n_layers,
|
||||
n_heads,
|
||||
p_dropout,
|
||||
filter_channels,
|
||||
mean_only=True,
|
||||
wn_sharing_parameter=self.wn,
|
||||
gin_channels=self.gin_channels,
|
||||
)
|
||||
)
|
||||
self.flows.append(modules.Flip())
|
||||
|
||||
def forward(self, x, x_mask, g=None, reverse=False):
|
||||
if not reverse:
|
||||
for flow in self.flows:
|
||||
x, _ = flow(x, x_mask, g=g, reverse=reverse)
|
||||
else:
|
||||
for flow in reversed(self.flows):
|
||||
x = flow(x, x_mask, g=g, reverse=reverse)
|
||||
return x
|
||||
|
||||
|
||||
class StochasticDurationPredictor(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
in_channels,
|
||||
filter_channels,
|
||||
kernel_size,
|
||||
p_dropout,
|
||||
n_flows=4,
|
||||
gin_channels=0,
|
||||
):
|
||||
super().__init__()
|
||||
filter_channels = in_channels # it needs to be removed from future version.
|
||||
self.in_channels = in_channels
|
||||
self.filter_channels = filter_channels
|
||||
self.kernel_size = kernel_size
|
||||
self.p_dropout = p_dropout
|
||||
self.n_flows = n_flows
|
||||
self.gin_channels = gin_channels
|
||||
|
||||
self.log_flow = modules.Log()
|
||||
self.flows = nn.ModuleList()
|
||||
self.flows.append(modules.ElementwiseAffine(2))
|
||||
for i in range(n_flows):
|
||||
self.flows.append(
|
||||
modules.ConvFlow(2, filter_channels, kernel_size, n_layers=3)
|
||||
)
|
||||
self.flows.append(modules.Flip())
|
||||
|
||||
self.post_pre = nn.Conv1d(1, filter_channels, 1)
|
||||
self.post_proj = nn.Conv1d(filter_channels, filter_channels, 1)
|
||||
self.post_convs = modules.DDSConv(
|
||||
filter_channels, kernel_size, n_layers=3, p_dropout=p_dropout
|
||||
)
|
||||
self.post_flows = nn.ModuleList()
|
||||
self.post_flows.append(modules.ElementwiseAffine(2))
|
||||
for i in range(4):
|
||||
self.post_flows.append(
|
||||
modules.ConvFlow(2, filter_channels, kernel_size, n_layers=3)
|
||||
)
|
||||
self.post_flows.append(modules.Flip())
|
||||
|
||||
self.pre = nn.Conv1d(in_channels, filter_channels, 1)
|
||||
self.proj = nn.Conv1d(filter_channels, filter_channels, 1)
|
||||
self.convs = modules.DDSConv(
|
||||
filter_channels, kernel_size, n_layers=3, p_dropout=p_dropout
|
||||
)
|
||||
if gin_channels != 0:
|
||||
self.cond = nn.Conv1d(gin_channels, filter_channels, 1)
|
||||
|
||||
def forward(self, x, x_mask, w=None, g=None, reverse=False, noise_scale=1.0):
|
||||
x = torch.detach(x)
|
||||
x = self.pre(x)
|
||||
if g is not None:
|
||||
g = torch.detach(g)
|
||||
x = x + self.cond(g)
|
||||
x = self.convs(x, x_mask)
|
||||
x = self.proj(x) * x_mask
|
||||
|
||||
if not reverse:
|
||||
flows = self.flows
|
||||
assert w is not None
|
||||
|
||||
logdet_tot_q = 0
|
||||
h_w = self.post_pre(w)
|
||||
h_w = self.post_convs(h_w, x_mask)
|
||||
h_w = self.post_proj(h_w) * x_mask
|
||||
e_q = (
|
||||
torch.randn(w.size(0), 2, w.size(2)).to(device=x.device, dtype=x.dtype)
|
||||
* x_mask
|
||||
)
|
||||
z_q = e_q
|
||||
for flow in self.post_flows:
|
||||
z_q, logdet_q = flow(z_q, x_mask, g=(x + h_w))
|
||||
logdet_tot_q += logdet_q
|
||||
z_u, z1 = torch.split(z_q, [1, 1], 1)
|
||||
u = torch.sigmoid(z_u) * x_mask
|
||||
z0 = (w - u) * x_mask
|
||||
logdet_tot_q += torch.sum(
|
||||
(F.logsigmoid(z_u) + F.logsigmoid(-z_u)) * x_mask, [1, 2]
|
||||
)
|
||||
logq = (
|
||||
torch.sum(-0.5 * (math.log(2 * math.pi) + (e_q**2)) * x_mask, [1, 2])
|
||||
- logdet_tot_q
|
||||
)
|
||||
|
||||
logdet_tot = 0
|
||||
z0, logdet = self.log_flow(z0, x_mask)
|
||||
logdet_tot += logdet
|
||||
z = torch.cat([z0, z1], 1)
|
||||
for flow in flows:
|
||||
z, logdet = flow(z, x_mask, g=x, reverse=reverse)
|
||||
logdet_tot = logdet_tot + logdet
|
||||
nll = (
|
||||
torch.sum(0.5 * (math.log(2 * math.pi) + (z**2)) * x_mask, [1, 2])
|
||||
- logdet_tot
|
||||
)
|
||||
return nll + logq # [b]
|
||||
else:
|
||||
flows = list(reversed(self.flows))
|
||||
flows = flows[:-2] + [flows[-1]] # remove a useless vflow
|
||||
z = (
|
||||
torch.randn(x.size(0), 2, x.size(2)).to(device=x.device, dtype=x.dtype)
|
||||
* noise_scale
|
||||
)
|
||||
for flow in flows:
|
||||
z = flow(z, x_mask, g=x, reverse=reverse)
|
||||
z0, z1 = torch.split(z, [1, 1], 1)
|
||||
logw = z0
|
||||
return logw
|
||||
|
||||
|
||||
class DurationPredictor(nn.Module):
|
||||
def __init__(
|
||||
self, in_channels, filter_channels, kernel_size, p_dropout, gin_channels=0
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self.in_channels = in_channels
|
||||
self.filter_channels = filter_channels
|
||||
self.kernel_size = kernel_size
|
||||
self.p_dropout = p_dropout
|
||||
self.gin_channels = gin_channels
|
||||
|
||||
self.drop = nn.Dropout(p_dropout)
|
||||
self.conv_1 = nn.Conv1d(
|
||||
in_channels, filter_channels, kernel_size, padding=kernel_size // 2
|
||||
)
|
||||
self.norm_1 = modules.LayerNorm(filter_channels)
|
||||
self.conv_2 = nn.Conv1d(
|
||||
filter_channels, filter_channels, kernel_size, padding=kernel_size // 2
|
||||
)
|
||||
self.norm_2 = modules.LayerNorm(filter_channels)
|
||||
self.proj = nn.Conv1d(filter_channels, 1, 1)
|
||||
|
||||
if gin_channels != 0:
|
||||
self.cond = nn.Conv1d(gin_channels, in_channels, 1)
|
||||
|
||||
def forward(self, x, x_mask, g=None):
|
||||
x = torch.detach(x)
|
||||
if g is not None:
|
||||
g = torch.detach(g)
|
||||
x = x + self.cond(g)
|
||||
x = self.conv_1(x * x_mask)
|
||||
x = torch.relu(x)
|
||||
x = self.norm_1(x)
|
||||
x = self.drop(x)
|
||||
x = self.conv_2(x * x_mask)
|
||||
x = torch.relu(x)
|
||||
x = self.norm_2(x)
|
||||
x = self.drop(x)
|
||||
x = self.proj(x * x_mask)
|
||||
return x * x_mask
|
||||
|
||||
|
||||
class TextEncoder(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
n_vocab,
|
||||
out_channels,
|
||||
hidden_channels,
|
||||
filter_channels,
|
||||
n_heads,
|
||||
n_layers,
|
||||
kernel_size,
|
||||
p_dropout,
|
||||
gin_channels=0,
|
||||
):
|
||||
super().__init__()
|
||||
self.n_vocab = n_vocab
|
||||
self.out_channels = out_channels
|
||||
self.hidden_channels = hidden_channels
|
||||
self.filter_channels = filter_channels
|
||||
self.n_heads = n_heads
|
||||
self.n_layers = n_layers
|
||||
self.kernel_size = kernel_size
|
||||
self.p_dropout = p_dropout
|
||||
self.gin_channels = gin_channels
|
||||
self.emb = nn.Embedding(len(symbols), hidden_channels)
|
||||
nn.init.normal_(self.emb.weight, 0.0, hidden_channels**-0.5)
|
||||
self.tone_emb = nn.Embedding(num_tones, hidden_channels)
|
||||
nn.init.normal_(self.tone_emb.weight, 0.0, hidden_channels**-0.5)
|
||||
self.language_emb = nn.Embedding(num_languages, hidden_channels)
|
||||
nn.init.normal_(self.language_emb.weight, 0.0, hidden_channels**-0.5)
|
||||
self.bert_proj = nn.Conv1d(1024, hidden_channels, 1)
|
||||
self.ja_bert_proj = nn.Conv1d(768, hidden_channels, 1)
|
||||
|
||||
self.encoder = attentions.Encoder(
|
||||
hidden_channels,
|
||||
filter_channels,
|
||||
n_heads,
|
||||
n_layers,
|
||||
kernel_size,
|
||||
p_dropout,
|
||||
gin_channels=self.gin_channels,
|
||||
)
|
||||
self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
|
||||
|
||||
def forward(self, x, x_lengths, tone, language, bert, ja_bert, g=None):
|
||||
bert_emb = self.bert_proj(bert).transpose(1, 2)
|
||||
ja_bert_emb = self.ja_bert_proj(ja_bert).transpose(1, 2)
|
||||
x = (
|
||||
self.emb(x)
|
||||
+ self.tone_emb(tone)
|
||||
+ self.language_emb(language)
|
||||
+ bert_emb
|
||||
+ ja_bert_emb
|
||||
) * math.sqrt(
|
||||
self.hidden_channels
|
||||
) # [b, t, h]
|
||||
x = torch.transpose(x, 1, -1) # [b, h, t]
|
||||
x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(
|
||||
x.dtype
|
||||
)
|
||||
|
||||
x = self.encoder(x * x_mask, x_mask, g=g)
|
||||
stats = self.proj(x) * x_mask
|
||||
|
||||
m, logs = torch.split(stats, self.out_channels, dim=1)
|
||||
return x, m, logs, x_mask
|
||||
|
||||
|
||||
class ResidualCouplingBlock(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
channels,
|
||||
hidden_channels,
|
||||
kernel_size,
|
||||
dilation_rate,
|
||||
n_layers,
|
||||
n_flows=4,
|
||||
gin_channels=0,
|
||||
):
|
||||
super().__init__()
|
||||
self.channels = channels
|
||||
self.hidden_channels = hidden_channels
|
||||
self.kernel_size = kernel_size
|
||||
self.dilation_rate = dilation_rate
|
||||
self.n_layers = n_layers
|
||||
self.n_flows = n_flows
|
||||
self.gin_channels = gin_channels
|
||||
|
||||
self.flows = nn.ModuleList()
|
||||
for i in range(n_flows):
|
||||
self.flows.append(
|
||||
modules.ResidualCouplingLayer(
|
||||
channels,
|
||||
hidden_channels,
|
||||
kernel_size,
|
||||
dilation_rate,
|
||||
n_layers,
|
||||
gin_channels=gin_channels,
|
||||
mean_only=True,
|
||||
)
|
||||
)
|
||||
self.flows.append(modules.Flip())
|
||||
|
||||
def forward(self, x, x_mask, g=None, reverse=False):
|
||||
if not reverse:
|
||||
for flow in self.flows:
|
||||
x, _ = flow(x, x_mask, g=g, reverse=reverse)
|
||||
else:
|
||||
for flow in reversed(self.flows):
|
||||
x = flow(x, x_mask, g=g, reverse=reverse)
|
||||
return x
|
||||
|
||||
|
||||
class PosteriorEncoder(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
in_channels,
|
||||
out_channels,
|
||||
hidden_channels,
|
||||
kernel_size,
|
||||
dilation_rate,
|
||||
n_layers,
|
||||
gin_channels=0,
|
||||
):
|
||||
super().__init__()
|
||||
self.in_channels = in_channels
|
||||
self.out_channels = out_channels
|
||||
self.hidden_channels = hidden_channels
|
||||
self.kernel_size = kernel_size
|
||||
self.dilation_rate = dilation_rate
|
||||
self.n_layers = n_layers
|
||||
self.gin_channels = gin_channels
|
||||
|
||||
self.pre = nn.Conv1d(in_channels, hidden_channels, 1)
|
||||
self.enc = modules.WN(
|
||||
hidden_channels,
|
||||
kernel_size,
|
||||
dilation_rate,
|
||||
n_layers,
|
||||
gin_channels=gin_channels,
|
||||
)
|
||||
self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
|
||||
|
||||
def forward(self, x, x_lengths, g=None):
|
||||
x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(
|
||||
x.dtype
|
||||
)
|
||||
x = self.pre(x) * x_mask
|
||||
x = self.enc(x, x_mask, g=g)
|
||||
stats = self.proj(x) * x_mask
|
||||
m, logs = torch.split(stats, self.out_channels, dim=1)
|
||||
z = (m + torch.randn_like(m) * torch.exp(logs)) * x_mask
|
||||
return z, m, logs, x_mask
|
||||
|
||||
|
||||
class Generator(torch.nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
initial_channel,
|
||||
resblock,
|
||||
resblock_kernel_sizes,
|
||||
resblock_dilation_sizes,
|
||||
upsample_rates,
|
||||
upsample_initial_channel,
|
||||
upsample_kernel_sizes,
|
||||
gin_channels=0,
|
||||
):
|
||||
super(Generator, self).__init__()
|
||||
self.num_kernels = len(resblock_kernel_sizes)
|
||||
self.num_upsamples = len(upsample_rates)
|
||||
self.conv_pre = Conv1d(
|
||||
initial_channel, upsample_initial_channel, 7, 1, padding=3
|
||||
)
|
||||
resblock = modules.ResBlock1 if resblock == "1" else modules.ResBlock2
|
||||
|
||||
self.ups = nn.ModuleList()
|
||||
for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):
|
||||
self.ups.append(
|
||||
weight_norm(
|
||||
ConvTranspose1d(
|
||||
upsample_initial_channel // (2**i),
|
||||
upsample_initial_channel // (2 ** (i + 1)),
|
||||
k,
|
||||
u,
|
||||
padding=(k - u) // 2,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
self.resblocks = nn.ModuleList()
|
||||
for i in range(len(self.ups)):
|
||||
ch = upsample_initial_channel // (2 ** (i + 1))
|
||||
for j, (k, d) in enumerate(
|
||||
zip(resblock_kernel_sizes, resblock_dilation_sizes)
|
||||
):
|
||||
self.resblocks.append(resblock(ch, k, d))
|
||||
|
||||
self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False)
|
||||
self.ups.apply(init_weights)
|
||||
|
||||
if gin_channels != 0:
|
||||
self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1)
|
||||
|
||||
def forward(self, x, g=None):
|
||||
x = self.conv_pre(x)
|
||||
if g is not None:
|
||||
x = x + self.cond(g)
|
||||
|
||||
for i in range(self.num_upsamples):
|
||||
x = F.leaky_relu(x, modules.LRELU_SLOPE)
|
||||
x = self.ups[i](x)
|
||||
xs = None
|
||||
for j in range(self.num_kernels):
|
||||
if xs is None:
|
||||
xs = self.resblocks[i * self.num_kernels + j](x)
|
||||
else:
|
||||
xs += self.resblocks[i * self.num_kernels + j](x)
|
||||
x = xs / self.num_kernels
|
||||
x = F.leaky_relu(x)
|
||||
x = self.conv_post(x)
|
||||
x = torch.tanh(x)
|
||||
|
||||
return x
|
||||
|
||||
def remove_weight_norm(self):
|
||||
print("Removing weight norm...")
|
||||
for layer in self.ups:
|
||||
remove_weight_norm(layer)
|
||||
for layer in self.resblocks:
|
||||
layer.remove_weight_norm()
|
||||
|
||||
|
||||
class DiscriminatorP(torch.nn.Module):
|
||||
def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False):
|
||||
super(DiscriminatorP, self).__init__()
|
||||
self.period = period
|
||||
self.use_spectral_norm = use_spectral_norm
|
||||
norm_f = weight_norm if use_spectral_norm is False else spectral_norm
|
||||
self.convs = nn.ModuleList(
|
||||
[
|
||||
norm_f(
|
||||
Conv2d(
|
||||
1,
|
||||
32,
|
||||
(kernel_size, 1),
|
||||
(stride, 1),
|
||||
padding=(get_padding(kernel_size, 1), 0),
|
||||
)
|
||||
),
|
||||
norm_f(
|
||||
Conv2d(
|
||||
32,
|
||||
128,
|
||||
(kernel_size, 1),
|
||||
(stride, 1),
|
||||
padding=(get_padding(kernel_size, 1), 0),
|
||||
)
|
||||
),
|
||||
norm_f(
|
||||
Conv2d(
|
||||
128,
|
||||
512,
|
||||
(kernel_size, 1),
|
||||
(stride, 1),
|
||||
padding=(get_padding(kernel_size, 1), 0),
|
||||
)
|
||||
),
|
||||
norm_f(
|
||||
Conv2d(
|
||||
512,
|
||||
1024,
|
||||
(kernel_size, 1),
|
||||
(stride, 1),
|
||||
padding=(get_padding(kernel_size, 1), 0),
|
||||
)
|
||||
),
|
||||
norm_f(
|
||||
Conv2d(
|
||||
1024,
|
||||
1024,
|
||||
(kernel_size, 1),
|
||||
1,
|
||||
padding=(get_padding(kernel_size, 1), 0),
|
||||
)
|
||||
),
|
||||
]
|
||||
)
|
||||
self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0)))
|
||||
|
||||
def forward(self, x):
|
||||
fmap = []
|
||||
|
||||
# 1d to 2d
|
||||
b, c, t = x.shape
|
||||
if t % self.period != 0: # pad first
|
||||
n_pad = self.period - (t % self.period)
|
||||
x = F.pad(x, (0, n_pad), "reflect")
|
||||
t = t + n_pad
|
||||
x = x.view(b, c, t // self.period, self.period)
|
||||
|
||||
for layer in self.convs:
|
||||
x = layer(x)
|
||||
x = F.leaky_relu(x, modules.LRELU_SLOPE)
|
||||
fmap.append(x)
|
||||
x = self.conv_post(x)
|
||||
fmap.append(x)
|
||||
x = torch.flatten(x, 1, -1)
|
||||
|
||||
return x, fmap
|
||||
|
||||
|
||||
class DiscriminatorS(torch.nn.Module):
|
||||
def __init__(self, use_spectral_norm=False):
|
||||
super(DiscriminatorS, self).__init__()
|
||||
norm_f = weight_norm if use_spectral_norm is False else spectral_norm
|
||||
self.convs = nn.ModuleList(
|
||||
[
|
||||
norm_f(Conv1d(1, 16, 15, 1, padding=7)),
|
||||
norm_f(Conv1d(16, 64, 41, 4, groups=4, padding=20)),
|
||||
norm_f(Conv1d(64, 256, 41, 4, groups=16, padding=20)),
|
||||
norm_f(Conv1d(256, 1024, 41, 4, groups=64, padding=20)),
|
||||
norm_f(Conv1d(1024, 1024, 41, 4, groups=256, padding=20)),
|
||||
norm_f(Conv1d(1024, 1024, 5, 1, padding=2)),
|
||||
]
|
||||
)
|
||||
self.conv_post = norm_f(Conv1d(1024, 1, 3, 1, padding=1))
|
||||
|
||||
def forward(self, x):
|
||||
fmap = []
|
||||
|
||||
for layer in self.convs:
|
||||
x = layer(x)
|
||||
x = F.leaky_relu(x, modules.LRELU_SLOPE)
|
||||
fmap.append(x)
|
||||
x = self.conv_post(x)
|
||||
fmap.append(x)
|
||||
x = torch.flatten(x, 1, -1)
|
||||
|
||||
return x, fmap
|
||||
|
||||
|
||||
class MultiPeriodDiscriminator(torch.nn.Module):
|
||||
def __init__(self, use_spectral_norm=False):
|
||||
super(MultiPeriodDiscriminator, self).__init__()
|
||||
periods = [2, 3, 5, 7, 11]
|
||||
|
||||
discs = [DiscriminatorS(use_spectral_norm=use_spectral_norm)]
|
||||
discs = discs + [
|
||||
DiscriminatorP(i, use_spectral_norm=use_spectral_norm) for i in periods
|
||||
]
|
||||
self.discriminators = nn.ModuleList(discs)
|
||||
|
||||
def forward(self, y, y_hat):
|
||||
y_d_rs = []
|
||||
y_d_gs = []
|
||||
fmap_rs = []
|
||||
fmap_gs = []
|
||||
for i, d in enumerate(self.discriminators):
|
||||
y_d_r, fmap_r = d(y)
|
||||
y_d_g, fmap_g = d(y_hat)
|
||||
y_d_rs.append(y_d_r)
|
||||
y_d_gs.append(y_d_g)
|
||||
fmap_rs.append(fmap_r)
|
||||
fmap_gs.append(fmap_g)
|
||||
|
||||
return y_d_rs, y_d_gs, fmap_rs, fmap_gs
|
||||
|
||||
|
||||
class ReferenceEncoder(nn.Module):
|
||||
"""
|
||||
inputs --- [N, Ty/r, n_mels*r] mels
|
||||
outputs --- [N, ref_enc_gru_size]
|
||||
"""
|
||||
|
||||
def __init__(self, spec_channels, gin_channels=0):
|
||||
super().__init__()
|
||||
self.spec_channels = spec_channels
|
||||
ref_enc_filters = [32, 32, 64, 64, 128, 128]
|
||||
K = len(ref_enc_filters)
|
||||
filters = [1] + ref_enc_filters
|
||||
convs = [
|
||||
weight_norm(
|
||||
nn.Conv2d(
|
||||
in_channels=filters[i],
|
||||
out_channels=filters[i + 1],
|
||||
kernel_size=(3, 3),
|
||||
stride=(2, 2),
|
||||
padding=(1, 1),
|
||||
)
|
||||
)
|
||||
for i in range(K)
|
||||
]
|
||||
self.convs = nn.ModuleList(convs)
|
||||
# self.wns = nn.ModuleList([weight_norm(num_features=ref_enc_filters[i]) for i in range(K)]) # noqa: E501
|
||||
|
||||
out_channels = self.calculate_channels(spec_channels, 3, 2, 1, K)
|
||||
self.gru = nn.GRU(
|
||||
input_size=ref_enc_filters[-1] * out_channels,
|
||||
hidden_size=256 // 2,
|
||||
batch_first=True,
|
||||
)
|
||||
self.proj = nn.Linear(128, gin_channels)
|
||||
|
||||
def forward(self, inputs, mask=None):
|
||||
N = inputs.size(0)
|
||||
out = inputs.view(N, 1, -1, self.spec_channels) # [N, 1, Ty, n_freqs]
|
||||
for conv in self.convs:
|
||||
out = conv(out)
|
||||
# out = wn(out)
|
||||
out = F.relu(out) # [N, 128, Ty//2^K, n_mels//2^K]
|
||||
|
||||
out = out.transpose(1, 2) # [N, Ty//2^K, 128, n_mels//2^K]
|
||||
T = out.size(1)
|
||||
N = out.size(0)
|
||||
out = out.contiguous().view(N, T, -1) # [N, Ty//2^K, 128*n_mels//2^K]
|
||||
|
||||
self.gru.flatten_parameters()
|
||||
memory, out = self.gru(out) # out --- [1, N, 128]
|
||||
|
||||
return self.proj(out.squeeze(0))
|
||||
|
||||
def calculate_channels(self, L, kernel_size, stride, pad, n_convs):
|
||||
for i in range(n_convs):
|
||||
L = (L - kernel_size + 2 * pad) // stride + 1
|
||||
return L
|
||||
|
||||
|
||||
class SynthesizerTrn(nn.Module):
|
||||
"""
|
||||
Synthesizer for Training
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
n_vocab,
|
||||
spec_channels,
|
||||
segment_size,
|
||||
inter_channels,
|
||||
hidden_channels,
|
||||
filter_channels,
|
||||
n_heads,
|
||||
n_layers,
|
||||
kernel_size,
|
||||
p_dropout,
|
||||
resblock,
|
||||
resblock_kernel_sizes,
|
||||
resblock_dilation_sizes,
|
||||
upsample_rates,
|
||||
upsample_initial_channel,
|
||||
upsample_kernel_sizes,
|
||||
n_speakers=256,
|
||||
gin_channels=256,
|
||||
use_sdp=True,
|
||||
n_flow_layer=4,
|
||||
n_layers_trans_flow=6,
|
||||
flow_share_parameter=False,
|
||||
use_transformer_flow=True,
|
||||
**kwargs
|
||||
):
|
||||
super().__init__()
|
||||
self.n_vocab = n_vocab
|
||||
self.spec_channels = spec_channels
|
||||
self.inter_channels = inter_channels
|
||||
self.hidden_channels = hidden_channels
|
||||
self.filter_channels = filter_channels
|
||||
self.n_heads = n_heads
|
||||
self.n_layers = n_layers
|
||||
self.kernel_size = kernel_size
|
||||
self.p_dropout = p_dropout
|
||||
self.resblock = resblock
|
||||
self.resblock_kernel_sizes = resblock_kernel_sizes
|
||||
self.resblock_dilation_sizes = resblock_dilation_sizes
|
||||
self.upsample_rates = upsample_rates
|
||||
self.upsample_initial_channel = upsample_initial_channel
|
||||
self.upsample_kernel_sizes = upsample_kernel_sizes
|
||||
self.segment_size = segment_size
|
||||
self.n_speakers = n_speakers
|
||||
self.gin_channels = gin_channels
|
||||
self.n_layers_trans_flow = n_layers_trans_flow
|
||||
self.use_spk_conditioned_encoder = kwargs.get(
|
||||
"use_spk_conditioned_encoder", True
|
||||
)
|
||||
self.use_sdp = use_sdp
|
||||
self.use_noise_scaled_mas = kwargs.get("use_noise_scaled_mas", False)
|
||||
self.mas_noise_scale_initial = kwargs.get("mas_noise_scale_initial", 0.01)
|
||||
self.noise_scale_delta = kwargs.get("noise_scale_delta", 2e-6)
|
||||
self.current_mas_noise_scale = self.mas_noise_scale_initial
|
||||
if self.use_spk_conditioned_encoder and gin_channels > 0:
|
||||
self.enc_gin_channels = gin_channels
|
||||
self.enc_p = TextEncoder(
|
||||
n_vocab,
|
||||
inter_channels,
|
||||
hidden_channels,
|
||||
filter_channels,
|
||||
n_heads,
|
||||
n_layers,
|
||||
kernel_size,
|
||||
p_dropout,
|
||||
gin_channels=self.enc_gin_channels,
|
||||
)
|
||||
self.dec = Generator(
|
||||
inter_channels,
|
||||
resblock,
|
||||
resblock_kernel_sizes,
|
||||
resblock_dilation_sizes,
|
||||
upsample_rates,
|
||||
upsample_initial_channel,
|
||||
upsample_kernel_sizes,
|
||||
gin_channels=gin_channels,
|
||||
)
|
||||
self.enc_q = PosteriorEncoder(
|
||||
spec_channels,
|
||||
inter_channels,
|
||||
hidden_channels,
|
||||
5,
|
||||
1,
|
||||
16,
|
||||
gin_channels=gin_channels,
|
||||
)
|
||||
if use_transformer_flow:
|
||||
self.flow = TransformerCouplingBlock(
|
||||
inter_channels,
|
||||
hidden_channels,
|
||||
filter_channels,
|
||||
n_heads,
|
||||
n_layers_trans_flow,
|
||||
5,
|
||||
p_dropout,
|
||||
n_flow_layer,
|
||||
gin_channels=gin_channels,
|
||||
share_parameter=flow_share_parameter,
|
||||
)
|
||||
else:
|
||||
self.flow = ResidualCouplingBlock(
|
||||
inter_channels,
|
||||
hidden_channels,
|
||||
5,
|
||||
1,
|
||||
n_flow_layer,
|
||||
gin_channels=gin_channels,
|
||||
)
|
||||
self.sdp = StochasticDurationPredictor(
|
||||
hidden_channels, 192, 3, 0.5, 4, gin_channels=gin_channels
|
||||
)
|
||||
self.dp = DurationPredictor(
|
||||
hidden_channels, 256, 3, 0.5, gin_channels=gin_channels
|
||||
)
|
||||
|
||||
if n_speakers > 0:
|
||||
self.emb_g = nn.Embedding(n_speakers, gin_channels)
|
||||
else:
|
||||
self.ref_enc = ReferenceEncoder(spec_channels, gin_channels)
|
||||
|
||||
def forward(self, x, x_lengths, y, y_lengths, sid, tone, language, bert, ja_bert):
|
||||
if self.n_speakers > 0:
|
||||
g = self.emb_g(sid).unsqueeze(-1) # [b, h, 1]
|
||||
else:
|
||||
g = self.ref_enc(y.transpose(1, 2)).unsqueeze(-1)
|
||||
x, m_p, logs_p, x_mask = self.enc_p(
|
||||
x, x_lengths, tone, language, bert, ja_bert, g=g
|
||||
)
|
||||
z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g)
|
||||
z_p = self.flow(z, y_mask, g=g)
|
||||
|
||||
with torch.no_grad():
|
||||
# negative cross-entropy
|
||||
s_p_sq_r = torch.exp(-2 * logs_p) # [b, d, t]
|
||||
neg_cent1 = torch.sum(
|
||||
-0.5 * math.log(2 * math.pi) - logs_p, [1], keepdim=True
|
||||
) # [b, 1, t_s]
|
||||
neg_cent2 = torch.matmul(
|
||||
-0.5 * (z_p**2).transpose(1, 2), s_p_sq_r
|
||||
) # [b, t_t, d] x [b, d, t_s] = [b, t_t, t_s]
|
||||
neg_cent3 = torch.matmul(
|
||||
z_p.transpose(1, 2), (m_p * s_p_sq_r)
|
||||
) # [b, t_t, d] x [b, d, t_s] = [b, t_t, t_s]
|
||||
neg_cent4 = torch.sum(
|
||||
-0.5 * (m_p**2) * s_p_sq_r, [1], keepdim=True
|
||||
) # [b, 1, t_s]
|
||||
neg_cent = neg_cent1 + neg_cent2 + neg_cent3 + neg_cent4
|
||||
if self.use_noise_scaled_mas:
|
||||
epsilon = (
|
||||
torch.std(neg_cent)
|
||||
* torch.randn_like(neg_cent)
|
||||
* self.current_mas_noise_scale
|
||||
)
|
||||
neg_cent = neg_cent + epsilon
|
||||
|
||||
attn_mask = torch.unsqueeze(x_mask, 2) * torch.unsqueeze(y_mask, -1)
|
||||
attn = (
|
||||
monotonic_align.maximum_path(neg_cent, attn_mask.squeeze(1))
|
||||
.unsqueeze(1)
|
||||
.detach()
|
||||
)
|
||||
|
||||
w = attn.sum(2)
|
||||
|
||||
l_length_sdp = self.sdp(x, x_mask, w, g=g)
|
||||
l_length_sdp = l_length_sdp / torch.sum(x_mask)
|
||||
|
||||
logw_ = torch.log(w + 1e-6) * x_mask
|
||||
logw = self.dp(x, x_mask, g=g)
|
||||
l_length_dp = torch.sum((logw - logw_) ** 2, [1, 2]) / torch.sum(
|
||||
x_mask
|
||||
) # for averaging
|
||||
|
||||
l_length = l_length_dp + l_length_sdp
|
||||
|
||||
# expand prior
|
||||
m_p = torch.matmul(attn.squeeze(1), m_p.transpose(1, 2)).transpose(1, 2)
|
||||
logs_p = torch.matmul(attn.squeeze(1), logs_p.transpose(1, 2)).transpose(1, 2)
|
||||
|
||||
z_slice, ids_slice = commons.rand_slice_segments(
|
||||
z, y_lengths, self.segment_size
|
||||
)
|
||||
o = self.dec(z_slice, g=g)
|
||||
return (
|
||||
o,
|
||||
l_length,
|
||||
attn,
|
||||
ids_slice,
|
||||
x_mask,
|
||||
y_mask,
|
||||
(z, z_p, m_p, logs_p, m_q, logs_q),
|
||||
(x, logw, logw_),
|
||||
)
|
||||
|
||||
def infer(
|
||||
self,
|
||||
x,
|
||||
x_lengths,
|
||||
sid,
|
||||
tone,
|
||||
language,
|
||||
bert,
|
||||
ja_bert,
|
||||
noise_scale=0.667,
|
||||
length_scale=1,
|
||||
noise_scale_w=0.8,
|
||||
max_len=None,
|
||||
sdp_ratio=0,
|
||||
y=None,
|
||||
):
|
||||
# x, m_p, logs_p, x_mask = self.enc_p(x, x_lengths, tone, language, bert)
|
||||
# g = self.gst(y)
|
||||
if self.n_speakers > 0:
|
||||
g = self.emb_g(sid).unsqueeze(-1) # [b, h, 1]
|
||||
else:
|
||||
g = self.ref_enc(y.transpose(1, 2)).unsqueeze(-1)
|
||||
x, m_p, logs_p, x_mask = self.enc_p(
|
||||
x, x_lengths, tone, language, bert, ja_bert, g=g
|
||||
)
|
||||
logw = self.sdp(x, x_mask, g=g, reverse=True, noise_scale=noise_scale_w) * (
|
||||
sdp_ratio
|
||||
) + self.dp(x, x_mask, g=g) * (1 - sdp_ratio)
|
||||
w = torch.exp(logw) * x_mask * length_scale
|
||||
w_ceil = torch.ceil(w)
|
||||
y_lengths = torch.clamp_min(torch.sum(w_ceil, [1, 2]), 1).long()
|
||||
y_mask = torch.unsqueeze(commons.sequence_mask(y_lengths, None), 1).to(
|
||||
x_mask.dtype
|
||||
)
|
||||
attn_mask = torch.unsqueeze(x_mask, 2) * torch.unsqueeze(y_mask, -1)
|
||||
attn = commons.generate_path(w_ceil, attn_mask)
|
||||
|
||||
m_p = torch.matmul(attn.squeeze(1), m_p.transpose(1, 2)).transpose(
|
||||
1, 2
|
||||
) # [b, t', t], [b, t, d] -> [b, d, t']
|
||||
logs_p = torch.matmul(attn.squeeze(1), logs_p.transpose(1, 2)).transpose(
|
||||
1, 2
|
||||
) # [b, t', t], [b, t, d] -> [b, d, t']
|
||||
|
||||
z_p = m_p + torch.randn_like(m_p) * torch.exp(logs_p) * noise_scale
|
||||
z = self.flow(z_p, y_mask, g=g, reverse=True)
|
||||
o = self.dec((z * y_mask)[:, :, :max_len], g=g)
|
||||
return o, attn, y_mask, (z, z_p, m_p, logs_p)
|
||||
@@ -1,29 +0,0 @@
|
||||
from .symbols import *
|
||||
|
||||
|
||||
_symbol_to_id = {s: i for i, s in enumerate(symbols)}
|
||||
|
||||
|
||||
def cleaned_text_to_sequence(cleaned_text, tones, language):
|
||||
"""Converts a string of text to a sequence of IDs corresponding to the symbols in the text.
|
||||
Args:
|
||||
text: string to convert to a sequence
|
||||
Returns:
|
||||
List of integers corresponding to the symbols in the text
|
||||
"""
|
||||
phones = [_symbol_to_id[symbol] for symbol in cleaned_text]
|
||||
tone_start = language_tone_start_map[language]
|
||||
tones = [i + tone_start for i in tones]
|
||||
lang_id = language_id_map[language]
|
||||
lang_ids = [lang_id for i in phones]
|
||||
return phones, tones, lang_ids
|
||||
|
||||
|
||||
def get_bert(norm_text, word2ph, language, device):
|
||||
from .chinese_bert import get_bert_feature as zh_bert
|
||||
from .english_bert_mock import get_bert_feature as en_bert
|
||||
from .japanese_bert import get_bert_feature as jp_bert
|
||||
|
||||
lang_bert_func_map = {"ZH": zh_bert, "EN": en_bert, "JP": jp_bert}
|
||||
bert = lang_bert_func_map[language](norm_text, word2ph, device)
|
||||
return bert
|
||||
@@ -1,198 +0,0 @@
|
||||
import os
|
||||
import re
|
||||
|
||||
import cn2an
|
||||
from pypinyin import lazy_pinyin, Style
|
||||
|
||||
from .symbols import punctuation
|
||||
from .tone_sandhi import ToneSandhi
|
||||
|
||||
current_file_path = os.path.dirname(__file__)
|
||||
pinyin_to_symbol_map = {
|
||||
line.split("\t")[0]: line.strip().split("\t")[1]
|
||||
for line in open(os.path.join(current_file_path, "opencpop-strict.txt")).readlines()
|
||||
}
|
||||
|
||||
import jieba.posseg as psg
|
||||
|
||||
|
||||
rep_map = {
|
||||
"：": ",",
|
||||
"；": ",",
|
||||
"，": ",",
|
||||
"。": ".",
|
||||
"！": "!",
|
||||
"？": "?",
|
||||
"\n": ".",
|
||||
"·": ",",
|
||||
"、": ",",
|
||||
"...": "…",
|
||||
"$": ".",
|
||||
"“": "'",
|
||||
"”": "'",
|
||||
"‘": "'",
|
||||
"’": "'",
|
||||
"（": "'",
|
||||
"）": "'",
|
||||
"(": "'",
|
||||
")": "'",
|
||||
"《": "'",
|
||||
"》": "'",
|
||||
"【": "'",
|
||||
"】": "'",
|
||||
"[": "'",
|
||||
"]": "'",
|
||||
"—": "-",
|
||||
"～": "-",
|
||||
"~": "-",
|
||||
"「": "'",
|
||||
"」": "'",
|
||||
}
|
||||
|
||||
tone_modifier = ToneSandhi()
|
||||
|
||||
|
||||
def replace_punctuation(text):
|
||||
text = text.replace("嗯", "恩").replace("呣", "母")
|
||||
pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys()))
|
||||
|
||||
replaced_text = pattern.sub(lambda x: rep_map[x.group()], text)
|
||||
|
||||
replaced_text = re.sub(
|
||||
r"[^\u4e00-\u9fa5" + "".join(punctuation) + r"]+", "", replaced_text
|
||||
)
|
||||
|
||||
return replaced_text
|
||||
|
||||
|
||||
def g2p(text):
|
||||
pattern = r"(?<=[{0}])\s*".format("".join(punctuation))
|
||||
sentences = [i for i in re.split(pattern, text) if i.strip() != ""]
|
||||
phones, tones, word2ph = _g2p(sentences)
|
||||
assert sum(word2ph) == len(phones)
|
||||
assert len(word2ph) == len(text) # Sometimes it will crash,you can add a try-catch.
|
||||
phones = ["_"] + phones + ["_"]
|
||||
tones = [0] + tones + [0]
|
||||
word2ph = [1] + word2ph + [1]
|
||||
return phones, tones, word2ph
|
||||
|
||||
|
||||
def _get_initials_finals(word):
|
||||
initials = []
|
||||
finals = []
|
||||
orig_initials = lazy_pinyin(word, neutral_tone_with_five=True, style=Style.INITIALS)
|
||||
orig_finals = lazy_pinyin(
|
||||
word, neutral_tone_with_five=True, style=Style.FINALS_TONE3
|
||||
)
|
||||
for c, v in zip(orig_initials, orig_finals):
|
||||
initials.append(c)
|
||||
finals.append(v)
|
||||
return initials, finals
|
||||
|
||||
|
||||
def _g2p(segments):
|
||||
phones_list = []
|
||||
tones_list = []
|
||||
word2ph = []
|
||||
for seg in segments:
|
||||
# Replace all English words in the sentence
|
||||
seg = re.sub("[a-zA-Z]+", "", seg)
|
||||
seg_cut = psg.lcut(seg)
|
||||
initials = []
|
||||
finals = []
|
||||
seg_cut = tone_modifier.pre_merge_for_modify(seg_cut)
|
||||
for word, pos in seg_cut:
|
||||
if pos == "eng":
|
||||
continue
|
||||
sub_initials, sub_finals = _get_initials_finals(word)
|
||||
sub_finals = tone_modifier.modified_tone(word, pos, sub_finals)
|
||||
initials.append(sub_initials)
|
||||
finals.append(sub_finals)
|
||||
|
||||
# assert len(sub_initials) == len(sub_finals) == len(word)
|
||||
initials = sum(initials, [])
|
||||
finals = sum(finals, [])
|
||||
#
|
||||
for c, v in zip(initials, finals):
|
||||
raw_pinyin = c + v
|
||||
# NOTE: post process for pypinyin outputs
|
||||
# we discriminate i, ii and iii
|
||||
if c == v:
|
||||
assert c in punctuation
|
||||
phone = [c]
|
||||
tone = "0"
|
||||
word2ph.append(1)
|
||||
else:
|
||||
v_without_tone = v[:-1]
|
||||
tone = v[-1]
|
||||
|
||||
pinyin = c + v_without_tone
|
||||
assert tone in "12345"
|
||||
|
||||
if c:
|
||||
# 多音节
|
||||
v_rep_map = {
|
||||
"uei": "ui",
|
||||
"iou": "iu",
|
||||
"uen": "un",
|
||||
}
|
||||
if v_without_tone in v_rep_map.keys():
|
||||
pinyin = c + v_rep_map[v_without_tone]
|
||||
else:
|
||||
# 单音节
|
||||
pinyin_rep_map = {
|
||||
"ing": "ying",
|
||||
"i": "yi",
|
||||
"in": "yin",
|
||||
"u": "wu",
|
||||
}
|
||||
if pinyin in pinyin_rep_map.keys():
|
||||
pinyin = pinyin_rep_map[pinyin]
|
||||
else:
|
||||
single_rep_map = {
|
||||
"v": "yu",
|
||||
"e": "e",
|
||||
"i": "y",
|
||||
"u": "w",
|
||||
}
|
||||
if pinyin[0] in single_rep_map.keys():
|
||||
pinyin = single_rep_map[pinyin[0]] + pinyin[1:]
|
||||
|
||||
assert pinyin in pinyin_to_symbol_map.keys(), (pinyin, seg, raw_pinyin)
|
||||
phone = pinyin_to_symbol_map[pinyin].split(" ")
|
||||
word2ph.append(len(phone))
|
||||
|
||||
phones_list += phone
|
||||
tones_list += [int(tone)] * len(phone)
|
||||
return phones_list, tones_list, word2ph
|
||||
|
||||
|
||||
def text_normalize(text):
|
||||
numbers = re.findall(r"\d+(?:\.?\d+)?", text)
|
||||
for number in numbers:
|
||||
text = text.replace(number, cn2an.an2cn(number), 1)
|
||||
text = replace_punctuation(text)
|
||||
return text
|
||||
|
||||
|
||||
def get_bert_feature(text, word2ph):
|
||||
from text import chinese_bert
|
||||
|
||||
return chinese_bert.get_bert_feature(text, word2ph)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
from text.chinese_bert import get_bert_feature
|
||||
|
||||
text = "啊！但是《原神》是由,米哈\游自主， [研发]的一款全.新开放世界.冒险游戏"
|
||||
text = text_normalize(text)
|
||||
print(text)
|
||||
phones, tones, word2ph = g2p(text)
|
||||
bert = get_bert_feature(text, word2ph)
|
||||
|
||||
print(phones, tones, word2ph, bert.shape)
|
||||
|
||||
|
||||
# # 示例用法
|
||||
# text = "这是一个示例文本：,你好！这是一个测试...."
|
||||
# print(g2p_paddle(text)) # 输出: 这是一个示例文本你好这是一个测试
|
||||
@@ -1,97 +0,0 @@
|
||||
import torch
|
||||
import sys
|
||||
from transformers import AutoTokenizer, AutoModelForMaskedLM
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained("./bert/chinese-roberta-wwm-ext-large")
|
||||
|
||||
|
||||
def get_bert_feature(text, word2ph, device=None):
|
||||
if (
|
||||
sys.platform == "darwin"
|
||||
and torch.backends.mps.is_available()
|
||||
and device == "cpu"
|
||||
):
|
||||
device = "mps"
|
||||
if not device:
|
||||
device = "cuda"
|
||||
model = AutoModelForMaskedLM.from_pretrained(
|
||||
"./bert/chinese-roberta-wwm-ext-large"
|
||||
).to(device)
|
||||
with torch.no_grad():
|
||||
inputs = tokenizer(text, return_tensors="pt")
|
||||
for i in inputs:
|
||||
inputs[i] = inputs[i].to(device)
|
||||
res = model(**inputs, output_hidden_states=True)
|
||||
res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
|
||||
|
||||
assert len(word2ph) == len(text) + 2
|
||||
word2phone = word2ph
|
||||
phone_level_feature = []
|
||||
for i in range(len(word2phone)):
|
||||
repeat_feature = res[i].repeat(word2phone[i], 1)
|
||||
phone_level_feature.append(repeat_feature)
|
||||
|
||||
phone_level_feature = torch.cat(phone_level_feature, dim=0)
|
||||
|
||||
return phone_level_feature.T
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import torch
|
||||
|
||||
word_level_feature = torch.rand(38, 1024) # 12个词,每个词1024维特征
|
||||
word2phone = [
|
||||
1,
|
||||
2,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
1,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
1,
|
||||
]
|
||||
|
||||
# 计算总帧数
|
||||
total_frames = sum(word2phone)
|
||||
print(word_level_feature.shape)
|
||||
print(word2phone)
|
||||
phone_level_feature = []
|
||||
for i in range(len(word2phone)):
|
||||
print(word_level_feature[i].shape)
|
||||
|
||||
# 对每个词重复word2phone[i]次
|
||||
repeat_feature = word_level_feature[i].repeat(word2phone[i], 1)
|
||||
phone_level_feature.append(repeat_feature)
|
||||
|
||||
phone_level_feature = torch.cat(phone_level_feature, dim=0)
|
||||
print(phone_level_feature.shape) # torch.Size([36, 1024])
|
||||
@@ -1,28 +0,0 @@
|
||||
from . import chinese, japanese, cleaned_text_to_sequence
|
||||
|
||||
|
||||
language_module_map = {"ZH": chinese, "JP": japanese}
|
||||
|
||||
|
||||
def clean_text(text, language):
|
||||
language_module = language_module_map[language]
|
||||
norm_text = language_module.text_normalize(text)
|
||||
phones, tones, word2ph = language_module.g2p(norm_text)
|
||||
return norm_text, phones, tones, word2ph
|
||||
|
||||
|
||||
def clean_text_bert(text, language):
|
||||
language_module = language_module_map[language]
|
||||
norm_text = language_module.text_normalize(text)
|
||||
phones, tones, word2ph = language_module.g2p(norm_text)
|
||||
bert = language_module.get_bert_feature(norm_text, word2ph)
|
||||
return phones, tones, bert
|
||||
|
||||
|
||||
def text_to_sequence(text, language):
|
||||
norm_text, phones, tones, word2ph = clean_text(text, language)
|
||||
return cleaned_text_to_sequence(phones, tones, language)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pass
|
||||
@@ -1,214 +0,0 @@
|
||||
import pickle
|
||||
import os
|
||||
import re
|
||||
from g2p_en import G2p
|
||||
|
||||
from . import symbols
|
||||
|
||||
current_file_path = os.path.dirname(__file__)
|
||||
CMU_DICT_PATH = os.path.join(current_file_path, "cmudict.rep")
|
||||
CACHE_PATH = os.path.join(current_file_path, "cmudict_cache.pickle")
|
||||
_g2p = G2p()
|
||||
|
||||
arpa = {
|
||||
"AH0",
|
||||
"S",
|
||||
"AH1",
|
||||
"EY2",
|
||||
"AE2",
|
||||
"EH0",
|
||||
"OW2",
|
||||
"UH0",
|
||||
"NG",
|
||||
"B",
|
||||
"G",
|
||||
"AY0",
|
||||
"M",
|
||||
"AA0",
|
||||
"F",
|
||||
"AO0",
|
||||
"ER2",
|
||||
"UH1",
|
||||
"IY1",
|
||||
"AH2",
|
||||
"DH",
|
||||
"IY0",
|
||||
"EY1",
|
||||
"IH0",
|
||||
"K",
|
||||
"N",
|
||||
"W",
|
||||
"IY2",
|
||||
"T",
|
||||
"AA1",
|
||||
"ER1",
|
||||
"EH2",
|
||||
"OY0",
|
||||
"UH2",
|
||||
"UW1",
|
||||
"Z",
|
||||
"AW2",
|
||||
"AW1",
|
||||
"V",
|
||||
"UW2",
|
||||
"AA2",
|
||||
"ER",
|
||||
"AW0",
|
||||
"UW0",
|
||||
"R",
|
||||
"OW1",
|
||||
"EH1",
|
||||
"ZH",
|
||||
"AE0",
|
||||
"IH2",
|
||||
"IH",
|
||||
"Y",
|
||||
"JH",
|
||||
"P",
|
||||
"AY1",
|
||||
"EY0",
|
||||
"OY2",
|
||||
"TH",
|
||||
"HH",
|
||||
"D",
|
||||
"ER0",
|
||||
"CH",
|
||||
"AO1",
|
||||
"AE1",
|
||||
"AO2",
|
||||
"OY1",
|
||||
"AY2",
|
||||
"IH1",
|
||||
"OW0",
|
||||
"L",
|
||||
"SH",
|
||||
}
|
||||
|
||||
|
||||
def post_replace_ph(ph):
|
||||
rep_map = {
|
||||
"：": ",",
|
||||
"；": ",",
|
||||
"，": ",",
|
||||
"。": ".",
|
||||
"！": "!",
|
||||
"？": "?",
|
||||
"\n": ".",
|
||||
"·": ",",
|
||||
"、": ",",
|
||||
"...": "…",
|
||||
"v": "V",
|
||||
}
|
||||
if ph in rep_map.keys():
|
||||
ph = rep_map[ph]
|
||||
if ph in symbols:
|
||||
return ph
|
||||
if ph not in symbols:
|
||||
ph = "UNK"
|
||||
return ph
|
||||
|
||||
|
||||
def read_dict():
|
||||
g2p_dict = {}
|
||||
start_line = 49
|
||||
with open(CMU_DICT_PATH) as f:
|
||||
line = f.readline()
|
||||
line_index = 1
|
||||
while line:
|
||||
if line_index >= start_line:
|
||||
line = line.strip()
|
||||
word_split = line.split(" ")
|
||||
word = word_split[0]
|
||||
|
||||
syllable_split = word_split[1].split(" - ")
|
||||
g2p_dict[word] = []
|
||||
for syllable in syllable_split:
|
||||
phone_split = syllable.split(" ")
|
||||
g2p_dict[word].append(phone_split)
|
||||
|
||||
line_index = line_index + 1
|
||||
line = f.readline()
|
||||
|
||||
return g2p_dict
|
||||
|
||||
|
||||
def cache_dict(g2p_dict, file_path):
|
||||
with open(file_path, "wb") as pickle_file:
|
||||
pickle.dump(g2p_dict, pickle_file)
|
||||
|
||||
|
||||
def get_dict():
|
||||
if os.path.exists(CACHE_PATH):
|
||||
with open(CACHE_PATH, "rb") as pickle_file:
|
||||
g2p_dict = pickle.load(pickle_file)
|
||||
else:
|
||||
g2p_dict = read_dict()
|
||||
cache_dict(g2p_dict, CACHE_PATH)
|
||||
|
||||
return g2p_dict
|
||||
|
||||
|
||||
eng_dict = get_dict()
|
||||
|
||||
|
||||
def refine_ph(phn):
|
||||
tone = 0
|
||||
if re.search(r"\d$", phn):
|
||||
tone = int(phn[-1]) + 1
|
||||
phn = phn[:-1]
|
||||
return phn.lower(), tone
|
||||
|
||||
|
||||
def refine_syllables(syllables):
|
||||
tones = []
|
||||
phonemes = []
|
||||
for phn_list in syllables:
|
||||
for i in range(len(phn_list)):
|
||||
phn = phn_list[i]
|
||||
phn, tone = refine_ph(phn)
|
||||
phonemes.append(phn)
|
||||
tones.append(tone)
|
||||
return phonemes, tones
|
||||
|
||||
|
||||
def text_normalize(text):
|
||||
# todo: eng text normalize
|
||||
return text
|
||||
|
||||
|
||||
def g2p(text):
|
||||
phones = []
|
||||
tones = []
|
||||
words = re.split(r"([,;.\-\?\!\s+])", text)
|
||||
for w in words:
|
||||
if w.upper() in eng_dict:
|
||||
phns, tns = refine_syllables(eng_dict[w.upper()])
|
||||
phones += phns
|
||||
tones += tns
|
||||
else:
|
||||
phone_list = list(filter(lambda p: p != " ", _g2p(w)))
|
||||
for ph in phone_list:
|
||||
if ph in arpa:
|
||||
ph, tn = refine_ph(ph)
|
||||
phones.append(ph)
|
||||
tones.append(tn)
|
||||
else:
|
||||
phones.append(ph)
|
||||
tones.append(0)
|
||||
# todo: implement word2ph
|
||||
word2ph = [1 for i in phones]
|
||||
|
||||
phones = [post_replace_ph(i) for i in phones]
|
||||
return phones, tones, word2ph
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# print(get_dict())
|
||||
# print(eng_word_to_phoneme("hello"))
|
||||
print(g2p("In this paper, we propose 1 DSPGAN, a GAN-based universal vocoder."))
|
||||
# all_phones = set()
|
||||
# for k, syllables in eng_dict.items():
|
||||
# for group in syllables:
|
||||
# for ph in group:
|
||||
# all_phones.add(ph)
|
||||
# print(all_phones)
|
||||
@@ -1,5 +0,0 @@
|
||||
import torch
|
||||
|
||||
|
||||
def get_bert_feature(norm_text, word2ph):
|
||||
return torch.zeros(1024, sum(word2ph))
|
||||
@@ -1,586 +0,0 @@
|
||||
# Convert Japanese text to phonemes which is
|
||||
# compatible with Julius https://github.com/julius-speech/segmentation-kit
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from . import punctuation, symbols
|
||||
|
||||
try:
|
||||
import MeCab
|
||||
except ImportError as e:
|
||||
raise ImportError("Japanese requires mecab-python3 and unidic-lite.") from e
|
||||
from num2words import num2words
|
||||
|
||||
_CONVRULES = [
|
||||
# Conversion of 2 letters
|
||||
"アァ/ a a",
|
||||
"イィ/ i i",
|
||||
"イェ/ i e",
|
||||
"イャ/ y a",
|
||||
"ウゥ/ u:",
|
||||
"エェ/ e e",
|
||||
"オォ/ o:",
|
||||
"カァ/ k a:",
|
||||
"キィ/ k i:",
|
||||
"クゥ/ k u:",
|
||||
"クャ/ ky a",
|
||||
"クュ/ ky u",
|
||||
"クョ/ ky o",
|
||||
"ケェ/ k e:",
|
||||
"コォ/ k o:",
|
||||
"ガァ/ g a:",
|
||||
"ギィ/ g i:",
|
||||
"グゥ/ g u:",
|
||||
"グャ/ gy a",
|
||||
"グュ/ gy u",
|
||||
"グョ/ gy o",
|
||||
"ゲェ/ g e:",
|
||||
"ゴォ/ g o:",
|
||||
"サァ/ s a:",
|
||||
"シィ/ sh i:",
|
||||
"スゥ/ s u:",
|
||||
"スャ/ sh a",
|
||||
"スュ/ sh u",
|
||||
"スョ/ sh o",
|
||||
"セェ/ s e:",
|
||||
"ソォ/ s o:",
|
||||
"ザァ/ z a:",
|
||||
"ジィ/ j i:",
|
||||
"ズゥ/ z u:",
|
||||
"ズャ/ zy a",
|
||||
"ズュ/ zy u",
|
||||
"ズョ/ zy o",
|
||||
"ゼェ/ z e:",
|
||||
"ゾォ/ z o:",
|
||||
"タァ/ t a:",
|
||||
"チィ/ ch i:",
|
||||
"ツァ/ ts a",
|
||||
"ツィ/ ts i",
|
||||
"ツゥ/ ts u:",
|
||||
"ツャ/ ch a",
|
||||
"ツュ/ ch u",
|
||||
"ツョ/ ch o",
|
||||
"ツェ/ ts e",
|
||||
"ツォ/ ts o",
|
||||
"テェ/ t e:",
|
||||
"トォ/ t o:",
|
||||
"ダァ/ d a:",
|
||||
"ヂィ/ j i:",
|
||||
"ヅゥ/ d u:",
|
||||
"ヅャ/ zy a",
|
||||
"ヅュ/ zy u",
|
||||
"ヅョ/ zy o",
|
||||
"デェ/ d e:",
|
||||
"ドォ/ d o:",
|
||||
"ナァ/ n a:",
|
||||
"ニィ/ n i:",
|
||||
"ヌゥ/ n u:",
|
||||
"ヌャ/ ny a",
|
||||
"ヌュ/ ny u",
|
||||
"ヌョ/ ny o",
|
||||
"ネェ/ n e:",
|
||||
"ノォ/ n o:",
|
||||
"ハァ/ h a:",
|
||||
"ヒィ/ h i:",
|
||||
"フゥ/ f u:",
|
||||
"フャ/ hy a",
|
||||
"フュ/ hy u",
|
||||
"フョ/ hy o",
|
||||
"ヘェ/ h e:",
|
||||
"ホォ/ h o:",
|
||||
"バァ/ b a:",
|
||||
"ビィ/ b i:",
|
||||
"ブゥ/ b u:",
|
||||
"フャ/ hy a",
|
||||
"ブュ/ by u",
|
||||
"フョ/ hy o",
|
||||
"ベェ/ b e:",
|
||||
"ボォ/ b o:",
|
||||
"パァ/ p a:",
|
||||
"ピィ/ p i:",
|
||||
"プゥ/ p u:",
|
||||
"プャ/ py a",
|
||||
"プュ/ py u",
|
||||
"プョ/ py o",
|
||||
"ペェ/ p e:",
|
||||
"ポォ/ p o:",
|
||||
"マァ/ m a:",
|
||||
"ミィ/ m i:",
|
||||
"ムゥ/ m u:",
|
||||
"ムャ/ my a",
|
||||
"ムュ/ my u",
|
||||
"ムョ/ my o",
|
||||
"メェ/ m e:",
|
||||
"モォ/ m o:",
|
||||
"ヤァ/ y a:",
|
||||
"ユゥ/ y u:",
|
||||
"ユャ/ y a:",
|
||||
"ユュ/ y u:",
|
||||
"ユョ/ y o:",
|
||||
"ヨォ/ y o:",
|
||||
"ラァ/ r a:",
|
||||
"リィ/ r i:",
|
||||
"ルゥ/ r u:",
|
||||
"ルャ/ ry a",
|
||||
"ルュ/ ry u",
|
||||
"ルョ/ ry o",
|
||||
"レェ/ r e:",
|
||||
"ロォ/ r o:",
|
||||
"ワァ/ w a:",
|
||||
"ヲォ/ o:",
|
||||
"ディ/ d i",
|
||||
"デェ/ d e:",
|
||||
"デャ/ dy a",
|
||||
"デュ/ dy u",
|
||||
"デョ/ dy o",
|
||||
"ティ/ t i",
|
||||
"テェ/ t e:",
|
||||
"テャ/ ty a",
|
||||
"テュ/ ty u",
|
||||
"テョ/ ty o",
|
||||
"スィ/ s i",
|
||||
"ズァ/ z u a",
|
||||
"ズィ/ z i",
|
||||
"ズゥ/ z u",
|
||||
"ズャ/ zy a",
|
||||
"ズュ/ zy u",
|
||||
"ズョ/ zy o",
|
||||
"ズェ/ z e",
|
||||
"ズォ/ z o",
|
||||
"キャ/ ky a",
|
||||
"キュ/ ky u",
|
||||
"キョ/ ky o",
|
||||
"シャ/ sh a",
|
||||
"シュ/ sh u",
|
||||
"シェ/ sh e",
|
||||
"ショ/ sh o",
|
||||
"チャ/ ch a",
|
||||
"チュ/ ch u",
|
||||
"チェ/ ch e",
|
||||
"チョ/ ch o",
|
||||
"トゥ/ t u",
|
||||
"トャ/ ty a",
|
||||
"トュ/ ty u",
|
||||
"トョ/ ty o",
|
||||
"ドァ/ d o a",
|
||||
"ドゥ/ d u",
|
||||
"ドャ/ dy a",
|
||||
"ドュ/ dy u",
|
||||
"ドョ/ dy o",
|
||||
"ドォ/ d o:",
|
||||
"ニャ/ ny a",
|
||||
"ニュ/ ny u",
|
||||
"ニョ/ ny o",
|
||||
"ヒャ/ hy a",
|
||||
"ヒュ/ hy u",
|
||||
"ヒョ/ hy o",
|
||||
"ミャ/ my a",
|
||||
"ミュ/ my u",
|
||||
"ミョ/ my o",
|
||||
"リャ/ ry a",
|
||||
"リュ/ ry u",
|
||||
"リョ/ ry o",
|
||||
"ギャ/ gy a",
|
||||
"ギュ/ gy u",
|
||||
"ギョ/ gy o",
|
||||
"ヂェ/ j e",
|
||||
"ヂャ/ j a",
|
||||
"ヂュ/ j u",
|
||||
"ヂョ/ j o",
|
||||
"ジェ/ j e",
|
||||
"ジャ/ j a",
|
||||
"ジュ/ j u",
|
||||
"ジョ/ j o",
|
||||
"ビャ/ by a",
|
||||
"ビュ/ by u",
|
||||
"ビョ/ by o",
|
||||
"ピャ/ py a",
|
||||
"ピュ/ py u",
|
||||
"ピョ/ py o",
|
||||
"ウァ/ u a",
|
||||
"ウィ/ w i",
|
||||
"ウェ/ w e",
|
||||
"ウォ/ w o",
|
||||
"ファ/ f a",
|
||||
"フィ/ f i",
|
||||
"フゥ/ f u",
|
||||
"フャ/ hy a",
|
||||
"フュ/ hy u",
|
||||
"フョ/ hy o",
|
||||
"フェ/ f e",
|
||||
"フォ/ f o",
|
||||
"ヴァ/ b a",
|
||||
"ヴィ/ b i",
|
||||
"ヴェ/ b e",
|
||||
"ヴォ/ b o",
|
||||
"ヴュ/ by u",
|
||||
# Conversion of 1 letter
|
||||
"ア/ a",
|
||||
"イ/ i",
|
||||
"ウ/ u",
|
||||
"エ/ e",
|
||||
"オ/ o",
|
||||
"カ/ k a",
|
||||
"キ/ k i",
|
||||
"ク/ k u",
|
||||
"ケ/ k e",
|
||||
"コ/ k o",
|
||||
"サ/ s a",
|
||||
"シ/ sh i",
|
||||
"ス/ s u",
|
||||
"セ/ s e",
|
||||
"ソ/ s o",
|
||||
"タ/ t a",
|
||||
"チ/ ch i",
|
||||
"ツ/ ts u",
|
||||
"テ/ t e",
|
||||
"ト/ t o",
|
||||
"ナ/ n a",
|
||||
"ニ/ n i",
|
||||
"ヌ/ n u",
|
||||
"ネ/ n e",
|
||||
"ノ/ n o",
|
||||
"ハ/ h a",
|
||||
"ヒ/ h i",
|
||||
"フ/ f u",
|
||||
"ヘ/ h e",
|
||||
"ホ/ h o",
|
||||
"マ/ m a",
|
||||
"ミ/ m i",
|
||||
"ム/ m u",
|
||||
"メ/ m e",
|
||||
"モ/ m o",
|
||||
"ラ/ r a",
|
||||
"リ/ r i",
|
||||
"ル/ r u",
|
||||
"レ/ r e",
|
||||
"ロ/ r o",
|
||||
"ガ/ g a",
|
||||
"ギ/ g i",
|
||||
"グ/ g u",
|
||||
"ゲ/ g e",
|
||||
"ゴ/ g o",
|
||||
"ザ/ z a",
|
||||
"ジ/ j i",
|
||||
"ズ/ z u",
|
||||
"ゼ/ z e",
|
||||
"ゾ/ z o",
|
||||
"ダ/ d a",
|
||||
"ヂ/ j i",
|
||||
"ヅ/ z u",
|
||||
"デ/ d e",
|
||||
"ド/ d o",
|
||||
"バ/ b a",
|
||||
"ビ/ b i",
|
||||
"ブ/ b u",
|
||||
"ベ/ b e",
|
||||
"ボ/ b o",
|
||||
"パ/ p a",
|
||||
"ピ/ p i",
|
||||
"プ/ p u",
|
||||
"ペ/ p e",
|
||||
"ポ/ p o",
|
||||
"ヤ/ y a",
|
||||
"ユ/ y u",
|
||||
"ヨ/ y o",
|
||||
"ワ/ w a",
|
||||
"ヰ/ i",
|
||||
"ヱ/ e",
|
||||
"ヲ/ o",
|
||||
"ン/ N",
|
||||
"ッ/ q",
|
||||
"ヴ/ b u",
|
||||
"ー/:",
|
||||
# Try converting broken text
|
||||
"ァ/ a",
|
||||
"ィ/ i",
|
||||
"ゥ/ u",
|
||||
"ェ/ e",
|
||||
"ォ/ o",
|
||||
"ヮ/ w a",
|
||||
"ォ/ o",
|
||||
# Symbols
|
||||
"、/ ,",
|
||||
"。/ .",
|
||||
"！/ !",
|
||||
"？/ ?",
|
||||
"・/ ,",
|
||||
]
|
||||
|
||||
_COLON_RX = re.compile(":+")
|
||||
_REJECT_RX = re.compile("[^ a-zA-Z:,.?]")
|
||||
|
||||
|
||||
def _makerulemap():
|
||||
l = [tuple(x.split("/")) for x in _CONVRULES]
|
||||
return tuple({k: v for k, v in l if len(k) == i} for i in (1, 2))
|
||||
|
||||
|
||||
_RULEMAP1, _RULEMAP2 = _makerulemap()
|
||||
|
||||
|
||||
def kata2phoneme(text: str) -> str:
|
||||
"""Convert katakana text to phonemes."""
|
||||
text = text.strip()
|
||||
res = []
|
||||
while text:
|
||||
if len(text) >= 2:
|
||||
x = _RULEMAP2.get(text[:2])
|
||||
if x is not None:
|
||||
text = text[2:]
|
||||
res += x.split(" ")[1:]
|
||||
continue
|
||||
x = _RULEMAP1.get(text[0])
|
||||
if x is not None:
|
||||
text = text[1:]
|
||||
res += x.split(" ")[1:]
|
||||
continue
|
||||
res.append(text[0])
|
||||
text = text[1:]
|
||||
# res = _COLON_RX.sub(":", res)
|
||||
return res
|
||||
|
||||
|
||||
_KATAKANA = "".join(chr(ch) for ch in range(ord("ァ"), ord("ン") + 1))
|
||||
_HIRAGANA = "".join(chr(ch) for ch in range(ord("ぁ"), ord("ん") + 1))
|
||||
_HIRA2KATATRANS = str.maketrans(_HIRAGANA, _KATAKANA)
|
||||
|
||||
|
||||
def hira2kata(text: str) -> str:
|
||||
text = text.translate(_HIRA2KATATRANS)
|
||||
return text.replace("う゛", "ヴ")
|
||||
|
||||
|
||||
_SYMBOL_TOKENS = set(list("・、。？！"))
|
||||
_NO_YOMI_TOKENS = set(list("「」『』―（）［］[]"))
|
||||
_TAGGER = MeCab.Tagger()
|
||||
|
||||
|
||||
def text2kata(text: str) -> str:
|
||||
parsed = _TAGGER.parse(text)
|
||||
res = []
|
||||
for line in parsed.split("\n"):
|
||||
if line == "EOS":
|
||||
break
|
||||
parts = line.split("\t")
|
||||
|
||||
word, yomi = parts[0], parts[1]
|
||||
if yomi:
|
||||
res.append(yomi)
|
||||
else:
|
||||
if word in _SYMBOL_TOKENS:
|
||||
res.append(word)
|
||||
elif word in ("っ", "ッ"):
|
||||
res.append("ッ")
|
||||
elif word in _NO_YOMI_TOKENS:
|
||||
pass
|
||||
else:
|
||||
res.append(word)
|
||||
return hira2kata("".join(res))
|
||||
|
||||
|
||||
_ALPHASYMBOL_YOMI = {
|
||||
"#": "シャープ",
|
||||
"%": "パーセント",
|
||||
"&": "アンド",
|
||||
"+": "プラス",
|
||||
"-": "マイナス",
|
||||
":": "コロン",
|
||||
";": "セミコロン",
|
||||
"<": "小なり",
|
||||
"=": "イコール",
|
||||
">": "大なり",
|
||||
"@": "アット",
|
||||
"a": "エー",
|
||||
"b": "ビー",
|
||||
"c": "シー",
|
||||
"d": "ディー",
|
||||
"e": "イー",
|
||||
"f": "エフ",
|
||||
"g": "ジー",
|
||||
"h": "エイチ",
|
||||
"i": "アイ",
|
||||
"j": "ジェー",
|
||||
"k": "ケー",
|
||||
"l": "エル",
|
||||
"m": "エム",
|
||||
"n": "エヌ",
|
||||
"o": "オー",
|
||||
"p": "ピー",
|
||||
"q": "キュー",
|
||||
"r": "アール",
|
||||
"s": "エス",
|
||||
"t": "ティー",
|
||||
"u": "ユー",
|
||||
"v": "ブイ",
|
||||
"w": "ダブリュー",
|
||||
"x": "エックス",
|
||||
"y": "ワイ",
|
||||
"z": "ゼット",
|
||||
"α": "アルファ",
|
||||
"β": "ベータ",
|
||||
"γ": "ガンマ",
|
||||
"δ": "デルタ",
|
||||
"ε": "イプシロン",
|
||||
"ζ": "ゼータ",
|
||||
"η": "イータ",
|
||||
"θ": "シータ",
|
||||
"ι": "イオタ",
|
||||
"κ": "カッパ",
|
||||
"λ": "ラムダ",
|
||||
"μ": "ミュー",
|
||||
"ν": "ニュー",
|
||||
"ξ": "クサイ",
|
||||
"ο": "オミクロン",
|
||||
"π": "パイ",
|
||||
"ρ": "ロー",
|
||||
"σ": "シグマ",
|
||||
"τ": "タウ",
|
||||
"υ": "ウプシロン",
|
||||
"φ": "ファイ",
|
||||
"χ": "カイ",
|
||||
"ψ": "プサイ",
|
||||
"ω": "オメガ",
|
||||
}
|
||||
|
||||
|
||||
_NUMBER_WITH_SEPARATOR_RX = re.compile("[0-9]{1,3}(,[0-9]{3})+")
|
||||
_CURRENCY_MAP = {"$": "ドル", "¥": "円", "£": "ポンド", "€": "ユーロ"}
|
||||
_CURRENCY_RX = re.compile(r"([$¥£€])([0-9.]*[0-9])")
|
||||
_NUMBER_RX = re.compile(r"[0-9]+(\.[0-9]+)?")
|
||||
|
||||
|
||||
def japanese_convert_numbers_to_words(text: str) -> str:
|
||||
res = _NUMBER_WITH_SEPARATOR_RX.sub(lambda m: m[0].replace(",", ""), text)
|
||||
res = _CURRENCY_RX.sub(lambda m: m[2] + _CURRENCY_MAP.get(m[1], m[1]), res)
|
||||
res = _NUMBER_RX.sub(lambda m: num2words(m[0], lang="ja"), res)
|
||||
return res
|
||||
|
||||
|
||||
def japanese_convert_alpha_symbols_to_words(text: str) -> str:
|
||||
return "".join([_ALPHASYMBOL_YOMI.get(ch, ch) for ch in text.lower()])
|
||||
|
||||
|
||||
def japanese_text_to_phonemes(text: str) -> str:
|
||||
"""Convert Japanese text to phonemes."""
|
||||
res = unicodedata.normalize("NFKC", text)
|
||||
res = japanese_convert_numbers_to_words(res)
|
||||
# res = japanese_convert_alpha_symbols_to_words(res)
|
||||
res = text2kata(res)
|
||||
res = kata2phoneme(res)
|
||||
return res
|
||||
|
||||
|
||||
def is_japanese_character(char):
|
||||
# 定义日语文字系统的 Unicode 范围
|
||||
japanese_ranges = [
|
||||
(0x3040, 0x309F), # 平假名
|
||||
(0x30A0, 0x30FF), # 片假名
|
||||
(0x4E00, 0x9FFF), # 汉字 (CJK Unified Ideographs)
|
||||
(0x3400, 0x4DBF), # 汉字扩展 A
|
||||
(0x20000, 0x2A6DF), # 汉字扩展 B
|
||||
# 可以根据需要添加其他汉字扩展范围
|
||||
]
|
||||
|
||||
# 将字符的 Unicode 编码转换为整数
|
||||
char_code = ord(char)
|
||||
|
||||
# 检查字符是否在任何一个日语范围内
|
||||
for start, end in japanese_ranges:
|
||||
if start <= char_code <= end:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
rep_map = {
|
||||
"：": ",",
|
||||
"；": ",",
|
||||
"，": ",",
|
||||
"。": ".",
|
||||
"！": "!",
|
||||
"？": "?",
|
||||
"\n": ".",
|
||||
"·": ",",
|
||||
"、": ",",
|
||||
"...": "…",
|
||||
}
|
||||
|
||||
|
||||
def replace_punctuation(text):
|
||||
pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys()))
|
||||
|
||||
replaced_text = pattern.sub(lambda x: rep_map[x.group()], text)
|
||||
|
||||
replaced_text = re.sub(
|
||||
r"[^\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\u3400-\u4DBF"
|
||||
+ "".join(punctuation)
|
||||
+ r"]+",
|
||||
"",
|
||||
replaced_text,
|
||||
)
|
||||
|
||||
return replaced_text
|
||||
|
||||
|
||||
def text_normalize(text):
|
||||
res = unicodedata.normalize("NFKC", text)
|
||||
res = japanese_convert_numbers_to_words(res)
|
||||
# res = "".join([i for i in res if is_japanese_character(i)])
|
||||
res = replace_punctuation(res)
|
||||
return res
|
||||
|
||||
|
||||
def distribute_phone(n_phone, n_word):
|
||||
phones_per_word = [0] * n_word
|
||||
for task in range(n_phone):
|
||||
min_tasks = min(phones_per_word)
|
||||
min_index = phones_per_word.index(min_tasks)
|
||||
phones_per_word[min_index] += 1
|
||||
return phones_per_word
|
||||
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained("./bert/bert-base-japanese-v3")
|
||||
|
||||
|
||||
def g2p(norm_text):
|
||||
tokenized = tokenizer.tokenize(norm_text)
|
||||
phs = []
|
||||
ph_groups = []
|
||||
for t in tokenized:
|
||||
if not t.startswith("#"):
|
||||
ph_groups.append([t])
|
||||
else:
|
||||
ph_groups[-1].append(t.replace("#", ""))
|
||||
word2ph = []
|
||||
for group in ph_groups:
|
||||
phonemes = kata2phoneme(text2kata("".join(group)))
|
||||
# phonemes = [i for i in phonemes if i in symbols]
|
||||
for i in phonemes:
|
||||
assert i in symbols, (group, norm_text, tokenized)
|
||||
phone_len = len(phonemes)
|
||||
word_len = len(group)
|
||||
|
||||
aaa = distribute_phone(phone_len, word_len)
|
||||
word2ph += aaa
|
||||
|
||||
phs += phonemes
|
||||
phones = ["_"] + phs + ["_"]
|
||||
tones = [0 for i in phones]
|
||||
word2ph = [1] + word2ph + [1]
|
||||
return phones, tones, word2ph
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
tokenizer = AutoTokenizer.from_pretrained("./bert/bert-base-japanese-v3")
|
||||
text = "hello,こんにちは、世界！……"
|
||||
from text.japanese_bert import get_bert_feature
|
||||
|
||||
text = text_normalize(text)
|
||||
print(text)
|
||||
phones, tones, word2ph = g2p(text)
|
||||
bert = get_bert_feature(text, word2ph)
|
||||
|
||||
print(phones, tones, word2ph, bert.shape)
|
||||
@@ -1,35 +0,0 @@
|
||||
import torch
|
||||
from transformers import AutoTokenizer, AutoModelForMaskedLM
|
||||
import sys
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained("./bert/bert-base-japanese-v3")
|
||||
|
||||
|
||||
def get_bert_feature(text, word2ph, device=None):
|
||||
if (
|
||||
sys.platform == "darwin"
|
||||
and torch.backends.mps.is_available()
|
||||
and device == "cpu"
|
||||
):
|
||||
device = "mps"
|
||||
if not device:
|
||||
device = "cuda"
|
||||
model = AutoModelForMaskedLM.from_pretrained("./bert/bert-base-japanese-v3").to(
|
||||
device
|
||||
)
|
||||
with torch.no_grad():
|
||||
inputs = tokenizer(text, return_tensors="pt")
|
||||
for i in inputs:
|
||||
inputs[i] = inputs[i].to(device)
|
||||
res = model(**inputs, output_hidden_states=True)
|
||||
res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
|
||||
assert inputs["input_ids"].shape[-1] == len(word2ph)
|
||||
word2phone = word2ph
|
||||
phone_level_feature = []
|
||||
for i in range(len(word2phone)):
|
||||
repeat_feature = res[i].repeat(word2phone[i], 1)
|
||||
phone_level_feature.append(repeat_feature)
|
||||
|
||||
phone_level_feature = torch.cat(phone_level_feature, dim=0)
|
||||
|
||||
return phone_level_feature.T
|
||||
@@ -1,429 +0,0 @@
|
||||
a AA a
|
||||
ai AA ai
|
||||
an AA an
|
||||
ang AA ang
|
||||
ao AA ao
|
||||
ba b a
|
||||
bai b ai
|
||||
ban b an
|
||||
bang b ang
|
||||
bao b ao
|
||||
bei b ei
|
||||
ben b en
|
||||
beng b eng
|
||||
bi b i
|
||||
bian b ian
|
||||
biao b iao
|
||||
bie b ie
|
||||
bin b in
|
||||
bing b ing
|
||||
bo b o
|
||||
bu b u
|
||||
ca c a
|
||||
cai c ai
|
||||
can c an
|
||||
cang c ang
|
||||
cao c ao
|
||||
ce c e
|
||||
cei c ei
|
||||
cen c en
|
||||
ceng c eng
|
||||
cha ch a
|
||||
chai ch ai
|
||||
chan ch an
|
||||
chang ch ang
|
||||
chao ch ao
|
||||
che ch e
|
||||
chen ch en
|
||||
cheng ch eng
|
||||
chi ch ir
|
||||
chong ch ong
|
||||
chou ch ou
|
||||
chu ch u
|
||||
chua ch ua
|
||||
chuai ch uai
|
||||
chuan ch uan
|
||||
chuang ch uang
|
||||
chui ch ui
|
||||
chun ch un
|
||||
chuo ch uo
|
||||
ci c i0
|
||||
cong c ong
|
||||
cou c ou
|
||||
cu c u
|
||||
cuan c uan
|
||||
cui c ui
|
||||
cun c un
|
||||
cuo c uo
|
||||
da d a
|
||||
dai d ai
|
||||
dan d an
|
||||
dang d ang
|
||||
dao d ao
|
||||
de d e
|
||||
dei d ei
|
||||
den d en
|
||||
deng d eng
|
||||
di d i
|
||||
dia d ia
|
||||
dian d ian
|
||||
diao d iao
|
||||
die d ie
|
||||
ding d ing
|
||||
diu d iu
|
||||
dong d ong
|
||||
dou d ou
|
||||
du d u
|
||||
duan d uan
|
||||
dui d ui
|
||||
dun d un
|
||||
duo d uo
|
||||
e EE e
|
||||
ei EE ei
|
||||
en EE en
|
||||
eng EE eng
|
||||
er EE er
|
||||
fa f a
|
||||
fan f an
|
||||
fang f ang
|
||||
fei f ei
|
||||
fen f en
|
||||
feng f eng
|
||||
fo f o
|
||||
fou f ou
|
||||
fu f u
|
||||
ga g a
|
||||
gai g ai
|
||||
gan g an
|
||||
gang g ang
|
||||
gao g ao
|
||||
ge g e
|
||||
gei g ei
|
||||
gen g en
|
||||
geng g eng
|
||||
gong g ong
|
||||
gou g ou
|
||||
gu g u
|
||||
gua g ua
|
||||
guai g uai
|
||||
guan g uan
|
||||
guang g uang
|
||||
gui g ui
|
||||
gun g un
|
||||
guo g uo
|
||||
ha h a
|
||||
hai h ai
|
||||
han h an
|
||||
hang h ang
|
||||
hao h ao
|
||||
he h e
|
||||
hei h ei
|
||||
hen h en
|
||||
heng h eng
|
||||
hong h ong
|
||||
hou h ou
|
||||
hu h u
|
||||
hua h ua
|
||||
huai h uai
|
||||
huan h uan
|
||||
huang h uang
|
||||
hui h ui
|
||||
hun h un
|
||||
huo h uo
|
||||
ji j i
|
||||
jia j ia
|
||||
jian j ian
|
||||
jiang j iang
|
||||
jiao j iao
|
||||
jie j ie
|
||||
jin j in
|
||||
jing j ing
|
||||
jiong j iong
|
||||
jiu j iu
|
||||
ju j v
|
||||
jv j v
|
||||
juan j van
|
||||
jvan j van
|
||||
jue j ve
|
||||
jve j ve
|
||||
jun j vn
|
||||
jvn j vn
|
||||
ka k a
|
||||
kai k ai
|
||||
kan k an
|
||||
kang k ang
|
||||
kao k ao
|
||||
ke k e
|
||||
kei k ei
|
||||
ken k en
|
||||
keng k eng
|
||||
kong k ong
|
||||
kou k ou
|
||||
ku k u
|
||||
kua k ua
|
||||
kuai k uai
|
||||
kuan k uan
|
||||
kuang k uang
|
||||
kui k ui
|
||||
kun k un
|
||||
kuo k uo
|
||||
la l a
|
||||
lai l ai
|
||||
lan l an
|
||||
lang l ang
|
||||
lao l ao
|
||||
le l e
|
||||
lei l ei
|
||||
leng l eng
|
||||
li l i
|
||||
lia l ia
|
||||
lian l ian
|
||||
liang l iang
|
||||
liao l iao
|
||||
lie l ie
|
||||
lin l in
|
||||
ling l ing
|
||||
liu l iu
|
||||
lo l o
|
||||
long l ong
|
||||
lou l ou
|
||||
lu l u
|
||||
luan l uan
|
||||
lun l un
|
||||
luo l uo
|
||||
lv l v
|
||||
lve l ve
|
||||
ma m a
|
||||
mai m ai
|
||||
man m an
|
||||
mang m ang
|
||||
mao m ao
|
||||
me m e
|
||||
mei m ei
|
||||
men m en
|
||||
meng m eng
|
||||
mi m i
|
||||
mian m ian
|
||||
miao m iao
|
||||
mie m ie
|
||||
min m in
|
||||
ming m ing
|
||||
miu m iu
|
||||
mo m o
|
||||
mou m ou
|
||||
mu m u
|
||||
na n a
|
||||
nai n ai
|
||||
nan n an
|
||||
nang n ang
|
||||
nao n ao
|
||||
ne n e
|
||||
nei n ei
|
||||
nen n en
|
||||
neng n eng
|
||||
ni n i
|
||||
nian n ian
|
||||
niang n iang
|
||||
niao n iao
|
||||
nie n ie
|
||||
nin n in
|
||||
ning n ing
|
||||
niu n iu
|
||||
nong n ong
|
||||
nou n ou
|
||||
nu n u
|
||||
nuan n uan
|
||||
nun n un
|
||||
nuo n uo
|
||||
nv n v
|
||||
nve n ve
|
||||
o OO o
|
||||
ou OO ou
|
||||
pa p a
|
||||
pai p ai
|
||||
pan p an
|
||||
pang p ang
|
||||
pao p ao
|
||||
pei p ei
|
||||
pen p en
|
||||
peng p eng
|
||||
pi p i
|
||||
pian p ian
|
||||
piao p iao
|
||||
pie p ie
|
||||
pin p in
|
||||
ping p ing
|
||||
po p o
|
||||
pou p ou
|
||||
pu p u
|
||||
qi q i
|
||||
qia q ia
|
||||
qian q ian
|
||||
qiang q iang
|
||||
qiao q iao
|
||||
qie q ie
|
||||
qin q in
|
||||
qing q ing
|
||||
qiong q iong
|
||||
qiu q iu
|
||||
qu q v
|
||||
qv q v
|
||||
quan q van
|
||||
qvan q van
|
||||
que q ve
|
||||
qve q ve
|
||||
qun q vn
|
||||
qvn q vn
|
||||
ran r an
|
||||
rang r ang
|
||||
rao r ao
|
||||
re r e
|
||||
ren r en
|
||||
reng r eng
|
||||
ri r ir
|
||||
rong r ong
|
||||
rou r ou
|
||||
ru r u
|
||||
rua r ua
|
||||
ruan r uan
|
||||
rui r ui
|
||||
run r un
|
||||
ruo r uo
|
||||
sa s a
|
||||
sai s ai
|
||||
san s an
|
||||
sang s ang
|
||||
sao s ao
|
||||
se s e
|
||||
sen s en
|
||||
seng s eng
|
||||
sha sh a
|
||||
shai sh ai
|
||||
shan sh an
|
||||
shang sh ang
|
||||
shao sh ao
|
||||
she sh e
|
||||
shei sh ei
|
||||
shen sh en
|
||||
sheng sh eng
|
||||
shi sh ir
|
||||
shou sh ou
|
||||
shu sh u
|
||||
shua sh ua
|
||||
shuai sh uai
|
||||
shuan sh uan
|
||||
shuang sh uang
|
||||
shui sh ui
|
||||
shun sh un
|
||||
shuo sh uo
|
||||
si s i0
|
||||
song s ong
|
||||
sou s ou
|
||||
su s u
|
||||
suan s uan
|
||||
sui s ui
|
||||
sun s un
|
||||
suo s uo
|
||||
ta t a
|
||||
tai t ai
|
||||
tan t an
|
||||
tang t ang
|
||||
tao t ao
|
||||
te t e
|
||||
tei t ei
|
||||
teng t eng
|
||||
ti t i
|
||||
tian t ian
|
||||
tiao t iao
|
||||
tie t ie
|
||||
ting t ing
|
||||
tong t ong
|
||||
tou t ou
|
||||
tu t u
|
||||
tuan t uan
|
||||
tui t ui
|
||||
tun t un
|
||||
tuo t uo
|
||||
wa w a
|
||||
wai w ai
|
||||
wan w an
|
||||
wang w ang
|
||||
wei w ei
|
||||
wen w en
|
||||
weng w eng
|
||||
wo w o
|
||||
wu w u
|
||||
xi x i
|
||||
xia x ia
|
||||
xian x ian
|
||||
xiang x iang
|
||||
xiao x iao
|
||||
xie x ie
|
||||
xin x in
|
||||
xing x ing
|
||||
xiong x iong
|
||||
xiu x iu
|
||||
xu x v
|
||||
xv x v
|
||||
xuan x van
|
||||
xvan x van
|
||||
xue x ve
|
||||
xve x ve
|
||||
xun x vn
|
||||
xvn x vn
|
||||
ya y a
|
||||
yan y En
|
||||
yang y ang
|
||||
yao y ao
|
||||
ye y E
|
||||
yi y i
|
||||
yin y in
|
||||
ying y ing
|
||||
yo y o
|
||||
yong y ong
|
||||
you y ou
|
||||
yu y v
|
||||
yv y v
|
||||
yuan y van
|
||||
yvan y van
|
||||
yue y ve
|
||||
yve y ve
|
||||
yun y vn
|
||||
yvn y vn
|
||||
za z a
|
||||
zai z ai
|
||||
zan z an
|
||||
zang z ang
|
||||
zao z ao
|
||||
ze z e
|
||||
zei z ei
|
||||
zen z en
|
||||
zeng z eng
|
||||
zha zh a
|
||||
zhai zh ai
|
||||
zhan zh an
|
||||
zhang zh ang
|
||||
zhao zh ao
|
||||
zhe zh e
|
||||
zhei zh ei
|
||||
zhen zh en
|
||||
zheng zh eng
|
||||
zhi zh ir
|
||||
zhong zh ong
|
||||
zhou zh ou
|
||||
zhu zh u
|
||||
zhua zh ua
|
||||
zhuai zh uai
|
||||
zhuan zh uan
|
||||
zhuang zh uang
|
||||
zhui zh ui
|
||||
zhun zh un
|
||||
zhuo zh uo
|
||||
zi z i0
|
||||
zong z ong
|
||||
zou z ou
|
||||
zu z u
|
||||
zuan z uan
|
||||
zui z ui
|
||||
zun z un
|
||||
zuo z uo
|
||||
@@ -1,187 +0,0 @@
|
||||
punctuation = ["!", "?", "…", ",", ".", "'", "-"]
|
||||
pu_symbols = punctuation + ["SP", "UNK"]
|
||||
pad = "_"
|
||||
|
||||
# chinese
|
||||
zh_symbols = [
|
||||
"E",
|
||||
"En",
|
||||
"a",
|
||||
"ai",
|
||||
"an",
|
||||
"ang",
|
||||
"ao",
|
||||
"b",
|
||||
"c",
|
||||
"ch",
|
||||
"d",
|
||||
"e",
|
||||
"ei",
|
||||
"en",
|
||||
"eng",
|
||||
"er",
|
||||
"f",
|
||||
"g",
|
||||
"h",
|
||||
"i",
|
||||
"i0",
|
||||
"ia",
|
||||
"ian",
|
||||
"iang",
|
||||
"iao",
|
||||
"ie",
|
||||
"in",
|
||||
"ing",
|
||||
"iong",
|
||||
"ir",
|
||||
"iu",
|
||||
"j",
|
||||
"k",
|
||||
"l",
|
||||
"m",
|
||||
"n",
|
||||
"o",
|
||||
"ong",
|
||||
"ou",
|
||||
"p",
|
||||
"q",
|
||||
"r",
|
||||
"s",
|
||||
"sh",
|
||||
"t",
|
||||
"u",
|
||||
"ua",
|
||||
"uai",
|
||||
"uan",
|
||||
"uang",
|
||||
"ui",
|
||||
"un",
|
||||
"uo",
|
||||
"v",
|
||||
"van",
|
||||
"ve",
|
||||
"vn",
|
||||
"w",
|
||||
"x",
|
||||
"y",
|
||||
"z",
|
||||
"zh",
|
||||
"AA",
|
||||
"EE",
|
||||
"OO",
|
||||
]
|
||||
num_zh_tones = 6
|
||||
|
||||
# japanese
|
||||
ja_symbols = [
|
||||
"N",
|
||||
"a",
|
||||
"a:",
|
||||
"b",
|
||||
"by",
|
||||
"ch",
|
||||
"d",
|
||||
"dy",
|
||||
"e",
|
||||
"e:",
|
||||
"f",
|
||||
"g",
|
||||
"gy",
|
||||
"h",
|
||||
"hy",
|
||||
"i",
|
||||
"i:",
|
||||
"j",
|
||||
"k",
|
||||
"ky",
|
||||
"m",
|
||||
"my",
|
||||
"n",
|
||||
"ny",
|
||||
"o",
|
||||
"o:",
|
||||
"p",
|
||||
"py",
|
||||
"q",
|
||||
"r",
|
||||
"ry",
|
||||
"s",
|
||||
"sh",
|
||||
"t",
|
||||
"ts",
|
||||
"ty",
|
||||
"u",
|
||||
"u:",
|
||||
"w",
|
||||
"y",
|
||||
"z",
|
||||
"zy",
|
||||
]
|
||||
num_ja_tones = 1
|
||||
|
||||
# English
|
||||
en_symbols = [
|
||||
"aa",
|
||||
"ae",
|
||||
"ah",
|
||||
"ao",
|
||||
"aw",
|
||||
"ay",
|
||||
"b",
|
||||
"ch",
|
||||
"d",
|
||||
"dh",
|
||||
"eh",
|
||||
"er",
|
||||
"ey",
|
||||
"f",
|
||||
"g",
|
||||
"hh",
|
||||
"ih",
|
||||
"iy",
|
||||
"jh",
|
||||
"k",
|
||||
"l",
|
||||
"m",
|
||||
"n",
|
||||
"ng",
|
||||
"ow",
|
||||
"oy",
|
||||
"p",
|
||||
"r",
|
||||
"s",
|
||||
"sh",
|
||||
"t",
|
||||
"th",
|
||||
"uh",
|
||||
"uw",
|
||||
"V",
|
||||
"w",
|
||||
"y",
|
||||
"z",
|
||||
"zh",
|
||||
]
|
||||
num_en_tones = 4
|
||||
|
||||
# combine all symbols
|
||||
normal_symbols = sorted(set(zh_symbols + ja_symbols + en_symbols))
|
||||
symbols = [pad] + normal_symbols + pu_symbols
|
||||
sil_phonemes_ids = [symbols.index(i) for i in pu_symbols]
|
||||
|
||||
# combine all tones
|
||||
num_tones = num_zh_tones + num_ja_tones + num_en_tones
|
||||
|
||||
# language maps
|
||||
language_id_map = {"ZH": 0, "JP": 1, "EN": 2}
|
||||
num_languages = len(language_id_map.keys())
|
||||
|
||||
language_tone_start_map = {
|
||||
"ZH": 0,
|
||||
"JP": num_zh_tones,
|
||||
"EN": num_zh_tones + num_ja_tones,
|
||||
}
|
||||
|
||||
if __name__ == "__main__":
|
||||
a = set(zh_symbols)
|
||||
b = set(en_symbols)
|
||||
print(sorted(a & b))
|
||||
@@ -1,769 +0,0 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from typing import List
|
||||
from typing import Tuple
|
||||
|
||||
import jieba
|
||||
from pypinyin import lazy_pinyin
|
||||
from pypinyin import Style
|
||||
|
||||
|
||||
class ToneSandhi:
|
||||
def __init__(self):
|
||||
self.must_neural_tone_words = {
|
||||
"麻烦",
|
||||
"麻利",
|
||||
"鸳鸯",
|
||||
"高粱",
|
||||
"骨头",
|
||||
"骆驼",
|
||||
"马虎",
|
||||
"首饰",
|
||||
"馒头",
|
||||
"馄饨",
|
||||
"风筝",
|
||||
"难为",
|
||||
"队伍",
|
||||
"阔气",
|
||||
"闺女",
|
||||
"门道",
|
||||
"锄头",
|
||||
"铺盖",
|
||||
"铃铛",
|
||||
"铁匠",
|
||||
"钥匙",
|
||||
"里脊",
|
||||
"里头",
|
||||
"部分",
|
||||
"那么",
|
||||
"道士",
|
||||
"造化",
|
||||
"迷糊",
|
||||
"连累",
|
||||
"这么",
|
||||
"这个",
|
||||
"运气",
|
||||
"过去",
|
||||
"软和",
|
||||
"转悠",
|
||||
"踏实",
|
||||
"跳蚤",
|
||||
"跟头",
|
||||
"趔趄",
|
||||
"财主",
|
||||
"豆腐",
|
||||
"讲究",
|
||||
"记性",
|
||||
"记号",
|
||||
"认识",
|
||||
"规矩",
|
||||
"见识",
|
||||
"裁缝",
|
||||
"补丁",
|
||||
"衣裳",
|
||||
"衣服",
|
||||
"衙门",
|
||||
"街坊",
|
||||
"行李",
|
||||
"行当",
|
||||
"蛤蟆",
|
||||
"蘑菇",
|
||||
"薄荷",
|
||||
"葫芦",
|
||||
"葡萄",
|
||||
"萝卜",
|
||||
"荸荠",
|
||||
"苗条",
|
||||
"苗头",
|
||||
"苍蝇",
|
||||
"芝麻",
|
||||
"舒服",
|
||||
"舒坦",
|
||||
"舌头",
|
||||
"自在",
|
||||
"膏药",
|
||||
"脾气",
|
||||
"脑袋",
|
||||
"脊梁",
|
||||
"能耐",
|
||||
"胳膊",
|
||||
"胭脂",
|
||||
"胡萝",
|
||||
"胡琴",
|
||||
"胡同",
|
||||
"聪明",
|
||||
"耽误",
|
||||
"耽搁",
|
||||
"耷拉",
|
||||
"耳朵",
|
||||
"老爷",
|
||||
"老实",
|
||||
"老婆",
|
||||
"老头",
|
||||
"老太",
|
||||
"翻腾",
|
||||
"罗嗦",
|
||||
"罐头",
|
||||
"编辑",
|
||||
"结实",
|
||||
"红火",
|
||||
"累赘",
|
||||
"糨糊",
|
||||
"糊涂",
|
||||
"精神",
|
||||
"粮食",
|
||||
"簸箕",
|
||||
"篱笆",
|
||||
"算计",
|
||||
"算盘",
|
||||
"答应",
|
||||
"笤帚",
|
||||
"笑语",
|
||||
"笑话",
|
||||
"窟窿",
|
||||
"窝囊",
|
||||
"窗户",
|
||||
"稳当",
|
||||
"稀罕",
|
||||
"称呼",
|
||||
"秧歌",
|
||||
"秀气",
|
||||
"秀才",
|
||||
"福气",
|
||||
"祖宗",
|
||||
"砚台",
|
||||
"码头",
|
||||
"石榴",
|
||||
"石头",
|
||||
"石匠",
|
||||
"知识",
|
||||
"眼睛",
|
||||
"眯缝",
|
||||
"眨巴",
|
||||
"眉毛",
|
||||
"相声",
|
||||
"盘算",
|
||||
"白净",
|
||||
"痢疾",
|
||||
"痛快",
|
||||
"疟疾",
|
||||
"疙瘩",
|
||||
"疏忽",
|
||||
"畜生",
|
||||
"生意",
|
||||
"甘蔗",
|
||||
"琵琶",
|
||||
"琢磨",
|
||||
"琉璃",
|
||||
"玻璃",
|
||||
"玫瑰",
|
||||
"玄乎",
|
||||
"狐狸",
|
||||
"状元",
|
||||
"特务",
|
||||
"牲口",
|
||||
"牙碜",
|
||||
"牌楼",
|
||||
"爽快",
|
||||
"爱人",
|
||||
"热闹",
|
||||
"烧饼",
|
||||
"烟筒",
|
||||
"烂糊",
|
||||
"点心",
|
||||
"炊帚",
|
||||
"灯笼",
|
||||
"火候",
|
||||
"漂亮",
|
||||
"滑溜",
|
||||
"溜达",
|
||||
"温和",
|
||||
"清楚",
|
||||
"消息",
|
||||
"浪头",
|
||||
"活泼",
|
||||
"比方",
|
||||
"正经",
|
||||
"欺负",
|
||||
"模糊",
|
||||
"槟榔",
|
||||
"棺材",
|
||||
"棒槌",
|
||||
"棉花",
|
||||
"核桃",
|
||||
"栅栏",
|
||||
"柴火",
|
||||
"架势",
|
||||
"枕头",
|
||||
"枇杷",
|
||||
"机灵",
|
||||
"本事",
|
||||
"木头",
|
||||
"木匠",
|
||||
"朋友",
|
||||
"月饼",
|
||||
"月亮",
|
||||
"暖和",
|
||||
"明白",
|
||||
"时候",
|
||||
"新鲜",
|
||||
"故事",
|
||||
"收拾",
|
||||
"收成",
|
||||
"提防",
|
||||
"挖苦",
|
||||
"挑剔",
|
||||
"指甲",
|
||||
"指头",
|
||||
"拾掇",
|
||||
"拳头",
|
||||
"拨弄",
|
||||
"招牌",
|
||||
"招呼",
|
||||
"抬举",
|
||||
"护士",
|
||||
"折腾",
|
||||
"扫帚",
|
||||
"打量",
|
||||
"打算",
|
||||
"打点",
|
||||
"打扮",
|
||||
"打听",
|
||||
"打发",
|
||||
"扎实",
|
||||
"扁担",
|
||||
"戒指",
|
||||
"懒得",
|
||||
"意识",
|
||||
"意思",
|
||||
"情形",
|
||||
"悟性",
|
||||
"怪物",
|
||||
"思量",
|
||||
"怎么",
|
||||
"念头",
|
||||
"念叨",
|
||||
"快活",
|
||||
"忙活",
|
||||
"志气",
|
||||
"心思",
|
||||
"得罪",
|
||||
"张罗",
|
||||
"弟兄",
|
||||
"开通",
|
||||
"应酬",
|
||||
"庄稼",
|
||||
"干事",
|
||||
"帮手",
|
||||
"帐篷",
|
||||
"希罕",
|
||||
"师父",
|
||||
"师傅",
|
||||
"巴结",
|
||||
"巴掌",
|
||||
"差事",
|
||||
"工夫",
|
||||
"岁数",
|
||||
"屁股",
|
||||
"尾巴",
|
||||
"少爷",
|
||||
"小气",
|
||||
"小伙",
|
||||
"将就",
|
||||
"对头",
|
||||
"对付",
|
||||
"寡妇",
|
||||
"家伙",
|
||||
"客气",
|
||||
"实在",
|
||||
"官司",
|
||||
"学问",
|
||||
"学生",
|
||||
"字号",
|
||||
"嫁妆",
|
||||
"媳妇",
|
||||
"媒人",
|
||||
"婆家",
|
||||
"娘家",
|
||||
"委屈",
|
||||
"姑娘",
|
||||
"姐夫",
|
||||
"妯娌",
|
||||
"妥当",
|
||||
"妖精",
|
||||
"奴才",
|
||||
"女婿",
|
||||
"头发",
|
||||
"太阳",
|
||||
"大爷",
|
||||
"大方",
|
||||
"大意",
|
||||
"大夫",
|
||||
"多少",
|
||||
"多么",
|
||||
"外甥",
|
||||
"壮实",
|
||||
"地道",
|
||||
"地方",
|
||||
"在乎",
|
||||
"困难",
|
||||
"嘴巴",
|
||||
"嘱咐",
|
||||
"嘟囔",
|
||||
"嘀咕",
|
||||
"喜欢",
|
||||
"喇嘛",
|
||||
"喇叭",
|
||||
"商量",
|
||||
"唾沫",
|
||||
"哑巴",
|
||||
"哈欠",
|
||||
"哆嗦",
|
||||
"咳嗽",
|
||||
"和尚",
|
||||
"告诉",
|
||||
"告示",
|
||||
"含糊",
|
||||
"吓唬",
|
||||
"后头",
|
||||
"名字",
|
||||
"名堂",
|
||||
"合同",
|
||||
"吆喝",
|
||||
"叫唤",
|
||||
"口袋",
|
||||
"厚道",
|
||||
"厉害",
|
||||
"千斤",
|
||||
"包袱",
|
||||
"包涵",
|
||||
"匀称",
|
||||
"勤快",
|
||||
"动静",
|
||||
"动弹",
|
||||
"功夫",
|
||||
"力气",
|
||||
"前头",
|
||||
"刺猬",
|
||||
"刺激",
|
||||
"别扭",
|
||||
"利落",
|
||||
"利索",
|
||||
"利害",
|
||||
"分析",
|
||||
"出息",
|
||||
"凑合",
|
||||
"凉快",
|
||||
"冷战",
|
||||
"冤枉",
|
||||
"冒失",
|
||||
"养活",
|
||||
"关系",
|
||||
"先生",
|
||||
"兄弟",
|
||||
"便宜",
|
||||
"使唤",
|
||||
"佩服",
|
||||
"作坊",
|
||||
"体面",
|
||||
"位置",
|
||||
"似的",
|
||||
"伙计",
|
||||
"休息",
|
||||
"什么",
|
||||
"人家",
|
||||
"亲戚",
|
||||
"亲家",
|
||||
"交情",
|
||||
"云彩",
|
||||
"事情",
|
||||
"买卖",
|
||||
"主意",
|
||||
"丫头",
|
||||
"丧气",
|
||||
"两口",
|
||||
"东西",
|
||||
"东家",
|
||||
"世故",
|
||||
"不由",
|
||||
"不在",
|
||||
"下水",
|
||||
"下巴",
|
||||
"上头",
|
||||
"上司",
|
||||
"丈夫",
|
||||
"丈人",
|
||||
"一辈",
|
||||
"那个",
|
||||
"菩萨",
|
||||
"父亲",
|
||||
"母亲",
|
||||
"咕噜",
|
||||
"邋遢",
|
||||
"费用",
|
||||
"冤家",
|
||||
"甜头",
|
||||
"介绍",
|
||||
"荒唐",
|
||||
"大人",
|
||||
"泥鳅",
|
||||
"幸福",
|
||||
"熟悉",
|
||||
"计划",
|
||||
"扑腾",
|
||||
"蜡烛",
|
||||
"姥爷",
|
||||
"照顾",
|
||||
"喉咙",
|
||||
"吉他",
|
||||
"弄堂",
|
||||
"蚂蚱",
|
||||
"凤凰",
|
||||
"拖沓",
|
||||
"寒碜",
|
||||
"糟蹋",
|
||||
"倒腾",
|
||||
"报复",
|
||||
"逻辑",
|
||||
"盘缠",
|
||||
"喽啰",
|
||||
"牢骚",
|
||||
"咖喱",
|
||||
"扫把",
|
||||
"惦记",
|
||||
}
|
||||
self.must_not_neural_tone_words = {
|
||||
"男子",
|
||||
"女子",
|
||||
"分子",
|
||||
"原子",
|
||||
"量子",
|
||||
"莲子",
|
||||
"石子",
|
||||
"瓜子",
|
||||
"电子",
|
||||
"人人",
|
||||
"虎虎",
|
||||
}
|
||||
self.punc = "：，；。？！“”‘’':,;.?!"
|
||||
|
||||
# the meaning of jieba pos tag: https://blog.csdn.net/weixin_44174352/article/details/113731041
|
||||
# e.g.
|
||||
# word: "家里"
|
||||
# pos: "s"
|
||||
# finals: ['ia1', 'i3']
|
||||
def _neural_sandhi(self, word: str, pos: str, finals: List[str]) -> List[str]:
|
||||
# reduplication words for n. and v. e.g. 奶奶, 试试, 旺旺
|
||||
for j, item in enumerate(word):
|
||||
if (
|
||||
j - 1 >= 0
|
||||
and item == word[j - 1]
|
||||
and pos[0] in {"n", "v", "a"}
|
||||
and word not in self.must_not_neural_tone_words
|
||||
):
|
||||
finals[j] = finals[j][:-1] + "5"
|
||||
ge_idx = word.find("个")
|
||||
if len(word) >= 1 and word[-1] in "吧呢啊呐噻嘛吖嗨呐哦哒额滴哩哟喽啰耶喔诶":
|
||||
finals[-1] = finals[-1][:-1] + "5"
|
||||
elif len(word) >= 1 and word[-1] in "的地得":
|
||||
finals[-1] = finals[-1][:-1] + "5"
|
||||
# e.g. 走了, 看着, 去过
|
||||
# elif len(word) == 1 and word in "了着过" and pos in {"ul", "uz", "ug"}:
|
||||
# finals[-1] = finals[-1][:-1] + "5"
|
||||
elif (
|
||||
len(word) > 1
|
||||
and word[-1] in "们子"
|
||||
and pos in {"r", "n"}
|
||||
and word not in self.must_not_neural_tone_words
|
||||
):
|
||||
finals[-1] = finals[-1][:-1] + "5"
|
||||
# e.g. 桌上, 地下, 家里
|
||||
elif len(word) > 1 and word[-1] in "上下里" and pos in {"s", "l", "f"}:
|
||||
finals[-1] = finals[-1][:-1] + "5"
|
||||
# e.g. 上来, 下去
|
||||
elif len(word) > 1 and word[-1] in "来去" and word[-2] in "上下进出回过起开":
|
||||
finals[-1] = finals[-1][:-1] + "5"
|
||||
# 个做量词
|
||||
elif (
|
||||
ge_idx >= 1
|
||||
and (word[ge_idx - 1].isnumeric() or word[ge_idx - 1] in "几有两半多各整每做是")
|
||||
) or word == "个":
|
||||
finals[ge_idx] = finals[ge_idx][:-1] + "5"
|
||||
else:
|
||||
if (
|
||||
word in self.must_neural_tone_words
|
||||
or word[-2:] in self.must_neural_tone_words
|
||||
):
|
||||
finals[-1] = finals[-1][:-1] + "5"
|
||||
|
||||
word_list = self._split_word(word)
|
||||
finals_list = [finals[: len(word_list[0])], finals[len(word_list[0]) :]]
|
||||
for i, word in enumerate(word_list):
|
||||
# conventional neural in Chinese
|
||||
if (
|
||||
word in self.must_neural_tone_words
|
||||
or word[-2:] in self.must_neural_tone_words
|
||||
):
|
||||
finals_list[i][-1] = finals_list[i][-1][:-1] + "5"
|
||||
finals = sum(finals_list, [])
|
||||
return finals
|
||||
|
||||
def _bu_sandhi(self, word: str, finals: List[str]) -> List[str]:
|
||||
# e.g. 看不懂
|
||||
if len(word) == 3 and word[1] == "不":
|
||||
finals[1] = finals[1][:-1] + "5"
|
||||
else:
|
||||
for i, char in enumerate(word):
|
||||
# "不" before tone4 should be bu2, e.g. 不怕
|
||||
if char == "不" and i + 1 < len(word) and finals[i + 1][-1] == "4":
|
||||
finals[i] = finals[i][:-1] + "2"
|
||||
return finals
|
||||
|
||||
def _yi_sandhi(self, word: str, finals: List[str]) -> List[str]:
|
||||
# "一" in number sequences, e.g. 一零零, 二一零
|
||||
if word.find("一") != -1 and all(
|
||||
[item.isnumeric() for item in word if item != "一"]
|
||||
):
|
||||
return finals
|
||||
# "一" between reduplication words should be yi5, e.g. 看一看
|
||||
elif len(word) == 3 and word[1] == "一" and word[0] == word[-1]:
|
||||
finals[1] = finals[1][:-1] + "5"
|
||||
# when "一" is ordinal word, it should be yi1
|
||||
elif word.startswith("第一"):
|
||||
finals[1] = finals[1][:-1] + "1"
|
||||
else:
|
||||
for i, char in enumerate(word):
|
||||
if char == "一" and i + 1 < len(word):
|
||||
# "一" before tone4 should be yi2, e.g. 一段
|
||||
if finals[i + 1][-1] == "4":
|
||||
finals[i] = finals[i][:-1] + "2"
|
||||
# "一" before non-tone4 should be yi4, e.g. 一天
|
||||
else:
|
||||
# "一" 后面如果是标点，还读一声
|
||||
if word[i + 1] not in self.punc:
|
||||
finals[i] = finals[i][:-1] + "4"
|
||||
return finals
|
||||
|
||||
def _split_word(self, word: str) -> List[str]:
|
||||
word_list = jieba.cut_for_search(word)
|
||||
word_list = sorted(word_list, key=lambda i: len(i), reverse=False)
|
||||
first_subword = word_list[0]
|
||||
first_begin_idx = word.find(first_subword)
|
||||
if first_begin_idx == 0:
|
||||
second_subword = word[len(first_subword) :]
|
||||
new_word_list = [first_subword, second_subword]
|
||||
else:
|
||||
second_subword = word[: -len(first_subword)]
|
||||
new_word_list = [second_subword, first_subword]
|
||||
return new_word_list
|
||||
|
||||
def _three_sandhi(self, word: str, finals: List[str]) -> List[str]:
|
||||
if len(word) == 2 and self._all_tone_three(finals):
|
||||
finals[0] = finals[0][:-1] + "2"
|
||||
elif len(word) == 3:
|
||||
word_list = self._split_word(word)
|
||||
if self._all_tone_three(finals):
|
||||
# disyllabic + monosyllabic, e.g. 蒙古/包
|
||||
if len(word_list[0]) == 2:
|
||||
finals[0] = finals[0][:-1] + "2"
|
||||
finals[1] = finals[1][:-1] + "2"
|
||||
# monosyllabic + disyllabic, e.g. 纸/老虎
|
||||
elif len(word_list[0]) == 1:
|
||||
finals[1] = finals[1][:-1] + "2"
|
||||
else:
|
||||
finals_list = [finals[: len(word_list[0])], finals[len(word_list[0]) :]]
|
||||
if len(finals_list) == 2:
|
||||
for i, sub in enumerate(finals_list):
|
||||
# e.g. 所有/人
|
||||
if self._all_tone_three(sub) and len(sub) == 2:
|
||||
finals_list[i][0] = finals_list[i][0][:-1] + "2"
|
||||
# e.g. 好/喜欢
|
||||
elif (
|
||||
i == 1
|
||||
and not self._all_tone_three(sub)
|
||||
and finals_list[i][0][-1] == "3"
|
||||
and finals_list[0][-1][-1] == "3"
|
||||
):
|
||||
finals_list[0][-1] = finals_list[0][-1][:-1] + "2"
|
||||
finals = sum(finals_list, [])
|
||||
# split idiom into two words who's length is 2
|
||||
elif len(word) == 4:
|
||||
finals_list = [finals[:2], finals[2:]]
|
||||
finals = []
|
||||
for sub in finals_list:
|
||||
if self._all_tone_three(sub):
|
||||
sub[0] = sub[0][:-1] + "2"
|
||||
finals += sub
|
||||
|
||||
return finals
|
||||
|
||||
def _all_tone_three(self, finals: List[str]) -> bool:
|
||||
return all(x[-1] == "3" for x in finals)
|
||||
|
||||
# merge "不" and the word behind it
|
||||
# if don't merge, "不" sometimes appears alone according to jieba, which may occur sandhi error
|
||||
def _merge_bu(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
|
||||
new_seg = []
|
||||
last_word = ""
|
||||
for word, pos in seg:
|
||||
if last_word == "不":
|
||||
word = last_word + word
|
||||
if word != "不":
|
||||
new_seg.append((word, pos))
|
||||
last_word = word[:]
|
||||
if last_word == "不":
|
||||
new_seg.append((last_word, "d"))
|
||||
last_word = ""
|
||||
return new_seg
|
||||
|
||||
# function 1: merge "一" and reduplication words in it's left and right, e.g. "听","一","听" ->"听一听"
|
||||
# function 2: merge single "一" and the word behind it
|
||||
# if don't merge, "一" sometimes appears alone according to jieba, which may occur sandhi error
|
||||
# e.g.
|
||||
# input seg: [('听', 'v'), ('一', 'm'), ('听', 'v')]
|
||||
# output seg: [['听一听', 'v']]
|
||||
def _merge_yi(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
|
||||
new_seg = []
|
||||
# function 1
|
||||
for i, (word, pos) in enumerate(seg):
|
||||
if (
|
||||
i - 1 >= 0
|
||||
and word == "一"
|
||||
and i + 1 < len(seg)
|
||||
and seg[i - 1][0] == seg[i + 1][0]
|
||||
and seg[i - 1][1] == "v"
|
||||
):
|
||||
new_seg[i - 1][0] = new_seg[i - 1][0] + "一" + new_seg[i - 1][0]
|
||||
else:
|
||||
if (
|
||||
i - 2 >= 0
|
||||
and seg[i - 1][0] == "一"
|
||||
and seg[i - 2][0] == word
|
||||
and pos == "v"
|
||||
):
|
||||
continue
|
||||
else:
|
||||
new_seg.append([word, pos])
|
||||
seg = new_seg
|
||||
new_seg = []
|
||||
# function 2
|
||||
for i, (word, pos) in enumerate(seg):
|
||||
if new_seg and new_seg[-1][0] == "一":
|
||||
new_seg[-1][0] = new_seg[-1][0] + word
|
||||
else:
|
||||
new_seg.append([word, pos])
|
||||
return new_seg
|
||||
|
||||
# the first and the second words are all_tone_three
|
||||
def _merge_continuous_three_tones(
|
||||
self, seg: List[Tuple[str, str]]
|
||||
) -> List[Tuple[str, str]]:
|
||||
new_seg = []
|
||||
sub_finals_list = [
|
||||
lazy_pinyin(word, neutral_tone_with_five=True, style=Style.FINALS_TONE3)
|
||||
for (word, pos) in seg
|
||||
]
|
||||
assert len(sub_finals_list) == len(seg)
|
||||
merge_last = [False] * len(seg)
|
||||
for i, (word, pos) in enumerate(seg):
|
||||
if (
|
||||
i - 1 >= 0
|
||||
and self._all_tone_three(sub_finals_list[i - 1])
|
||||
and self._all_tone_three(sub_finals_list[i])
|
||||
and not merge_last[i - 1]
|
||||
):
|
||||
# if the last word is reduplication, not merge, because reduplication need to be _neural_sandhi
|
||||
if (
|
||||
not self._is_reduplication(seg[i - 1][0])
|
||||
and len(seg[i - 1][0]) + len(seg[i][0]) <= 3
|
||||
):
|
||||
new_seg[-1][0] = new_seg[-1][0] + seg[i][0]
|
||||
merge_last[i] = True
|
||||
else:
|
||||
new_seg.append([word, pos])
|
||||
else:
|
||||
new_seg.append([word, pos])
|
||||
|
||||
return new_seg
|
||||
|
||||
def _is_reduplication(self, word: str) -> bool:
|
||||
return len(word) == 2 and word[0] == word[1]
|
||||
|
||||
# the last char of first word and the first char of second word is tone_three
|
||||
def _merge_continuous_three_tones_2(
|
||||
self, seg: List[Tuple[str, str]]
|
||||
) -> List[Tuple[str, str]]:
|
||||
new_seg = []
|
||||
sub_finals_list = [
|
||||
lazy_pinyin(word, neutral_tone_with_five=True, style=Style.FINALS_TONE3)
|
||||
for (word, pos) in seg
|
||||
]
|
||||
assert len(sub_finals_list) == len(seg)
|
||||
merge_last = [False] * len(seg)
|
||||
for i, (word, pos) in enumerate(seg):
|
||||
if (
|
||||
i - 1 >= 0
|
||||
and sub_finals_list[i - 1][-1][-1] == "3"
|
||||
and sub_finals_list[i][0][-1] == "3"
|
||||
and not merge_last[i - 1]
|
||||
):
|
||||
# if the last word is reduplication, not merge, because reduplication need to be _neural_sandhi
|
||||
if (
|
||||
not self._is_reduplication(seg[i - 1][0])
|
||||
and len(seg[i - 1][0]) + len(seg[i][0]) <= 3
|
||||
):
|
||||
new_seg[-1][0] = new_seg[-1][0] + seg[i][0]
|
||||
merge_last[i] = True
|
||||
else:
|
||||
new_seg.append([word, pos])
|
||||
else:
|
||||
new_seg.append([word, pos])
|
||||
return new_seg
|
||||
|
||||
def _merge_er(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
|
||||
new_seg = []
|
||||
for i, (word, pos) in enumerate(seg):
|
||||
if i - 1 >= 0 and word == "儿" and seg[i - 1][0] != "#":
|
||||
new_seg[-1][0] = new_seg[-1][0] + seg[i][0]
|
||||
else:
|
||||
new_seg.append([word, pos])
|
||||
return new_seg
|
||||
|
||||
def _merge_reduplication(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
|
||||
new_seg = []
|
||||
for i, (word, pos) in enumerate(seg):
|
||||
if new_seg and word == new_seg[-1][0]:
|
||||
new_seg[-1][0] = new_seg[-1][0] + seg[i][0]
|
||||
else:
|
||||
new_seg.append([word, pos])
|
||||
return new_seg
|
||||
|
||||
def pre_merge_for_modify(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
|
||||
seg = self._merge_bu(seg)
|
||||
try:
|
||||
seg = self._merge_yi(seg)
|
||||
except:
|
||||
print("_merge_yi failed")
|
||||
seg = self._merge_reduplication(seg)
|
||||
seg = self._merge_continuous_three_tones(seg)
|
||||
seg = self._merge_continuous_three_tones_2(seg)
|
||||
seg = self._merge_er(seg)
|
||||
return seg
|
||||
|
||||
def modified_tone(self, word: str, pos: str, finals: List[str]) -> List[str]:
|
||||
finals = self._bu_sandhi(word, finals)
|
||||
finals = self._yi_sandhi(word, finals)
|
||||
finals = self._neural_sandhi(word, pos, finals)
|
||||
finals = self._three_sandhi(word, finals)
|
||||
return finals
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user