From 16af947b6728e04704ed5cc031cbe4f555267f03 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stardust=C2=B7=E5=87=8F?= <2225664821@qq.com>
Date: Tue, 7 Nov 2023 12:51:18 +0800
Subject: [PATCH 1/6] add emo and vo

---
 models.py | 56 ++++++++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 49 insertions(+), 7 deletions(-)

diff --git a/models.py b/models.py
index d53b430..5e98754 100644
--- a/models.py
+++ b/models.py
@@ -10,6 +10,7 @@ import monotonic_align
 
 from torch.nn import Conv1d, ConvTranspose1d, Conv2d
 from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm
+from vector_quantize_pytorch import VectorQuantize
 
 from commons import init_weights, get_padding
 from text import symbols, num_tones, num_languages
@@ -321,6 +322,7 @@ class TextEncoder(nn.Module):
         n_layers,
         kernel_size,
         p_dropout,
+        n_speakers,
         gin_channels=0,
     ):
         super().__init__()
@@ -342,6 +344,18 @@ class TextEncoder(nn.Module):
         self.bert_proj = nn.Conv1d(1024, hidden_channels, 1)
         self.ja_bert_proj = nn.Conv1d(1024, hidden_channels, 1)
         self.en_bert_proj = nn.Conv1d(1024, hidden_channels, 1)
+        self.emo_proj = nn.Linear(1024, 1024)
+        self.emo_quantizer = [
+            VectorQuantize(
+                dim=1024,
+                codebook_size=5,
+                decay=0.8,
+                commitment_weight=1.0,
+                learnable_codebook=True,
+                ema_update=False,
+            )
+        ] * n_speakers
+        self.emo_q_proj = nn.Linear(1024, hidden_channels)
 
         self.encoder = attentions.Encoder(
             hidden_channels,
@@ -354,10 +368,33 @@ class TextEncoder(nn.Module):
         )
         self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
 
-    def forward(self, x, x_lengths, tone, language, bert, ja_bert, en_bert, g=None):
+    def forward(
+        self, x, x_lengths, tone, language, bert, ja_bert, en_bert, emo, sid, g=None
+    ):
+        sid = sid.cpu()
         bert_emb = self.bert_proj(bert).transpose(1, 2)
         ja_bert_emb = self.ja_bert_proj(ja_bert).transpose(1, 2)
         en_bert_emb = self.en_bert_proj(en_bert).transpose(1, 2)
+        if emo.size(-1) == 1024:
+            emo_emb = self.emo_proj(emo.unsqueeze(1))
+            emo_commit_loss = torch.zeros(1)
+            emo_emb_ = []
+            for i in range(emo_emb.size(0)):
+                temp_emo_emb, _, temp_emo_commit_loss = self.emo_quantizer[sid[i]](
+                    emo_emb[i].unsqueeze(0).cpu()
+                )
+                emo_commit_loss += temp_emo_commit_loss
+                emo_emb_.append(temp_emo_emb)
+            emo_emb = torch.cat(emo_emb_, dim=0).to(emo_emb.device)
+            emo_commit_loss = emo_commit_loss.to(emo_emb.device)
+        else:
+            emo_emb = (
+                self.emo_quantizer[sid[0]]
+                .get_output_from_indices(emo.to(torch.int).cpu())
+                .unsqueeze(0)
+                .to(emo.device)
+            )
+            emo_commit_loss = torch.zeros(1)
         x = (
             self.emb(x)
             + self.tone_emb(tone)
@@ -365,6 +402,7 @@ class TextEncoder(nn.Module):
             + bert_emb
             + ja_bert_emb
             + en_bert_emb
+            + self.emo_q_proj(emo_emb)
         ) * math.sqrt(
             self.hidden_channels
         )  # [b, t, h]
@@ -377,7 +415,7 @@ class TextEncoder(nn.Module):
         stats = self.proj(x) * x_mask
 
         m, logs = torch.split(stats, self.out_channels, dim=1)
-        return x, m, logs, x_mask
+        return x, m, logs, x_mask, emo_commit_loss
 
 
 class ResidualCouplingBlock(nn.Module):
@@ -810,6 +848,7 @@ class SynthesizerTrn(nn.Module):
             n_layers,
             kernel_size,
             p_dropout,
+            self.n_speakers,
             gin_channels=self.enc_gin_channels,
         )
         self.dec = Generator(
@@ -860,7 +899,7 @@ class SynthesizerTrn(nn.Module):
             hidden_channels, 256, 3, 0.5, gin_channels=gin_channels
         )
 
-        if n_speakers >= 1:
+        if n_speakers > 1:
             self.emb_g = nn.Embedding(n_speakers, gin_channels)
         else:
             self.ref_enc = ReferenceEncoder(spec_channels, gin_channels)
@@ -877,13 +916,14 @@ class SynthesizerTrn(nn.Module):
         bert,
         ja_bert,
         en_bert,
+        emo=None,
     ):
         if self.n_speakers > 0:
             g = self.emb_g(sid).unsqueeze(-1)  # [b, h, 1]
         else:
             g = self.ref_enc(y.transpose(1, 2)).unsqueeze(-1)
-        x, m_p, logs_p, x_mask = self.enc_p(
-            x, x_lengths, tone, language, bert, ja_bert, en_bert, g=g
+        x, m_p, logs_p, x_mask, loss_commit = self.enc_p(
+            x, x_lengths, tone, language, bert, ja_bert, en_bert, emo, sid, g=g
         )
         z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g)
         z_p = self.flow(z, y_mask, g=g)
@@ -949,6 +989,7 @@ class SynthesizerTrn(nn.Module):
             y_mask,
             (z, z_p, m_p, logs_p, m_q, logs_q),
             (x, logw, logw_),
+            loss_commit,
         )
 
     def infer(
@@ -961,6 +1002,7 @@ class SynthesizerTrn(nn.Module):
         bert,
         ja_bert,
         en_bert,
+        emo=None,
         noise_scale=0.667,
         length_scale=1,
         noise_scale_w=0.8,
@@ -974,8 +1016,8 @@ class SynthesizerTrn(nn.Module):
             g = self.emb_g(sid).unsqueeze(-1)  # [b, h, 1]
         else:
             g = self.ref_enc(y.transpose(1, 2)).unsqueeze(-1)
-        x, m_p, logs_p, x_mask = self.enc_p(
-            x, x_lengths, tone, language, bert, ja_bert, en_bert, g=g
+        x, m_p, logs_p, x_mask, _ = self.enc_p(
+            x, x_lengths, tone, language, bert, ja_bert, en_bert, emo, sid, g=g
         )
         logw = self.sdp(x, x_mask, g=g, reverse=True, noise_scale=noise_scale_w) * (
             sdp_ratio

From 59805037c1c057d7155c1a28b5d9a3071b6753a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stardust=C2=B7=E5=87=8F?= <star_dust_chen@foxmail.com>
Date: Tue, 7 Nov 2023 12:51:59 +0800
Subject: [PATCH 2/6] Update models.py

---
 models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/models.py b/models.py
index 5e98754..d493878 100644
--- a/models.py
+++ b/models.py
@@ -899,7 +899,7 @@ class SynthesizerTrn(nn.Module):
             hidden_channels, 256, 3, 0.5, gin_channels=gin_channels
         )
 
-        if n_speakers > 1:
+        if n_speakers > =1:
             self.emb_g = nn.Embedding(n_speakers, gin_channels)
         else:
             self.ref_enc = ReferenceEncoder(spec_channels, gin_channels)

From c864828afa95833709149d6744cddbd1e3b50d3f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stardust=C2=B7=E5=87=8F?= <star_dust_chen@foxmail.com>
Date: Tue, 7 Nov 2023 12:55:48 +0800
Subject: [PATCH 3/6] add emo

---
 train_ms.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/train_ms.py b/train_ms.py
index 341dcca..67bd1dd 100644
--- a/train_ms.py
+++ b/train_ms.py
@@ -340,6 +340,7 @@ def train_and_evaluate(
         bert,
         ja_bert,
         en_bert,
+        emo,
     ) in tqdm(enumerate(train_loader)):
         if net_g.module.use_noise_scaled_mas:
             current_mas_noise_scale = (
@@ -362,6 +363,7 @@ def train_and_evaluate(
         bert = bert.cuda(rank, non_blocking=True)
         ja_bert = ja_bert.cuda(rank, non_blocking=True)
         en_bert = en_bert.cuda(rank, non_blocking=True)
+        emo = emo.cuda(rank, non_blocking=True)
 
         with autocast(enabled=hps.train.fp16_run):
             (
@@ -384,6 +386,7 @@ def train_and_evaluate(
                 bert,
                 ja_bert,
                 en_bert,
+                emo,
             )
             mel = spec_to_mel_torch(
                 spec,
@@ -579,6 +582,7 @@ def evaluate(hps, generator, eval_loader, writer_eval):
             bert,
             ja_bert,
             en_bert,
+            emo,
         ) in enumerate(eval_loader):
             x, x_lengths = x.cuda(), x_lengths.cuda()
             spec, spec_lengths = spec.cuda(), spec_lengths.cuda()

From 8ea241b57ad9d5479418ced7682acc01e6ebde38 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stardust=C2=B7=E5=87=8F?= <star_dust_chen@foxmail.com>
Date: Tue, 7 Nov 2023 12:58:03 +0800
Subject: [PATCH 4/6] Update loss

---
 train_ms.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/train_ms.py b/train_ms.py
index 67bd1dd..3c25e32 100644
--- a/train_ms.py
+++ b/train_ms.py
@@ -375,6 +375,7 @@ def train_and_evaluate(
                 z_mask,
                 (z, z_p, m_p, logs_p, m_q, logs_q),
                 (hidden_x, logw, logw_),
+                loss_commit,
             ) = net_g(
                 x,
                 x_lengths,
@@ -457,7 +458,9 @@ def train_and_evaluate(
 
                 loss_fm = feature_loss(fmap_r, fmap_g)
                 loss_gen, losses_gen = generator_loss(y_d_hat_g)
-                loss_gen_all = loss_gen + loss_fm + loss_mel + loss_dur + loss_kl
+                loss_gen_all = (
+                    loss_gen + loss_fm + loss_mel + loss_dur + loss_kl + loss_commit
+                )
                 if net_dur_disc is not None:
                     loss_dur_gen, losses_dur_gen = generator_loss(y_dur_hat_g)
                     loss_gen_all += loss_dur_gen

From 9616b7b4b597f3d28474b210038e1ee161195113 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stardust=C2=B7=E5=87=8F?= <star_dust_chen@foxmail.com>
Date: Tue, 7 Nov 2023 12:58:39 +0800
Subject: [PATCH 5/6] Create emo_gen.py

---
 emo_gen.py | 169 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 169 insertions(+)
 create mode 100644 emo_gen.py

diff --git a/emo_gen.py b/emo_gen.py
new file mode 100644
index 0000000..0856ab2
--- /dev/null
+++ b/emo_gen.py
@@ -0,0 +1,169 @@
+import torch
+import torch.nn as nn
+from torch.utils.data import Dataset
+from torch.utils.data import DataLoader
+from transformers import Wav2Vec2Processor
+from transformers.models.wav2vec2.modeling_wav2vec2 import (
+    Wav2Vec2Model,
+    Wav2Vec2PreTrainedModel,
+)
+import librosa
+import numpy as np
+import argparse
+from config import config
+import utils
+import os
+from tqdm import tqdm
+
+
+class RegressionHead(nn.Module):
+    r"""Classification head."""
+
+    def __init__(self, config):
+        super().__init__()
+
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        self.dropout = nn.Dropout(config.final_dropout)
+        self.out_proj = nn.Linear(config.hidden_size, config.num_labels)
+
+    def forward(self, features, **kwargs):
+        x = features
+        x = self.dropout(x)
+        x = self.dense(x)
+        x = torch.tanh(x)
+        x = self.dropout(x)
+        x = self.out_proj(x)
+
+        return x
+
+
+class EmotionModel(Wav2Vec2PreTrainedModel):
+    r"""Speech emotion classifier."""
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        self.config = config
+        self.wav2vec2 = Wav2Vec2Model(config)
+        self.classifier = RegressionHead(config)
+        self.init_weights()
+
+    def forward(
+        self,
+        input_values,
+    ):
+        outputs = self.wav2vec2(input_values)
+        hidden_states = outputs[0]
+        hidden_states = torch.mean(hidden_states, dim=1)
+        logits = self.classifier(hidden_states)
+
+        return hidden_states, logits
+
+
+class AudioDataset(Dataset):
+    def __init__(self, list_of_wav_files, sr, processor):
+        self.list_of_wav_files = list_of_wav_files
+        self.processor = processor
+        self.sr = sr
+
+    def __len__(self):
+        return len(self.list_of_wav_files)
+
+    def __getitem__(self, idx):
+        wav_file = self.list_of_wav_files[idx]
+        audio_data, _ = librosa.load(wav_file, sr=self.sr)
+        processed_data = self.processor(audio_data, sampling_rate=self.sr)[
+            "input_values"
+        ][0]
+        return torch.from_numpy(processed_data)
+
+
+model_name = "./emotional/wav2vec2-large-robust-12-ft-emotion-msp-dim"
+processor = Wav2Vec2Processor.from_pretrained(model_name)
+model = EmotionModel.from_pretrained(model_name)
+
+
+def process_func(
+    x: np.ndarray,
+    sampling_rate: int,
+    model: EmotionModel,
+    processor: Wav2Vec2Processor,
+    device: str,
+    embeddings: bool = False,
+) -> np.ndarray:
+    r"""Predict emotions or extract embeddings from raw audio signal."""
+    model = model.to(device)
+    y = processor(x, sampling_rate=sampling_rate)
+    y = y["input_values"][0]
+    y = torch.from_numpy(y).unsqueeze(0).to(device)
+
+    # run through model
+    with torch.no_grad():
+        y = model(y)[0 if embeddings else 1]
+
+    # convert to numpy
+    y = y.detach().cpu().numpy()
+
+    return y
+
+
+def get_emo(path):
+    wav, sr = librosa.load(path, 16000)
+    device = config.bert_gen_config.device
+    return process_func(
+        np.expand_dims(wav, 0).astype(np.float),
+        sr,
+        model,
+        processor,
+        device,
+        embeddings=True,
+    ).squeeze(0)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-c", "--config", type=str, default=config.bert_gen_config.config_path
+    )
+    parser.add_argument(
+        "--num_processes", type=int, default=config.bert_gen_config.num_processes
+    )
+    args, _ = parser.parse_known_args()
+    config_path = args.config
+    hps = utils.get_hparams_from_file(config_path)
+
+    device = config.bert_gen_config.device
+
+    model_name = "./emotional/wav2vec2-large-robust-12-ft-emotion-msp-dim"
+    processor = (
+        Wav2Vec2Processor.from_pretrained(model_name)
+        if processor is None
+        else processor
+    )
+    model = (
+        EmotionModel.from_pretrained(model_name).to(device)
+        if model is None
+        else model.to(device)
+    )
+
+    lines = []
+    with open(hps.data.training_files, encoding="utf-8") as f:
+        lines.extend(f.readlines())
+
+    with open(hps.data.validation_files, encoding="utf-8") as f:
+        lines.extend(f.readlines())
+
+    wavnames = [line.split("|")[0] for line in lines]
+    dataset = AudioDataset(wavnames, 16000, processor)
+    data_loader = DataLoader(dataset, batch_size=1, shuffle=False, num_workers=16)
+
+    with torch.no_grad():
+        for i, data in tqdm(enumerate(data_loader), total=len(data_loader)):
+            wavname = wavnames[i]
+            emo_path = wavname.replace(".wav", ".emo.npy")
+            if os.path.exists(emo_path):
+                continue
+            emb = model(data.to(device))[0].detach().cpu().numpy()
+            np.save(emo_path, emb)
+
+    print("Emo vec 生成完毕!")

From 33d153505a3c25fdd1f70f513b2877b18c6476ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stardust=C2=B7=E5=87=8F?= <star_dust_chen@foxmail.com>
Date: Wed, 8 Nov 2023 17:09:15 +0800
Subject: [PATCH 6/6] sync emo branch (#159)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Update README.md

* 更新 bert_models.json

* fix

* Update data_utils.py

* Update infer.py

* performance improve

* Feat: support auto split in webui (#158)

* Feat: support auto split in webui

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

---------

Co-authored-by: Sora <atri@suzakuintsubaki.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
---
 README.md             |  4 +++-
 bert/bert_models.json |  2 +-
 data_utils.py         | 24 +++++-------------------
 infer.py              | 10 +++++-----
 server_fastapi.py     |  2 ++
 tools/log.py          |  4 +++-
 train_ms.py           |  4 ++--
 7 files changed, 21 insertions(+), 29 deletions(-)

diff --git a/README.md b/README.md
index 0e9edbf..a76aa78 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,9 @@
 # Bert-VITS2
 
 VITS2 Backbone with bert
-
+# 紧急通知
+我们在2.0版本中发现了重大bug，该bug导致日文和英文bert被置0后训练，即失去bert效果。
+我们将重炼2.0版本底模，已经开炉的建议关炉静候。
 ## 请注意，本项目核心思路来源于[anyvoiceai/MassTTS](https://github.com/anyvoiceai/MassTTS) 一个非常好的tts项目
 ## MassTTS的演示demo为[ai版峰哥锐评峰哥本人,并找回了在金三角失落的腰子](https://www.bilibili.com/video/BV1w24y1c7z9)
 
diff --git a/bert/bert_models.json b/bert/bert_models.json
index 61d4e6a..78721c3 100644
--- a/bert/bert_models.json
+++ b/bert/bert_models.json
@@ -1,7 +1,7 @@
 {
     "deberta-v2-large-japanese": {
         "repo_id": "ku-nlp/deberta-v2-large-japanese",
-        "files": ["spm.model", "pytorch_model.bin"]
+        "files": ["pytorch_model.bin"]
     },
     "chinese-roberta-wwm-ext-large": {
         "repo_id": "hfl/chinese-roberta-wwm-ext-large",
diff --git a/data_utils.py b/data_utils.py
index d6526c4..1e297b2 100644
--- a/data_utils.py
+++ b/data_utils.py
@@ -145,38 +145,24 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
             word2ph[0] += 1
         bert_path = wav_path.replace(".wav", ".bert.pt")
         try:
-            bert = torch.load(bert_path)
-            assert bert.shape[-1] == len(phone)
+            bert_ori = torch.load(bert_path)
+            assert bert_ori.shape[-1] == len(phone)
         except Exception as e:
             logger.warn("Bert load Failed")
             logger.warn(e)
 
         if language_str == "ZH":
-            bert = bert
+            bert = bert_ori
             ja_bert = torch.zeros(1024, len(phone))
             en_bert = torch.zeros(1024, len(phone))
         elif language_str == "JP":
             bert = torch.zeros(1024, len(phone))
-            ja_bert = bert
+            ja_bert = bert_ori
             en_bert = torch.zeros(1024, len(phone))
         elif language_str == "EN":
             bert = torch.zeros(1024, len(phone))
             ja_bert = torch.zeros(1024, len(phone))
-            en_bert = bert
-        assert bert.shape[-1] == len(phone), (
-            bert.shape,
-            len(phone),
-            sum(word2ph),
-            p1,
-            p2,
-            t1,
-            t2,
-            pold,
-            pold2,
-            word2ph,
-            text,
-            w2pho,
-        )
+            en_bert = bert_ori
         phone = torch.LongTensor(phone)
         tone = torch.LongTensor(tone)
         language = torch.LongTensor(language)
diff --git a/infer.py b/infer.py
index 8497998..441e52c 100644
--- a/infer.py
+++ b/infer.py
@@ -85,22 +85,22 @@ def get_text(text, language_str, hps, device):
         for i in range(len(word2ph)):
             word2ph[i] = word2ph[i] * 2
         word2ph[0] += 1
-    bert = get_bert(norm_text, word2ph, language_str, device)
+    bert_ori = get_bert(norm_text, word2ph, language_str, device)
     del word2ph
-    assert bert.shape[-1] == len(phone), phone
+    assert bert_ori.shape[-1] == len(phone), phone
 
     if language_str == "ZH":
-        bert = bert
+        bert = bert_ori
         ja_bert = torch.zeros(1024, len(phone))
         en_bert = torch.zeros(1024, len(phone))
     elif language_str == "JP":
         bert = torch.zeros(1024, len(phone))
-        ja_bert = bert
+        ja_bert = bert_ori
         en_bert = torch.zeros(1024, len(phone))
     elif language_str == "EN":
         bert = torch.zeros(1024, len(phone))
         ja_bert = torch.zeros(1024, len(phone))
-        en_bert = bert
+        en_bert = bert_ori
     else:
         raise ValueError("language_str should be ZH, JP or EN")
 
diff --git a/server_fastapi.py b/server_fastapi.py
index 1f22860..00debb3 100644
--- a/server_fastapi.py
+++ b/server_fastapi.py
@@ -5,6 +5,7 @@ import logging
 import gc
 import random
 
+import gradio
 import numpy as np
 import utils
 from fastapi import FastAPI, Query, Request
@@ -245,6 +246,7 @@ if __name__ == "__main__":
                     )
                 audios.append(np.zeros((int)(44100 * 0.3)))
                 audio = np.concatenate(audios)
+                audio = gradio.processing_utils.convert_to_16_bit_wav(audio)
         wavContent = BytesIO()
         wavfile.write(
             wavContent, loaded_models.models[model_id].hps.data.sampling_rate, audio
diff --git a/tools/log.py b/tools/log.py
index 626b9e9..47e51ae 100644
--- a/tools/log.py
+++ b/tools/log.py
@@ -9,6 +9,8 @@ import sys
 logger.remove()
 
 # 自定义格式并添加到标准输出
-log_format = "<g>{time:MM-DD HH:mm:ss}</g> [<lvl>{level}</lvl>] | {message}"
+log_format = (
+    "<g>{time:MM-DD HH:mm:ss}</g> [<lvl>{level}</lvl>] | {file}:{line} | {message}"
+)
 
 logger.add(sys.stdout, format=log_format)
diff --git a/train_ms.py b/train_ms.py
index 3c25e32..37933d1 100644
--- a/train_ms.py
+++ b/train_ms.py
@@ -206,8 +206,8 @@ def run():
         )
     else:
         optim_dur_disc = None
-    net_g = DDP(net_g, device_ids=[rank], find_unused_parameters=True)
-    net_d = DDP(net_d, device_ids=[rank], find_unused_parameters=True)
+    net_g = DDP(net_g, device_ids=[rank])
+    net_d = DDP(net_d, device_ids=[rank])
     dur_resume_lr = None
     if net_dur_disc is not None:
         net_dur_disc = DDP(net_dur_disc, device_ids=[rank], find_unused_parameters=True)