Dev 2.3. (#242)

* Fix inputs of duration discriminator

* Add LSTM

* Update models.py

* Update tensorboard scalar

* Noise injection for minimizing modality gap

* Update infer.py

* support bf16 run

* del unused_para flag

* support bf16 config

* add grad clip

* fix(logger and grad):add dur grad,fix grad clip

* Update webui_preprocess.py

* Fix English G2P

* fix(bert_gen):add pass

* Pass SDP to DD

* Update webui_preprocess.py

* Update config.json

* Update webui.py

* Update chinese_bert.py

* Upload webui for deploy

* Update webui.py

* torch.save as pt not npy

* Update config.json

* add freeze emo vq

* Update webui_preprocess.py

* Fix tone_sandhi.py

* Comment up grad clip

* Fix in-place addition

* Add SLM discriminator

* Add DDP for WD

* Feat: Style text: make emotions and style similar to the style text by mixing bert (#240) (#241)

* fix:(oldVersion210) Load on demand Emotion model

* feat: update fastapi.py. 添加更多错误日志信息

* Switch pyopenjtalk to pyopenjtalk-prebuilt

* fix: update fastapi.py. 2.2 reference适配

* Update resample.py

* 修复Onnx导出的BUG (#237)

* Add files via upload

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Add files via upload

* Add files via upload

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Delete attentions_onnx.py

* Delete models_onnx.py

* Add files via upload

* Add files via upload

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Update __init__.py

* Update __init__.py

* Update __init__.py

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------



* Fix onnx

* Format export

* Feat: style-text and bert mixing (JA only)

* Ensure the same tensor shape

* Update

* update gradio version

* Fix

* Style text for chinese and english (ver 2.2)

* Style text for chinese and english (ver 2.1)

* Style text in FastAPI

* Translate style text desc in chinese

---------

Co-authored-by: litagin02 <139731664+litagin02@users.noreply.github.com>
Co-authored-by: Sora <654163754@qq.com>
Co-authored-by: Sihan Wang <wangsihan1995@gmail.com>
Co-authored-by: Ναρουσέ·μ·γιουμεμί·Χινακάννα <40709280+NaruseMioShirakana@users.noreply.github.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

* Remove CLAP

* Revert "Remove CLAP"

This reverts commit 62fd59bc837c580239840a2bc84b15e0663730fc.

Revert

* Remove CLAP

* bf16 audo grad cilp

* Update webui and infer utils

* Update webui.py

* Update webui.py

* Update webui-preprocess.py

* Update webui_preprocess.py

---------

Co-authored-by: Sihan Wang <wangsihan1995@gmail.com>
Co-authored-by: OedoSoldier <31711261+OedoSoldier@users.noreply.github.com>
Co-authored-by: litagin02 <139731664+litagin02@users.noreply.github.com>
Co-authored-by: Sora <654163754@qq.com>
Co-authored-by: Ναρουσέ·μ·γιουμεμί·Χινακάννα <40709280+NaruseMioShirakana@users.noreply.github.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.gitignore

@@ -170,6 +170,7 @@ data/*
 !/default_config.yml
 /Web/
 /emotional/*/*.bin
+/slm/*/*.bin
 /bert/*/*.bin
 /bert/*/*.h5
 /bert/*/*.model

bert_gen.py

@@ -1,17 +1,16 @@
-import argparse
-from multiprocessing import Pool, cpu_count
-
 import torch
-import torch.multiprocessing as mp
+from multiprocessing import Pool
-from tqdm import tqdm
-
 import commons
 import utils
+from tqdm import tqdm
+from text import check_bert_models, cleaned_text_to_sequence, get_bert
+import argparse
+import torch.multiprocessing as mp
 from config import config
-from text import cleaned_text_to_sequence, get_bert
 
 
-def process_line(line):
+def process_line(x):
+    line, add_blank = x
     device = config.bert_gen_config.device
     if config.bert_gen_config.use_multi_device:
         rank = mp.current_process()._identity
@@ -28,12 +27,13 @@ def process_line(line):
     word2ph = [i for i in word2ph]
     phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
 
-    phone = commons.intersperse(phone, 0)
+    if add_blank:
-    tone = commons.intersperse(tone, 0)
+        phone = commons.intersperse(phone, 0)
-    language = commons.intersperse(language, 0)
+        tone = commons.intersperse(tone, 0)
-    for i in range(len(word2ph)):
+        language = commons.intersperse(language, 0)
-        word2ph[i] = word2ph[i] * 2
+        for i in range(len(word2ph)):
-    word2ph[0] += 1
+            word2ph[i] = word2ph[i] * 2
+        word2ph[0] += 1
 
     bert_path = wav_path.replace(".WAV", ".wav").replace(".wav", ".bert.pt")
 
@@ -59,16 +59,23 @@ if __name__ == "__main__":
     args, _ = parser.parse_known_args()
     config_path = args.config
     hps = utils.get_hparams_from_file(config_path)
+    check_bert_models()
     lines = []
     with open(hps.data.training_files, encoding="utf-8") as f:
         lines.extend(f.readlines())
 
     with open(hps.data.validation_files, encoding="utf-8") as f:
         lines.extend(f.readlines())
+    add_blank = [hps.data.add_blank] * len(lines)
+
     if len(lines) != 0:
-        num_processes = min(args.num_processes, cpu_count())
+        num_processes = args.num_processes
         with Pool(processes=num_processes) as pool:
-            for _ in tqdm(pool.imap_unordered(process_line, lines), total=len(lines)):
+            for _ in tqdm(
-                pass
+                pool.imap_unordered(process_line, zip(lines, add_blank)),
+                total=len(lines),
+            ):
+                # 这里是缩进的代码块，表示循环体
+                pass  # 使用pass语句作为占位符
 
     print(f"bert生成完毕!, 共有{len(lines)}个bert.pt生成!")

clap_gen.py

@@ -27,7 +27,7 @@ def process_line(line):
             device = torch.device("cpu")
     wav_path, _, language_str, text, phones, tone, word2ph = line.strip().split("|")
 
-    clap_path = wav_path.replace(".WAV", ".wav").replace(".wav", ".emo.npy")
+    clap_path = wav_path.replace(".WAV", ".wav").replace(".wav", ".emo.pt")
     if os.path.isfile(clap_path):
         return
 

data_utils.py

@@ -44,10 +44,6 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
         self.min_text_len = getattr(hparams, "min_text_len", 1)
         self.max_text_len = getattr(hparams, "max_text_len", 384)
 
-        self.empty_emo = torch.squeeze(
-            torch.load("empty_emo.npy", map_location="cpu"), dim=1
-        )
-
         random.seed(1234)
         random.shuffle(self.audiopaths_sid_text)
         self._filter()
@@ -98,14 +94,7 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
         spec, wav = self.get_audio(audiopath)
         sid = torch.LongTensor([int(self.spk_map[sid])])
 
-        if np.random.rand() > 0.1:
+        return (phones, spec, wav, sid, tone, language, bert, ja_bert, en_bert)
-            emo = torch.squeeze(
-                torch.load(audiopath.replace(".wav", ".emo.npy"), map_location="cpu"),
-                dim=1,
-            )
-        else:
-            emo = self.empty_emo
-        return (phones, spec, wav, sid, tone, language, bert, ja_bert, en_bert, emo)
 
     def get_audio(self, filename):
         audio, sampling_rate = load_wav_to_torch(filename)
@@ -168,15 +157,15 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
 
         if language_str == "ZH":
             bert = bert_ori
-            ja_bert = torch.rand(1024, len(phone))
+            ja_bert = torch.randn(1024, len(phone))
-            en_bert = torch.rand(1024, len(phone))
+            en_bert = torch.randn(1024, len(phone))
         elif language_str == "JP":
-            bert = torch.rand(1024, len(phone))
+            bert = torch.randn(1024, len(phone))
             ja_bert = bert_ori
-            en_bert = torch.rand(1024, len(phone))
+            en_bert = torch.randn(1024, len(phone))
         elif language_str == "EN":
-            bert = torch.rand(1024, len(phone))
+            bert = torch.randn(1024, len(phone))
-            ja_bert = torch.rand(1024, len(phone))
+            ja_bert = torch.randn(1024, len(phone))
             en_bert = bert_ori
         phone = torch.LongTensor(phone)
         tone = torch.LongTensor(tone)
@@ -226,7 +215,6 @@ class TextAudioSpeakerCollate:
         bert_padded = torch.FloatTensor(len(batch), 1024, max_text_len)
         ja_bert_padded = torch.FloatTensor(len(batch), 1024, max_text_len)
         en_bert_padded = torch.FloatTensor(len(batch), 1024, max_text_len)
-        emo = torch.FloatTensor(len(batch), 512)
 
         spec_padded = torch.FloatTensor(len(batch), batch[0][1].size(0), max_spec_len)
         wav_padded = torch.FloatTensor(len(batch), 1, max_wav_len)
@@ -238,7 +226,6 @@ class TextAudioSpeakerCollate:
         bert_padded.zero_()
         ja_bert_padded.zero_()
         en_bert_padded.zero_()
-        emo.zero_()
 
         for i in range(len(ids_sorted_decreasing)):
             row = batch[ids_sorted_decreasing[i]]
@@ -272,8 +259,6 @@ class TextAudioSpeakerCollate:
             en_bert = row[8]
             en_bert_padded[i, :, : en_bert.size(1)] = en_bert
 
-            emo[i, :] = row[9]
-
         return (
             text_padded,
             text_lengths,
@@ -287,7 +272,6 @@ class TextAudioSpeakerCollate:
             bert_padded,
             ja_bert_padded,
             en_bert_padded,
-            emo,
         )
 
 

for_deploy/infer.py

@@ -0,0 +1,386 @@
+"""
+版本管理、兼容推理及模型加载实现。
+版本说明：
+    1. 版本号与github的release版本号对应，使用哪个release版本训练的模型即对应其版本号
+    2. 请在模型的config.json中显示声明版本号，添加一个字段"version" : "你的版本号"
+特殊版本说明：
+    1.1.1-fix： 1.1.1版本训练的模型，但是在推理时使用dev的日语修复
+    2.2：当前版本
+"""
+import torch
+import commons
+from text import cleaned_text_to_sequence, get_bert
+from clap_wrapper import get_clap_audio_feature, get_clap_text_feature
+from text.cleaner import clean_text
+import utils
+import numpy as np
+
+from models import SynthesizerTrn
+from text.symbols import symbols
+
+from oldVersion.V210.models import SynthesizerTrn as V210SynthesizerTrn
+from oldVersion.V210.text import symbols as V210symbols
+from oldVersion.V200.models import SynthesizerTrn as V200SynthesizerTrn
+from oldVersion.V200.text import symbols as V200symbols
+from oldVersion.V111.models import SynthesizerTrn as V111SynthesizerTrn
+from oldVersion.V111.text import symbols as V111symbols
+from oldVersion.V110.models import SynthesizerTrn as V110SynthesizerTrn
+from oldVersion.V110.text import symbols as V110symbols
+from oldVersion.V101.models import SynthesizerTrn as V101SynthesizerTrn
+from oldVersion.V101.text import symbols as V101symbols
+
+from oldVersion import V111, V110, V101, V200, V210
+
+# 当前版本信息
+latest_version = "2.2"
+
+# 版本兼容
+SynthesizerTrnMap = {
+    "2.1": V210SynthesizerTrn,
+    "2.0.2-fix": V200SynthesizerTrn,
+    "2.0.1": V200SynthesizerTrn,
+    "2.0": V200SynthesizerTrn,
+    "1.1.1-fix": V111SynthesizerTrn,
+    "1.1.1": V111SynthesizerTrn,
+    "1.1": V110SynthesizerTrn,
+    "1.1.0": V110SynthesizerTrn,
+    "1.0.1": V101SynthesizerTrn,
+    "1.0": V101SynthesizerTrn,
+    "1.0.0": V101SynthesizerTrn,
+}
+
+symbolsMap = {
+    "2.1": V210symbols,
+    "2.0.2-fix": V200symbols,
+    "2.0.1": V200symbols,
+    "2.0": V200symbols,
+    "1.1.1-fix": V111symbols,
+    "1.1.1": V111symbols,
+    "1.1": V110symbols,
+    "1.1.0": V110symbols,
+    "1.0.1": V101symbols,
+    "1.0": V101symbols,
+    "1.0.0": V101symbols,
+}
+
+
+# def get_emo_(reference_audio, emotion, sid):
+#     emo = (
+#         torch.from_numpy(get_emo(reference_audio))
+#         if reference_audio and emotion == -1
+#         else torch.FloatTensor(
+#             np.load(f"emo_clustering/{sid}/cluster_center_{emotion}.npy")
+#         )
+#     )
+#     return emo
+
+
+def get_net_g(model_path: str, version: str, device: str, hps):
+    if version != latest_version:
+        net_g = SynthesizerTrnMap[version](
+            len(symbolsMap[version]),
+            hps.data.filter_length // 2 + 1,
+            hps.train.segment_size // hps.data.hop_length,
+            n_speakers=hps.data.n_speakers,
+            **hps.model,
+        ).to(device)
+    else:
+        # 当前版本模型 net_g
+        net_g = SynthesizerTrn(
+            len(symbols),
+            hps.data.filter_length // 2 + 1,
+            hps.train.segment_size // hps.data.hop_length,
+            n_speakers=hps.data.n_speakers,
+            **hps.model,
+        ).to(device)
+    _ = net_g.eval()
+    _ = utils.load_checkpoint(model_path, net_g, None, skip_optimizer=True)
+    return net_g
+
+
+def get_text(text, language_str, bert, hps, device):
+    # 在此处实现当前版本的get_text
+    norm_text, phone, tone, word2ph = clean_text(text, language_str)
+    phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
+
+    if hps.data.add_blank:
+        phone = commons.intersperse(phone, 0)
+        tone = commons.intersperse(tone, 0)
+        language = commons.intersperse(language, 0)
+        for i in range(len(word2ph)):
+            word2ph[i] = word2ph[i] * 2
+        word2ph[0] += 1
+    # bert_ori = get_bert(norm_text, word2ph, language_str, device)
+    bert_ori = bert[language_str].get_bert_feature(norm_text, word2ph, device)
+    del word2ph
+    assert bert_ori.shape[-1] == len(phone), phone
+
+    if language_str == "ZH":
+        bert = bert_ori
+        ja_bert = torch.randn(1024, len(phone))
+        en_bert = torch.randn(1024, len(phone))
+    elif language_str == "JP":
+        bert = torch.randn(1024, len(phone))
+        ja_bert = bert_ori
+        en_bert = torch.randn(1024, len(phone))
+    elif language_str == "EN":
+        bert = torch.randn(1024, len(phone))
+        ja_bert = torch.randn(1024, len(phone))
+        en_bert = bert_ori
+    else:
+        raise ValueError("language_str should be ZH, JP or EN")
+
+    assert bert.shape[-1] == len(
+        phone
+    ), f"Bert seq len {bert.shape[-1]} != {len(phone)}"
+
+    phone = torch.LongTensor(phone)
+    tone = torch.LongTensor(tone)
+    language = torch.LongTensor(language)
+    return bert, ja_bert, en_bert, phone, tone, language
+
+
+def infer(
+    text,
+    emotion,
+    sdp_ratio,
+    noise_scale,
+    noise_scale_w,
+    length_scale,
+    sid,
+    language,
+    hps,
+    net_g,
+    device,
+    bert=None,
+    clap=None,
+    reference_audio=None,
+    skip_start=False,
+    skip_end=False,
+):
+    # 2.2版本参数位置变了
+    # 2.1 参数新增 emotion reference_audio skip_start skip_end
+    inferMap_V3 = {
+        "2.1": V210.infer,
+    }
+    # 支持中日英三语版本
+    inferMap_V2 = {
+        "2.0.2-fix": V200.infer,
+        "2.0.1": V200.infer,
+        "2.0": V200.infer,
+        "1.1.1-fix": V111.infer_fix,
+        "1.1.1": V111.infer,
+        "1.1": V110.infer,
+        "1.1.0": V110.infer,
+    }
+    # 仅支持中文版本
+    # 在测试中，并未发现两个版本的模型不能互相通用
+    inferMap_V1 = {
+        "1.0.1": V101.infer,
+        "1.0": V101.infer,
+        "1.0.0": V101.infer,
+    }
+    version = hps.version if hasattr(hps, "version") else latest_version
+    # 非当前版本，根据版本号选择合适的infer
+    if version != latest_version:
+        if version in inferMap_V3.keys():
+            return inferMap_V3[version](
+                text,
+                sdp_ratio,
+                noise_scale,
+                noise_scale_w,
+                length_scale,
+                sid,
+                language,
+                hps,
+                net_g,
+                device,
+                reference_audio,
+                emotion,
+                skip_start,
+                skip_end,
+            )
+        if version in inferMap_V2.keys():
+            return inferMap_V2[version](
+                text,
+                sdp_ratio,
+                noise_scale,
+                noise_scale_w,
+                length_scale,
+                sid,
+                language,
+                hps,
+                net_g,
+                device,
+            )
+        if version in inferMap_V1.keys():
+            return inferMap_V1[version](
+                text,
+                sdp_ratio,
+                noise_scale,
+                noise_scale_w,
+                length_scale,
+                sid,
+                hps,
+                net_g,
+                device,
+            )
+    # 在此处实现当前版本的推理
+    # emo = get_emo_(reference_audio, emotion, sid)
+    if isinstance(reference_audio, np.ndarray):
+        emo = clap.get_clap_audio_feature(reference_audio, device)
+    else:
+        emo = clap.get_clap_text_feature(emotion, device)
+    emo = torch.squeeze(emo, dim=1)
+
+    bert, ja_bert, en_bert, phones, tones, lang_ids = get_text(
+        text, language, bert, hps, device
+    )
+    if skip_start:
+        phones = phones[3:]
+        tones = tones[3:]
+        lang_ids = lang_ids[3:]
+        bert = bert[:, 3:]
+        ja_bert = ja_bert[:, 3:]
+        en_bert = en_bert[:, 3:]
+    if skip_end:
+        phones = phones[:-2]
+        tones = tones[:-2]
+        lang_ids = lang_ids[:-2]
+        bert = bert[:, :-2]
+        ja_bert = ja_bert[:, :-2]
+        en_bert = en_bert[:, :-2]
+    with torch.no_grad():
+        x_tst = phones.to(device).unsqueeze(0)
+        tones = tones.to(device).unsqueeze(0)
+        lang_ids = lang_ids.to(device).unsqueeze(0)
+        bert = bert.to(device).unsqueeze(0)
+        ja_bert = ja_bert.to(device).unsqueeze(0)
+        en_bert = en_bert.to(device).unsqueeze(0)
+        x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
+        emo = emo.to(device).unsqueeze(0)
+        del phones
+        speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
+        audio = (
+            net_g.infer(
+                x_tst,
+                x_tst_lengths,
+                speakers,
+                tones,
+                lang_ids,
+                bert,
+                ja_bert,
+                en_bert,
+                emo,
+                sdp_ratio=sdp_ratio,
+                noise_scale=noise_scale,
+                noise_scale_w=noise_scale_w,
+                length_scale=length_scale,
+            )[0][0, 0]
+            .data.cpu()
+            .float()
+            .numpy()
+        )
+        del x_tst, tones, lang_ids, bert, x_tst_lengths, speakers, ja_bert, en_bert, emo
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        return audio
+
+
+def infer_multilang(
+    text,
+    sdp_ratio,
+    noise_scale,
+    noise_scale_w,
+    length_scale,
+    sid,
+    language,
+    hps,
+    net_g,
+    device,
+    bert=None,
+    clap=None,
+    reference_audio=None,
+    emotion=None,
+    skip_start=False,
+    skip_end=False,
+):
+    bert, ja_bert, en_bert, phones, tones, lang_ids = [], [], [], [], [], []
+    # emo = get_emo_(reference_audio, emotion, sid)
+    if isinstance(reference_audio, np.ndarray):
+        emo = clap.get_clap_audio_feature(reference_audio, device)
+    else:
+        emo = clap.get_clap_text_feature(emotion, device)
+    emo = torch.squeeze(emo, dim=1)
+    for idx, (txt, lang) in enumerate(zip(text, language)):
+        skip_start = (idx != 0) or (skip_start and idx == 0)
+        skip_end = (idx != len(text) - 1) or (skip_end and idx == len(text) - 1)
+        (
+            temp_bert,
+            temp_ja_bert,
+            temp_en_bert,
+            temp_phones,
+            temp_tones,
+            temp_lang_ids,
+        ) = get_text(txt, lang, bert, hps, device)
+        if skip_start:
+            temp_bert = temp_bert[:, 3:]
+            temp_ja_bert = temp_ja_bert[:, 3:]
+            temp_en_bert = temp_en_bert[:, 3:]
+            temp_phones = temp_phones[3:]
+            temp_tones = temp_tones[3:]
+            temp_lang_ids = temp_lang_ids[3:]
+        if skip_end:
+            temp_bert = temp_bert[:, :-2]
+            temp_ja_bert = temp_ja_bert[:, :-2]
+            temp_en_bert = temp_en_bert[:, :-2]
+            temp_phones = temp_phones[:-2]
+            temp_tones = temp_tones[:-2]
+            temp_lang_ids = temp_lang_ids[:-2]
+        bert.append(temp_bert)
+        ja_bert.append(temp_ja_bert)
+        en_bert.append(temp_en_bert)
+        phones.append(temp_phones)
+        tones.append(temp_tones)
+        lang_ids.append(temp_lang_ids)
+    bert = torch.concatenate(bert, dim=1)
+    ja_bert = torch.concatenate(ja_bert, dim=1)
+    en_bert = torch.concatenate(en_bert, dim=1)
+    phones = torch.concatenate(phones, dim=0)
+    tones = torch.concatenate(tones, dim=0)
+    lang_ids = torch.concatenate(lang_ids, dim=0)
+    with torch.no_grad():
+        x_tst = phones.to(device).unsqueeze(0)
+        tones = tones.to(device).unsqueeze(0)
+        lang_ids = lang_ids.to(device).unsqueeze(0)
+        bert = bert.to(device).unsqueeze(0)
+        ja_bert = ja_bert.to(device).unsqueeze(0)
+        en_bert = en_bert.to(device).unsqueeze(0)
+        emo = emo.to(device).unsqueeze(0)
+        x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
+        del phones
+        speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
+        audio = (
+            net_g.infer(
+                x_tst,
+                x_tst_lengths,
+                speakers,
+                tones,
+                lang_ids,
+                bert,
+                ja_bert,
+                en_bert,
+                emo,
+                sdp_ratio=sdp_ratio,
+                noise_scale=noise_scale,
+                noise_scale_w=noise_scale_w,
+                length_scale=length_scale,
+            )[0][0, 0]
+            .data.cpu()
+            .float()
+            .numpy()
+        )
+        del x_tst, tones, lang_ids, bert, x_tst_lengths, speakers, ja_bert, en_bert, emo
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        return audio

for_deploy/infer_utils.py

@@ -0,0 +1,111 @@
+import sys
+
+import torch
+from transformers import (
+    AutoModelForMaskedLM,
+    AutoTokenizer,
+    DebertaV2Model,
+    DebertaV2Tokenizer,
+    ClapModel,
+    ClapProcessor,
+)
+
+from config import config
+from text.japanese import text2sep_kata
+
+
+class BertFeature:
+    def __init__(self, model_path, language="ZH"):
+        self.model_path = model_path
+        self.language = language
+        self.tokenizer = None
+        self.model = None
+        self.device = None
+
+        self._prepare()
+
+    def _get_device(self, device=config.bert_gen_config.device):
+        if (
+            sys.platform == "darwin"
+            and torch.backends.mps.is_available()
+            and device == "cpu"
+        ):
+            device = "mps"
+        if not device:
+            device = "cuda"
+        return device
+
+    def _prepare(self):
+        self.device = self._get_device()
+
+        if self.language == "EN":
+            self.tokenizer = DebertaV2Tokenizer.from_pretrained(self.model_path)
+            self.model = DebertaV2Model.from_pretrained(self.model_path).to(self.device)
+        else:
+            self.tokenizer = AutoTokenizer.from_pretrained(self.model_path)
+            self.model = AutoModelForMaskedLM.from_pretrained(self.model_path).to(
+                self.device
+            )
+        self.model.eval()
+
+    def get_bert_feature(self, text, word2ph):
+        if self.language == "JP":
+            text = "".join(text2sep_kata(text)[0])
+        with torch.no_grad():
+            inputs = self.tokenizer(text, return_tensors="pt")
+            for i in inputs:
+                inputs[i] = inputs[i].to(self.device)
+            res = self.model(**inputs, output_hidden_states=True)
+            res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
+
+        word2phone = word2ph
+        phone_level_feature = []
+        for i in range(len(word2phone)):
+            repeat_feature = res[i].repeat(word2phone[i], 1)
+            phone_level_feature.append(repeat_feature)
+
+        phone_level_feature = torch.cat(phone_level_feature, dim=0)
+
+        return phone_level_feature.T
+
+
+class ClapFeature:
+    def __init__(self, model_path):
+        self.model_path = model_path
+        self.processor = None
+        self.model = None
+        self.device = None
+
+        self._prepare()
+
+    def _get_device(self, device=config.bert_gen_config.device):
+        if (
+            sys.platform == "darwin"
+            and torch.backends.mps.is_available()
+            and device == "cpu"
+        ):
+            device = "mps"
+        if not device:
+            device = "cuda"
+        return device
+
+    def _prepare(self):
+        self.device = self._get_device()
+
+        self.processor = ClapProcessor.from_pretrained(self.model_path)
+        self.model = ClapModel.from_pretrained(self.model_path).to(self.device)
+        self.model.eval()
+
+    def get_clap_audio_feature(self, audio_data):
+        with torch.no_grad():
+            inputs = self.processor(
+                audios=audio_data, return_tensors="pt", sampling_rate=48000
+            ).to(self.device)
+            emb = self.model.get_audio_features(**inputs)
+        return emb.T
+
+    def get_clap_text_feature(self, text):
+        with torch.no_grad():
+            inputs = self.processor(text=text, return_tensors="pt").to(self.device)
+            emb = self.model.get_text_features(**inputs)
+        return emb.T

for_deploy/webui.py

@@ -0,0 +1,556 @@
+# flake8: noqa: E402
+import os
+import logging
+import re_matching
+from tools.sentence import split_by_language
+
+logging.getLogger("numba").setLevel(logging.WARNING)
+logging.getLogger("markdown_it").setLevel(logging.WARNING)
+logging.getLogger("urllib3").setLevel(logging.WARNING)
+logging.getLogger("matplotlib").setLevel(logging.WARNING)
+
+logging.basicConfig(
+    level=logging.INFO, format="| %(name)s | %(levelname)s | %(message)s"
+)
+
+logger = logging.getLogger(__name__)
+
+import torch
+import utils
+from infer import infer, latest_version, get_net_g, infer_multilang
+import gradio as gr
+import webbrowser
+import numpy as np
+from config import config
+from tools.translate import translate
+import librosa
+from infer_utils import BertFeature, ClapFeature
+
+
+net_g = None
+
+device = config.webui_config.device
+if device == "mps":
+    os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
+
+os.environ["OMP_NUM_THREADS"] = "1"
+os.environ["MKL_NUM_THREADS"] = "1"
+
+bert_feature_map = {
+    "ZH": BertFeature(
+        "./bert/chinese-roberta-wwm-ext-large",
+        language="ZH",
+    ),
+    "JP": BertFeature(
+        "./bert/deberta-v2-large-japanese-char-wwm",
+        language="JP",
+    ),
+    "EN": BertFeature(
+        "./bert/deberta-v3-large",
+        language="EN",
+    ),
+}
+
+clap_feature = ClapFeature("./emotional/clap-htsat-fused")
+
+
+def generate_audio(
+    slices,
+    sdp_ratio,
+    noise_scale,
+    noise_scale_w,
+    length_scale,
+    speaker,
+    language,
+    reference_audio,
+    emotion,
+    skip_start=False,
+    skip_end=False,
+):
+    audio_list = []
+    # silence = np.zeros(hps.data.sampling_rate // 2, dtype=np.int16)
+    with torch.no_grad():
+        for idx, piece in enumerate(slices):
+            skip_start = (idx != 0) and skip_start
+            skip_end = (idx != len(slices) - 1) and skip_end
+            audio = infer(
+                piece,
+                reference_audio=reference_audio,
+                emotion=emotion,
+                sdp_ratio=sdp_ratio,
+                noise_scale=noise_scale,
+                noise_scale_w=noise_scale_w,
+                length_scale=length_scale,
+                sid=speaker,
+                language=language,
+                hps=hps,
+                net_g=net_g,
+                device=device,
+                skip_start=skip_start,
+                skip_end=skip_end,
+                bert=bert_feature_map,
+                clap=clap_feature,
+            )
+            audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
+            audio_list.append(audio16bit)
+            # audio_list.append(silence)  # 将静音添加到列表中
+    return audio_list
+
+
+def generate_audio_multilang(
+    slices,
+    sdp_ratio,
+    noise_scale,
+    noise_scale_w,
+    length_scale,
+    speaker,
+    language,
+    reference_audio,
+    emotion,
+    skip_start=False,
+    skip_end=False,
+):
+    audio_list = []
+    # silence = np.zeros(hps.data.sampling_rate // 2, dtype=np.int16)
+    with torch.no_grad():
+        for idx, piece in enumerate(slices):
+            skip_start = (idx != 0) and skip_start
+            skip_end = (idx != len(slices) - 1) and skip_end
+            audio = infer_multilang(
+                piece,
+                reference_audio=reference_audio,
+                emotion=emotion,
+                sdp_ratio=sdp_ratio,
+                noise_scale=noise_scale,
+                noise_scale_w=noise_scale_w,
+                length_scale=length_scale,
+                sid=speaker,
+                language=language[idx],
+                hps=hps,
+                net_g=net_g,
+                device=device,
+                skip_start=skip_start,
+                skip_end=skip_end,
+            )
+            audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
+            audio_list.append(audio16bit)
+            # audio_list.append(silence)  # 将静音添加到列表中
+    return audio_list
+
+
+def tts_split(
+    text: str,
+    speaker,
+    sdp_ratio,
+    noise_scale,
+    noise_scale_w,
+    length_scale,
+    language,
+    cut_by_sent,
+    interval_between_para,
+    interval_between_sent,
+    reference_audio,
+    emotion,
+):
+    if language == "mix":
+        return ("invalid", None)
+    while text.find("\n\n") != -1:
+        text = text.replace("\n\n", "\n")
+    para_list = re_matching.cut_para(text)
+    audio_list = []
+    if not cut_by_sent:
+        for idx, p in enumerate(para_list):
+            skip_start = idx != 0
+            skip_end = idx != len(para_list) - 1
+            audio = infer(
+                p,
+                reference_audio=reference_audio,
+                emotion=emotion,
+                sdp_ratio=sdp_ratio,
+                noise_scale=noise_scale,
+                noise_scale_w=noise_scale_w,
+                length_scale=length_scale,
+                sid=speaker,
+                language=language,
+                hps=hps,
+                net_g=net_g,
+                device=device,
+                skip_start=skip_start,
+                skip_end=skip_end,
+            )
+            audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
+            audio_list.append(audio16bit)
+            silence = np.zeros((int)(44100 * interval_between_para), dtype=np.int16)
+            audio_list.append(silence)
+    else:
+        for idx, p in enumerate(para_list):
+            skip_start = idx != 0
+            skip_end = idx != len(para_list) - 1
+            audio_list_sent = []
+            sent_list = re_matching.cut_sent(p)
+            for idx, s in enumerate(sent_list):
+                skip_start = (idx != 0) and skip_start
+                skip_end = (idx != len(sent_list) - 1) and skip_end
+                audio = infer(
+                    s,
+                    reference_audio=reference_audio,
+                    emotion=emotion,
+                    sdp_ratio=sdp_ratio,
+                    noise_scale=noise_scale,
+                    noise_scale_w=noise_scale_w,
+                    length_scale=length_scale,
+                    sid=speaker,
+                    language=language,
+                    hps=hps,
+                    net_g=net_g,
+                    device=device,
+                    skip_start=skip_start,
+                    skip_end=skip_end,
+                )
+                audio_list_sent.append(audio)
+                silence = np.zeros((int)(44100 * interval_between_sent))
+                audio_list_sent.append(silence)
+            if (interval_between_para - interval_between_sent) > 0:
+                silence = np.zeros(
+                    (int)(44100 * (interval_between_para - interval_between_sent))
+                )
+                audio_list_sent.append(silence)
+            audio16bit = gr.processing_utils.convert_to_16_bit_wav(
+                np.concatenate(audio_list_sent)
+            )  # 对完整句子做音量归一
+            audio_list.append(audio16bit)
+    audio_concat = np.concatenate(audio_list)
+    return ("Success", (44100, audio_concat))
+
+
+def tts_fn(
+    text: str,
+    speaker,
+    sdp_ratio,
+    noise_scale,
+    noise_scale_w,
+    length_scale,
+    language,
+    reference_audio,
+    emotion,
+    prompt_mode,
+):
+    if prompt_mode == "Audio prompt":
+        if reference_audio == None:
+            return ("Invalid audio prompt", None)
+        else:
+            reference_audio = load_audio(reference_audio)[1]
+    else:
+        reference_audio = None
+    audio_list = []
+    if language == "mix":
+        bool_valid, str_valid = re_matching.validate_text(text)
+        if not bool_valid:
+            return str_valid, (
+                hps.data.sampling_rate,
+                np.concatenate([np.zeros(hps.data.sampling_rate // 2)]),
+            )
+        result = []
+        for slice in re_matching.text_matching(text):
+            _speaker = slice.pop()
+            temp_contant = []
+            temp_lang = []
+            for lang, content in slice:
+                if "|" in content:
+                    temp = []
+                    temp_ = []
+                    for i in content.split("|"):
+                        if i != "":
+                            temp.append([i])
+                            temp_.append([lang])
+                        else:
+                            temp.append([])
+                            temp_.append([])
+                    temp_contant += temp
+                    temp_lang += temp_
+                else:
+                    if len(temp_contant) == 0:
+                        temp_contant.append([])
+                        temp_lang.append([])
+                    temp_contant[-1].append(content)
+                    temp_lang[-1].append(lang)
+            for i, j in zip(temp_lang, temp_contant):
+                result.append([*zip(i, j), _speaker])
+        for i, one in enumerate(result):
+            skip_start = i != 0
+            skip_end = i != len(result) - 1
+            _speaker = one.pop()
+            idx = 0
+            while idx < len(one):
+                text_to_generate = []
+                lang_to_generate = []
+                while True:
+                    lang, content = one[idx]
+                    temp_text = [content]
+                    if len(text_to_generate) > 0:
+                        text_to_generate[-1] += [temp_text.pop(0)]
+                        lang_to_generate[-1] += [lang]
+                    if len(temp_text) > 0:
+                        text_to_generate += [[i] for i in temp_text]
+                        lang_to_generate += [[lang]] * len(temp_text)
+                    if idx + 1 < len(one):
+                        idx += 1
+                    else:
+                        break
+                skip_start = (idx != 0) and skip_start
+                skip_end = (idx != len(one) - 1) and skip_end
+                print(text_to_generate, lang_to_generate)
+                audio_list.extend(
+                    generate_audio_multilang(
+                        text_to_generate,
+                        sdp_ratio,
+                        noise_scale,
+                        noise_scale_w,
+                        length_scale,
+                        _speaker,
+                        lang_to_generate,
+                        reference_audio,
+                        emotion,
+                        skip_start,
+                        skip_end,
+                    )
+                )
+                idx += 1
+    elif language.lower() == "auto":
+        for idx, slice in enumerate(text.split("|")):
+            if slice == "":
+                continue
+            skip_start = idx != 0
+            skip_end = idx != len(text.split("|")) - 1
+            sentences_list = split_by_language(
+                slice, target_languages=["zh", "ja", "en"]
+            )
+            idx = 0
+            while idx < len(sentences_list):
+                text_to_generate = []
+                lang_to_generate = []
+                while True:
+                    content, lang = sentences_list[idx]
+                    temp_text = [content]
+                    lang = lang.upper()
+                    if lang == "JA":
+                        lang = "JP"
+                    if len(text_to_generate) > 0:
+                        text_to_generate[-1] += [temp_text.pop(0)]
+                        lang_to_generate[-1] += [lang]
+                    if len(temp_text) > 0:
+                        text_to_generate += [[i] for i in temp_text]
+                        lang_to_generate += [[lang]] * len(temp_text)
+                    if idx + 1 < len(sentences_list):
+                        idx += 1
+                    else:
+                        break
+                skip_start = (idx != 0) and skip_start
+                skip_end = (idx != len(sentences_list) - 1) and skip_end
+                print(text_to_generate, lang_to_generate)
+                audio_list.extend(
+                    generate_audio_multilang(
+                        text_to_generate,
+                        sdp_ratio,
+                        noise_scale,
+                        noise_scale_w,
+                        length_scale,
+                        speaker,
+                        lang_to_generate,
+                        reference_audio,
+                        emotion,
+                        skip_start,
+                        skip_end,
+                    )
+                )
+                idx += 1
+    else:
+        audio_list.extend(
+            generate_audio(
+                text.split("|"),
+                sdp_ratio,
+                noise_scale,
+                noise_scale_w,
+                length_scale,
+                speaker,
+                language,
+                reference_audio,
+                emotion,
+            )
+        )
+
+    audio_concat = np.concatenate(audio_list)
+    return "Success", (hps.data.sampling_rate, audio_concat)
+
+
+def load_audio(path):
+    audio, sr = librosa.load(path, 48000)
+    # audio = librosa.resample(audio, 44100, 48000)
+    return sr, audio
+
+
+def gr_util(item):
+    if item == "Text prompt":
+        return {"visible": True, "__type__": "update"}, {
+            "visible": False,
+            "__type__": "update",
+        }
+    else:
+        return {"visible": False, "__type__": "update"}, {
+            "visible": True,
+            "__type__": "update",
+        }
+
+
+if __name__ == "__main__":
+    if config.webui_config.debug:
+        logger.info("Enable DEBUG-LEVEL log")
+        logging.basicConfig(level=logging.DEBUG)
+    hps = utils.get_hparams_from_file(config.webui_config.config_path)
+    # 若config.json中未指定版本则默认为最新版本
+    version = hps.version if hasattr(hps, "version") else latest_version
+    net_g = get_net_g(
+        model_path=config.webui_config.model, version=version, device=device, hps=hps
+    )
+    speaker_ids = hps.data.spk2id
+    speakers = list(speaker_ids.keys())
+    languages = ["ZH", "JP", "EN", "mix", "auto"]
+    with gr.Blocks() as app:
+        with gr.Row():
+            with gr.Column():
+                text = gr.TextArea(
+                    label="输入文本内容",
+                    placeholder="""
+                    如果你选择语言为\'mix\'，必须按照格式输入，否则报错:
+                        格式举例(zh是中文，jp是日语，不区分大小写；说话人举例:gongzi):
+                         [说话人1]<zh>你好，こんにちは！ <jp>こんにちは，世界。
+                         [说话人2]<zh>你好吗？<jp>元気ですか？
+                         [说话人3]<zh>谢谢。<jp>どういたしまして。
+                         ...
+                    另外，所有的语言选项都可以用'|'分割长段实现分句生成。
+                    """,
+                )
+                trans = gr.Button("中翻日", variant="primary")
+                slicer = gr.Button("快速切分", variant="primary")
+                speaker = gr.Dropdown(
+                    choices=speakers, value=speakers[0], label="Speaker"
+                )
+                _ = gr.Markdown(
+                    value="提示模式（Prompt mode）：可选文字提示或音频提示，用于生成文字或音频指定风格的声音。\n"
+                )
+                prompt_mode = gr.Radio(
+                    ["Text prompt", "Audio prompt"],
+                    label="Prompt Mode",
+                    value="Text prompt",
+                )
+                text_prompt = gr.Textbox(
+                    label="Text prompt",
+                    placeholder="用文字描述生成风格。如：Happy",
+                    value="Happy",
+                    visible=True,
+                )
+                audio_prompt = gr.Audio(
+                    label="Audio prompt", type="filepath", visible=False
+                )
+                sdp_ratio = gr.Slider(
+                    minimum=0, maximum=1, value=0.2, step=0.1, label="SDP Ratio"
+                )
+                noise_scale = gr.Slider(
+                    minimum=0.1, maximum=2, value=0.6, step=0.1, label="Noise"
+                )
+                noise_scale_w = gr.Slider(
+                    minimum=0.1, maximum=2, value=0.8, step=0.1, label="Noise_W"
+                )
+                length_scale = gr.Slider(
+                    minimum=0.1, maximum=2, value=1.0, step=0.1, label="Length"
+                )
+                language = gr.Dropdown(
+                    choices=languages, value=languages[0], label="Language"
+                )
+                btn = gr.Button("生成音频！", variant="primary")
+            with gr.Column():
+                with gr.Row():
+                    with gr.Column():
+                        interval_between_sent = gr.Slider(
+                            minimum=0,
+                            maximum=5,
+                            value=0.2,
+                            step=0.1,
+                            label="句间停顿(秒)，勾选按句切分才生效",
+                        )
+                        interval_between_para = gr.Slider(
+                            minimum=0,
+                            maximum=10,
+                            value=1,
+                            step=0.1,
+                            label="段间停顿(秒)，需要大于句间停顿才有效",
+                        )
+                        opt_cut_by_sent = gr.Checkbox(
+                            label="按句切分    在按段落切分的基础上再按句子切分文本"
+                        )
+                        slicer = gr.Button("切分生成", variant="primary")
+                text_output = gr.Textbox(label="状态信息")
+                audio_output = gr.Audio(label="输出音频")
+                # explain_image = gr.Image(
+                #     label="参数解释信息",
+                #     show_label=True,
+                #     show_share_button=False,
+                #     show_download_button=False,
+                #     value=os.path.abspath("./img/参数说明.png"),
+                # )
+        btn.click(
+            tts_fn,
+            inputs=[
+                text,
+                speaker,
+                sdp_ratio,
+                noise_scale,
+                noise_scale_w,
+                length_scale,
+                language,
+                audio_prompt,
+                text_prompt,
+                prompt_mode,
+            ],
+            outputs=[text_output, audio_output],
+        )
+
+        trans.click(
+            translate,
+            inputs=[text],
+            outputs=[text],
+        )
+        slicer.click(
+            tts_split,
+            inputs=[
+                text,
+                speaker,
+                sdp_ratio,
+                noise_scale,
+                noise_scale_w,
+                length_scale,
+                language,
+                opt_cut_by_sent,
+                interval_between_para,
+                interval_between_sent,
+                audio_prompt,
+                text_prompt,
+            ],
+            outputs=[text_output, audio_output],
+        )
+
+        prompt_mode.change(
+            lambda x: gr_util(x),
+            inputs=[prompt_mode],
+            outputs=[text_prompt, audio_prompt],
+        )
+
+        audio_prompt.upload(
+            lambda x: load_audio(x),
+            inputs=[audio_prompt],
+            outputs=[audio_prompt],
+        )
+
+    print("推理页面已开启!")
+    webbrowser.open(f"http://127.0.0.1:{config.webui_config.port}")
+    app.launch(share=config.webui_config.share, server_port=config.webui_config.port)

infer.py

@@ -10,7 +10,8 @@
 import torch
 import commons
 from text import cleaned_text_to_sequence, get_bert
-from clap_wrapper import get_clap_audio_feature, get_clap_text_feature
+
+# from clap_wrapper import get_clap_audio_feature, get_clap_text_feature
 from text.cleaner import clean_text
 import utils
 import numpy as np
@@ -98,7 +99,8 @@ def get_net_g(model_path: str, version: str, device: str, hps):
     return net_g
 
 
-def get_text(text, language_str, hps, device):
+def get_text(text, language_str, hps, device, style_text=None, style_weight=0.7):
+    style_text = None if style_text == "" else style_text
     # 在此处实现当前版本的get_text
     norm_text, phone, tone, word2ph = clean_text(text, language_str)
     phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
@@ -110,21 +112,23 @@ def get_text(text, language_str, hps, device):
         for i in range(len(word2ph)):
             word2ph[i] = word2ph[i] * 2
         word2ph[0] += 1
-    bert_ori = get_bert(norm_text, word2ph, language_str, device)
+    bert_ori = get_bert(
+        norm_text, word2ph, language_str, device, style_text, style_weight
+    )
     del word2ph
     assert bert_ori.shape[-1] == len(phone), phone
 
     if language_str == "ZH":
         bert = bert_ori
-        ja_bert = torch.rand(1024, len(phone))
+        ja_bert = torch.randn(1024, len(phone))
-        en_bert = torch.rand(1024, len(phone))
+        en_bert = torch.randn(1024, len(phone))
     elif language_str == "JP":
-        bert = torch.rand(1024, len(phone))
+        bert = torch.randn(1024, len(phone))
         ja_bert = bert_ori
-        en_bert = torch.rand(1024, len(phone))
+        en_bert = torch.randn(1024, len(phone))
     elif language_str == "EN":
-        bert = torch.rand(1024, len(phone))
+        bert = torch.randn(1024, len(phone))
-        ja_bert = torch.rand(1024, len(phone))
+        ja_bert = torch.randn(1024, len(phone))
         en_bert = bert_ori
     else:
         raise ValueError("language_str should be ZH, JP or EN")
@@ -154,6 +158,8 @@ def infer(
     reference_audio=None,
     skip_start=False,
     skip_end=False,
+    style_text=None,
+    style_weight=0.7,
 ):
     # 2.2版本参数位置变了
     # 2.1 参数新增 emotion reference_audio skip_start skip_end
@@ -181,6 +187,7 @@ def infer(
     # 非当前版本，根据版本号选择合适的infer
     if version != latest_version:
         if version in inferMap_V3.keys():
+            emotion = 0
             return inferMap_V3[version](
                 text,
                 sdp_ratio,
@@ -196,6 +203,8 @@ def infer(
                 emotion,
                 skip_start,
                 skip_end,
+                style_text,
+                style_weight,
             )
         if version in inferMap_V2.keys():
             return inferMap_V2[version](
@@ -224,14 +233,19 @@ def infer(
             )
     # 在此处实现当前版本的推理
     # emo = get_emo_(reference_audio, emotion, sid)
-    if isinstance(reference_audio, np.ndarray):
+    # if isinstance(reference_audio, np.ndarray):
-        emo = get_clap_audio_feature(reference_audio, device)
+    #     emo = get_clap_audio_feature(reference_audio, device)
-    else:
+    # else:
-        emo = get_clap_text_feature(emotion, device)
+    #     emo = get_clap_text_feature(emotion, device)
-    emo = torch.squeeze(emo, dim=1)
+    # emo = torch.squeeze(emo, dim=1)
 
     bert, ja_bert, en_bert, phones, tones, lang_ids = get_text(
-        text, language, hps, device
+        text,
+        language,
+        hps,
+        device,
+        style_text=style_text,
+        style_weight=style_weight,
     )
     if skip_start:
         phones = phones[3:]
@@ -255,7 +269,7 @@ def infer(
         ja_bert = ja_bert.to(device).unsqueeze(0)
         en_bert = en_bert.to(device).unsqueeze(0)
         x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
-        emo = emo.to(device).unsqueeze(0)
+        # emo = emo.to(device).unsqueeze(0)
         del phones
         speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
         audio = (
@@ -268,7 +282,6 @@ def infer(
                 bert,
                 ja_bert,
                 en_bert,
-                emo,
                 sdp_ratio=sdp_ratio,
                 noise_scale=noise_scale,
                 noise_scale_w=noise_scale_w,
@@ -278,7 +291,16 @@ def infer(
             .float()
             .numpy()
         )
-        del x_tst, tones, lang_ids, bert, x_tst_lengths, speakers, ja_bert, en_bert, emo
+        del (
+            x_tst,
+            tones,
+            lang_ids,
+            bert,
+            x_tst_lengths,
+            speakers,
+            ja_bert,
+            en_bert,
+        )  # , emo
         if torch.cuda.is_available():
             torch.cuda.empty_cache()
         return audio
@@ -302,14 +324,14 @@ def infer_multilang(
 ):
     bert, ja_bert, en_bert, phones, tones, lang_ids = [], [], [], [], [], []
     # emo = get_emo_(reference_audio, emotion, sid)
-    if isinstance(reference_audio, np.ndarray):
+    # if isinstance(reference_audio, np.ndarray):
-        emo = get_clap_audio_feature(reference_audio, device)
+    #     emo = get_clap_audio_feature(reference_audio, device)
-    else:
+    # else:
-        emo = get_clap_text_feature(emotion, device)
+    #     emo = get_clap_text_feature(emotion, device)
-    emo = torch.squeeze(emo, dim=1)
+    # emo = torch.squeeze(emo, dim=1)
     for idx, (txt, lang) in enumerate(zip(text, language)):
-        skip_start = (idx != 0) or (skip_start and idx == 0)
+        _skip_start = (idx != 0) or (skip_start and idx == 0)
-        skip_end = (idx != len(text) - 1) or (skip_end and idx == len(text) - 1)
+        _skip_end = (idx != len(language) - 1) or skip_end
         (
             temp_bert,
             temp_ja_bert,
@@ -318,14 +340,14 @@ def infer_multilang(
             temp_tones,
             temp_lang_ids,
         ) = get_text(txt, lang, hps, device)
-        if skip_start:
+        if _skip_start:
             temp_bert = temp_bert[:, 3:]
             temp_ja_bert = temp_ja_bert[:, 3:]
             temp_en_bert = temp_en_bert[:, 3:]
             temp_phones = temp_phones[3:]
             temp_tones = temp_tones[3:]
             temp_lang_ids = temp_lang_ids[3:]
-        if skip_end:
+        if _skip_end:
             temp_bert = temp_bert[:, :-2]
             temp_ja_bert = temp_ja_bert[:, :-2]
             temp_en_bert = temp_en_bert[:, :-2]
@@ -351,7 +373,7 @@ def infer_multilang(
         bert = bert.to(device).unsqueeze(0)
         ja_bert = ja_bert.to(device).unsqueeze(0)
         en_bert = en_bert.to(device).unsqueeze(0)
-        emo = emo.to(device).unsqueeze(0)
+        # emo = emo.to(device).unsqueeze(0)
         x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
         del phones
         speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
@@ -365,7 +387,6 @@ def infer_multilang(
                 bert,
                 ja_bert,
                 en_bert,
-                emo,
                 sdp_ratio=sdp_ratio,
                 noise_scale=noise_scale,
                 noise_scale_w=noise_scale_w,
@@ -375,7 +396,16 @@ def infer_multilang(
             .float()
             .numpy()
         )
-        del x_tst, tones, lang_ids, bert, x_tst_lengths, speakers, ja_bert, en_bert, emo
+        del (
+            x_tst,
+            tones,
+            lang_ids,
+            bert,
+            x_tst_lengths,
+            speakers,
+            ja_bert,
+            en_bert,
+        )  # , emo
         if torch.cuda.is_available():
             torch.cuda.empty_cache()
         return audio

losses.py

@@ -1,4 +1,6 @@
 import torch
+import torchaudio
+from transformers import AutoModel
 
 
 def feature_loss(fmap_r, fmap_g):
@@ -56,3 +58,93 @@ def kl_loss(z_p, logs_q, m_p, logs_p, z_mask):
     kl = torch.sum(kl * z_mask)
     l = kl / torch.sum(z_mask)
     return l
+
+
+class WavLMLoss(torch.nn.Module):
+    def __init__(self, model, wd, model_sr, slm_sr=16000):
+        super(WavLMLoss, self).__init__()
+        self.wavlm = AutoModel.from_pretrained(model)
+        self.wd = wd
+        self.resample = torchaudio.transforms.Resample(model_sr, slm_sr)
+
+    def forward(self, wav, y_rec):
+        with torch.no_grad():
+            wav_16 = self.resample(wav)
+            wav_embeddings = self.wavlm(
+                input_values=wav_16, output_hidden_states=True
+            ).hidden_states
+        y_rec_16 = self.resample(y_rec)
+        y_rec_embeddings = self.wavlm(
+            input_values=y_rec_16.squeeze(), output_hidden_states=True
+        ).hidden_states
+
+        floss = 0
+        for er, eg in zip(wav_embeddings, y_rec_embeddings):
+            floss += torch.mean(torch.abs(er - eg))
+
+        return floss.mean()
+
+    def generator(self, y_rec):
+        y_rec_16 = self.resample(y_rec)
+        y_rec_embeddings = self.wavlm(
+            input_values=y_rec_16, output_hidden_states=True
+        ).hidden_states
+        y_rec_embeddings = (
+            torch.stack(y_rec_embeddings, dim=1)
+            .transpose(-1, -2)
+            .flatten(start_dim=1, end_dim=2)
+        )
+        y_df_hat_g = self.wd(y_rec_embeddings)
+        loss_gen = torch.mean((1 - y_df_hat_g) ** 2)
+
+        return loss_gen
+
+    def discriminator(self, wav, y_rec):
+        with torch.no_grad():
+            wav_16 = self.resample(wav)
+            wav_embeddings = self.wavlm(
+                input_values=wav_16, output_hidden_states=True
+            ).hidden_states
+            y_rec_16 = self.resample(y_rec)
+            y_rec_embeddings = self.wavlm(
+                input_values=y_rec_16, output_hidden_states=True
+            ).hidden_states
+
+            y_embeddings = (
+                torch.stack(wav_embeddings, dim=1)
+                .transpose(-1, -2)
+                .flatten(start_dim=1, end_dim=2)
+            )
+            y_rec_embeddings = (
+                torch.stack(y_rec_embeddings, dim=1)
+                .transpose(-1, -2)
+                .flatten(start_dim=1, end_dim=2)
+            )
+
+        y_d_rs = self.wd(y_embeddings)
+        y_d_gs = self.wd(y_rec_embeddings)
+
+        y_df_hat_r, y_df_hat_g = y_d_rs, y_d_gs
+
+        r_loss = torch.mean((1 - y_df_hat_r) ** 2)
+        g_loss = torch.mean((y_df_hat_g) ** 2)
+
+        loss_disc_f = r_loss + g_loss
+
+        return loss_disc_f.mean()
+
+    def discriminator_forward(self, wav):
+        with torch.no_grad():
+            wav_16 = self.resample(wav)
+            wav_embeddings = self.wavlm(
+                input_values=wav_16, output_hidden_states=True
+            ).hidden_states
+            y_embeddings = (
+                torch.stack(wav_embeddings, dim=1)
+                .transpose(-1, -2)
+                .flatten(start_dim=1, end_dim=2)
+            )
+
+        y_d_rs = self.wd(y_embeddings)
+
+        return y_d_rs

models.py

@@ -40,33 +40,22 @@ class DurationDiscriminator(nn.Module):  # vits2
         self.norm_2 = modules.LayerNorm(filter_channels)
         self.dur_proj = nn.Conv1d(1, filter_channels, 1)
 
-        self.pre_out_conv_1 = nn.Conv1d(
+        self.LSTM = nn.LSTM(
-            2 * filter_channels, filter_channels, kernel_size, padding=kernel_size // 2
+            2 * filter_channels, filter_channels, batch_first=True, bidirectional=True
         )
-        self.pre_out_norm_1 = modules.LayerNorm(filter_channels)
-        self.pre_out_conv_2 = nn.Conv1d(
-            filter_channels, filter_channels, kernel_size, padding=kernel_size // 2
-        )
-        self.pre_out_norm_2 = modules.LayerNorm(filter_channels)
 
         if gin_channels != 0:
             self.cond = nn.Conv1d(gin_channels, in_channels, 1)
 
-        self.output_layer = nn.Sequential(nn.Linear(filter_channels, 1), nn.Sigmoid())
+        self.output_layer = nn.Sequential(
+            nn.Linear(2 * filter_channels, 1), nn.Sigmoid()
+        )
 
-    def forward_probability(self, x, x_mask, dur, g=None):
+    def forward_probability(self, x, dur):
         dur = self.dur_proj(dur)
         x = torch.cat([x, dur], dim=1)
-        x = self.pre_out_conv_1(x * x_mask)
-        x = torch.relu(x)
-        x = self.pre_out_norm_1(x)
-        x = self.drop(x)
-        x = self.pre_out_conv_2(x * x_mask)
-        x = torch.relu(x)
-        x = self.pre_out_norm_2(x)
-        x = self.drop(x)
-        x = x * x_mask
         x = x.transpose(1, 2)
+        x, _ = self.LSTM(x)
         output_prob = self.output_layer(x)
         return output_prob
 
@@ -86,7 +75,7 @@ class DurationDiscriminator(nn.Module):  # vits2
 
         output_probs = []
         for dur in [dur_r, dur_hat]:
-            output_prob = self.forward_probability(x, x_mask, dur, g)
+            output_prob = self.forward_probability(x, dur)
             output_probs.append(output_prob)
 
         return output_probs
@@ -354,7 +343,6 @@ class TextEncoder(nn.Module):
         n_layers,
         kernel_size,
         p_dropout,
-        n_speakers,
         gin_channels=0,
     ):
         super().__init__()
@@ -376,31 +364,6 @@ class TextEncoder(nn.Module):
         self.bert_proj = nn.Conv1d(1024, hidden_channels, 1)
         self.ja_bert_proj = nn.Conv1d(1024, hidden_channels, 1)
         self.en_bert_proj = nn.Conv1d(1024, hidden_channels, 1)
-        # self.emo_proj = nn.Linear(512, hidden_channels)
-        self.in_feature_net = nn.Sequential(
-            # input is assumed to an already normalized embedding
-            nn.Linear(512, 1028, bias=False),
-            nn.GELU(),
-            nn.LayerNorm(1028),
-            *[Block(1028, 512) for _ in range(1)],
-            nn.Linear(1028, 512, bias=False),
-            # normalize before passing to VQ?
-            # nn.GELU(),
-            # nn.LayerNorm(512),
-        )
-        self.emo_vq = VectorQuantize(
-            dim=512,
-            codebook_size=64,
-            codebook_dim=32,
-            commitment_weight=0.1,
-            decay=0.85,
-            heads=32,
-            kmeans_iters=20,
-            separate_codebook_per_head=True,
-            stochastic_sample_codes=True,
-            threshold_ema_dead_code=2,
-        )
-        self.out_feature_net = nn.Linear(512, hidden_channels)
 
         self.encoder = attentions.Encoder(
             hidden_channels,
@@ -413,18 +376,10 @@ class TextEncoder(nn.Module):
         )
         self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
 
-    def forward(
+    def forward(self, x, x_lengths, tone, language, bert, ja_bert, en_bert, g=None):
-        self, x, x_lengths, tone, language, bert, ja_bert, en_bert, emo, sid, g=None
-    ):
-        sid = sid.cpu()
         bert_emb = self.bert_proj(bert).transpose(1, 2)
         ja_bert_emb = self.ja_bert_proj(ja_bert).transpose(1, 2)
         en_bert_emb = self.en_bert_proj(en_bert).transpose(1, 2)
-        emo_emb = self.in_feature_net(emo)
-        emo_emb, _, loss_commit = self.emo_vq(emo_emb.unsqueeze(1))
-        loss_commit = loss_commit.mean()
-        emo_emb = self.out_feature_net(emo_emb)
-        # emo_emb = self.emo_proj(emo.unsqueeze(1))
         x = (
             self.emb(x)
             + self.tone_emb(tone)
@@ -432,7 +387,6 @@ class TextEncoder(nn.Module):
             + bert_emb
             + ja_bert_emb
             + en_bert_emb
-            + emo_emb
         ) * math.sqrt(
             self.hidden_channels
         )  # [b, t, h]
@@ -445,7 +399,7 @@ class TextEncoder(nn.Module):
         stats = self.proj(x) * x_mask
 
         m, logs = torch.split(stats, self.out_channels, dim=1)
-        return x, m, logs, x_mask, loss_commit
+        return x, m, logs, x_mask
 
 
 class ResidualCouplingBlock(nn.Module):
@@ -748,6 +702,55 @@ class MultiPeriodDiscriminator(torch.nn.Module):
         return y_d_rs, y_d_gs, fmap_rs, fmap_gs
 
 
+class WavLMDiscriminator(nn.Module):
+    """docstring for Discriminator."""
+
+    def __init__(
+        self, slm_hidden=768, slm_layers=13, initial_channel=64, use_spectral_norm=False
+    ):
+        super(WavLMDiscriminator, self).__init__()
+        norm_f = weight_norm if use_spectral_norm == False else spectral_norm
+        self.pre = norm_f(
+            Conv1d(slm_hidden * slm_layers, initial_channel, 1, 1, padding=0)
+        )
+
+        self.convs = nn.ModuleList(
+            [
+                norm_f(
+                    nn.Conv1d(
+                        initial_channel, initial_channel * 2, kernel_size=5, padding=2
+                    )
+                ),
+                norm_f(
+                    nn.Conv1d(
+                        initial_channel * 2,
+                        initial_channel * 4,
+                        kernel_size=5,
+                        padding=2,
+                    )
+                ),
+                norm_f(
+                    nn.Conv1d(initial_channel * 4, initial_channel * 4, 5, 1, padding=2)
+                ),
+            ]
+        )
+
+        self.conv_post = norm_f(Conv1d(initial_channel * 4, 1, 3, 1, padding=1))
+
+    def forward(self, x):
+        x = self.pre(x)
+
+        fmap = []
+        for l in self.convs:
+            x = l(x)
+            x = F.leaky_relu(x, modules.LRELU_SLOPE)
+            fmap.append(x)
+        x = self.conv_post(x)
+        x = torch.flatten(x, 1, -1)
+
+        return x
+
+
 class ReferenceEncoder(nn.Module):
     """
     inputs --- [N, Ty/r, n_mels*r]  mels
@@ -878,7 +881,6 @@ class SynthesizerTrn(nn.Module):
             n_layers,
             kernel_size,
             p_dropout,
-            self.n_speakers,
             gin_channels=self.enc_gin_channels,
         )
         self.dec = Generator(
@@ -946,14 +948,13 @@ class SynthesizerTrn(nn.Module):
         bert,
         ja_bert,
         en_bert,
-        emo=None,
     ):
         if self.n_speakers > 0:
             g = self.emb_g(sid).unsqueeze(-1)  # [b, h, 1]
         else:
             g = self.ref_enc(y.transpose(1, 2)).unsqueeze(-1)
-        x, m_p, logs_p, x_mask, loss_commit = self.enc_p(
+        x, m_p, logs_p, x_mask = self.enc_p(
-            x, x_lengths, tone, language, bert, ja_bert, en_bert, emo, sid, g=g
+            x, x_lengths, tone, language, bert, ja_bert, en_bert, g=g
         )
         z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g)
         z_p = self.flow(z, y_mask, g=g)
@@ -996,9 +997,11 @@ class SynthesizerTrn(nn.Module):
 
         logw_ = torch.log(w + 1e-6) * x_mask
         logw = self.dp(x, x_mask, g=g)
+        logw_sdp = self.sdp(x, x_mask, g=g, reverse=True, noise_scale=1.0)
         l_length_dp = torch.sum((logw - logw_) ** 2, [1, 2]) / torch.sum(
             x_mask
         )  # for averaging
+        l_length_sdp += torch.sum((logw_sdp - logw_) ** 2, [1, 2]) / torch.sum(x_mask)
 
         l_length = l_length_dp + l_length_sdp
 
@@ -1018,9 +1021,8 @@ class SynthesizerTrn(nn.Module):
             x_mask,
             y_mask,
             (z, z_p, m_p, logs_p, m_q, logs_q),
-            (x, logw, logw_),
+            (x, logw, logw_, logw_sdp),
             g,
-            loss_commit,
         )
 
     def infer(
@@ -1033,7 +1035,6 @@ class SynthesizerTrn(nn.Module):
         bert,
         ja_bert,
         en_bert,
-        emo=None,
         noise_scale=0.667,
         length_scale=1,
         noise_scale_w=0.8,
@@ -1047,8 +1048,8 @@ class SynthesizerTrn(nn.Module):
             g = self.emb_g(sid).unsqueeze(-1)  # [b, h, 1]
         else:
             g = self.ref_enc(y.transpose(1, 2)).unsqueeze(-1)
-        x, m_p, logs_p, x_mask, _ = self.enc_p(
+        x, m_p, logs_p, x_mask = self.enc_p(
-            x, x_lengths, tone, language, bert, ja_bert, en_bert, emo, sid, g=g
+            x, x_lengths, tone, language, bert, ja_bert, en_bert, g=g
         )
         logw = self.sdp(x, x_mask, g=g, reverse=True, noise_scale=noise_scale_w) * (
             sdp_ratio

oldVersion/V210/__init__.py

@@ -7,7 +7,7 @@ from .text import cleaned_text_to_sequence, get_bert
 from .text.cleaner import clean_text
 
 
-def get_text(text, language_str, hps, device):
+def get_text(text, language_str, hps, device, style_text=None, style_weight=0.7):
     # 在此处实现当前版本的get_text
     norm_text, phone, tone, word2ph = clean_text(text, language_str)
     phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
@@ -19,7 +19,9 @@ def get_text(text, language_str, hps, device):
         for i in range(len(word2ph)):
             word2ph[i] = word2ph[i] * 2
         word2ph[0] += 1
-    bert_ori = get_bert(norm_text, word2ph, language_str, device)
+    bert_ori = get_bert(
+        norm_text, word2ph, language_str, device, style_text, style_weight
+    )
     del word2ph
     assert bert_ori.shape[-1] == len(phone), phone
 
@@ -74,9 +76,11 @@ def infer(
     emotion=None,
     skip_start=False,
     skip_end=False,
+    style_text=None,
+    style_weight=0.7,
 ):
     bert, ja_bert, en_bert, phones, tones, lang_ids = get_text(
-        text, language, hps, device
+        text, language, hps, device, style_text, style_weight
     )
     emo = get_emo_(reference_audio, emotion)
     if skip_start:

oldVersion/V210/text/__init__.py

@@ -18,13 +18,15 @@ def cleaned_text_to_sequence(cleaned_text, tones, language):
     return phones, tones, lang_ids
 
 
-def get_bert(norm_text, word2ph, language, device):
+def get_bert(norm_text, word2ph, language, device, style_text, style_weight):
     from .chinese_bert import get_bert_feature as zh_bert
     from .english_bert_mock import get_bert_feature as en_bert
     from .japanese_bert import get_bert_feature as jp_bert
 
     lang_bert_func_map = {"ZH": zh_bert, "EN": en_bert, "JP": jp_bert}
-    bert = lang_bert_func_map[language](norm_text, word2ph, device)
+    bert = lang_bert_func_map[language](
+        norm_text, word2ph, device, style_text, style_weight
+    )
     return bert
 
 

oldVersion/V210/text/chinese_bert.py

@@ -12,7 +12,13 @@ tokenizer = AutoTokenizer.from_pretrained(LOCAL_PATH)
 models = dict()
 
 
-def get_bert_feature(text, word2ph, device=config.bert_gen_config.device):
+def get_bert_feature(
+    text,
+    word2ph,
+    device=config.bert_gen_config.device,
+    style_text=None,
+    style_weight=0.7,
+):
     if (
         sys.platform == "darwin"
         and torch.backends.mps.is_available()
@@ -29,12 +35,25 @@ def get_bert_feature(text, word2ph, device=config.bert_gen_config.device):
             inputs[i] = inputs[i].to(device)
         res = models[device](**inputs, output_hidden_states=True)
         res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
+        if style_text:
+            style_inputs = tokenizer(style_text, return_tensors="pt")
+            for i in style_inputs:
+                style_inputs[i] = style_inputs[i].to(device)
+            style_res = models[device](**style_inputs, output_hidden_states=True)
+            style_res = torch.cat(style_res["hidden_states"][-3:-2], -1)[0].cpu()
+            style_res_mean = style_res.mean(0)
 
     assert len(word2ph) == len(text) + 2
     word2phone = word2ph
     phone_level_feature = []
     for i in range(len(word2phone)):
-        repeat_feature = res[i].repeat(word2phone[i], 1)
+        if style_text:
+            repeat_feature = (
+                res[i].repeat(word2phone[i], 1) * (1 - style_weight)
+                + style_res_mean.repeat(word2phone[i], 1) * style_weight
+            )
+        else:
+            repeat_feature = res[i].repeat(word2phone[i], 1)
         phone_level_feature.append(repeat_feature)
 
     phone_level_feature = torch.cat(phone_level_feature, dim=0)

oldVersion/V210/text/english_bert_mock.py

@@ -13,7 +13,13 @@ tokenizer = DebertaV2Tokenizer.from_pretrained(LOCAL_PATH)
 models = dict()
 
 
-def get_bert_feature(text, word2ph, device=config.bert_gen_config.device):
+def get_bert_feature(
+    text,
+    word2ph,
+    device=config.bert_gen_config.device,
+    style_text=None,
+    style_weight=0.7,
+):
     if (
         sys.platform == "darwin"
         and torch.backends.mps.is_available()
@@ -30,11 +36,24 @@ def get_bert_feature(text, word2ph, device=config.bert_gen_config.device):
             inputs[i] = inputs[i].to(device)
         res = models[device](**inputs, output_hidden_states=True)
         res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
+        if style_text:
+            style_inputs = tokenizer(style_text, return_tensors="pt")
+            for i in style_inputs:
+                style_inputs[i] = style_inputs[i].to(device)
+            style_res = models[device](**style_inputs, output_hidden_states=True)
+            style_res = torch.cat(style_res["hidden_states"][-3:-2], -1)[0].cpu()
+            style_res_mean = style_res.mean(0)
     assert len(word2ph) == res.shape[0], (text, res.shape[0], len(word2ph))
     word2phone = word2ph
     phone_level_feature = []
     for i in range(len(word2phone)):
-        repeat_feature = res[i].repeat(word2phone[i], 1)
+        if style_text:
+            repeat_feature = (
+                res[i].repeat(word2phone[i], 1) * (1 - style_weight)
+                + style_res_mean.repeat(word2phone[i], 1) * style_weight
+            )
+        else:
+            repeat_feature = res[i].repeat(word2phone[i], 1)
         phone_level_feature.append(repeat_feature)
 
     phone_level_feature = torch.cat(phone_level_feature, dim=0)

oldVersion/V210/text/japanese_bert.py

@@ -13,8 +13,16 @@ tokenizer = AutoTokenizer.from_pretrained(LOCAL_PATH)
 models = dict()
 
 
-def get_bert_feature(text, word2ph, device=config.bert_gen_config.device):
+def get_bert_feature(
+    text,
+    word2ph,
+    device=config.bert_gen_config.device,
+    style_text=None,
+    style_weight=0.7,
+):
     text = "".join(text2sep_kata(text)[0])
+    if style_text:
+        style_text = "".join(text2sep_kata(style_text)[0])
     if (
         sys.platform == "darwin"
         and torch.backends.mps.is_available()
@@ -31,12 +39,25 @@ def get_bert_feature(text, word2ph, device=config.bert_gen_config.device):
             inputs[i] = inputs[i].to(device)
         res = models[device](**inputs, output_hidden_states=True)
         res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
+        if style_text:
+            style_inputs = tokenizer(style_text, return_tensors="pt")
+            for i in style_inputs:
+                style_inputs[i] = style_inputs[i].to(device)
+            style_res = models[device](**style_inputs, output_hidden_states=True)
+            style_res = torch.cat(style_res["hidden_states"][-3:-2], -1)[0].cpu()
+            style_res_mean = style_res.mean(0)
 
     assert len(word2ph) == len(text) + 2
     word2phone = word2ph
     phone_level_feature = []
     for i in range(len(word2phone)):
-        repeat_feature = res[i].repeat(word2phone[i], 1)
+        if style_text:
+            repeat_feature = (
+                res[i].repeat(word2phone[i], 1) * (1 - style_weight)
+                + style_res_mean.repeat(word2phone[i], 1) * style_weight
+            )
+        else:
+            repeat_feature = res[i].repeat(word2phone[i], 1)
         phone_level_feature.append(repeat_feature)
 
     phone_level_feature = torch.cat(phone_level_feature, dim=0)

re_matching.py

@@ -44,7 +44,6 @@ def text_matching(text: str) -> list:
     result = []
     for speaker, dialogue in matches:
         result.append(extract_language_and_text_updated(speaker, dialogue))
-    print(result)
     return result
 
 

resample_legacy.py

@@ -0,0 +1,71 @@
+import os
+import argparse
+import librosa
+from multiprocessing import Pool, cpu_count
+
+import soundfile
+from tqdm import tqdm
+
+from config import config
+
+
+def process(item):
+    wav_name, args = item
+    wav_path = os.path.join(args.in_dir, wav_name)
+    if os.path.exists(wav_path) and wav_path.lower().endswith(".wav"):
+        wav, sr = librosa.load(wav_path, sr=args.sr)
+        soundfile.write(os.path.join(args.out_dir, wav_name), wav, sr)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--sr",
+        type=int,
+        default=config.resample_config.sampling_rate,
+        help="sampling rate",
+    )
+    parser.add_argument(
+        "--in_dir",
+        type=str,
+        default=config.resample_config.in_dir,
+        help="path to source dir",
+    )
+    parser.add_argument(
+        "--out_dir",
+        type=str,
+        default=config.resample_config.out_dir,
+        help="path to target dir",
+    )
+    parser.add_argument(
+        "--processes",
+        type=int,
+        default=0,
+        help="cpu_processes",
+    )
+    args, _ = parser.parse_known_args()
+    # autodl 无卡模式会识别出46个cpu
+    if args.processes == 0:
+        processes = cpu_count() - 2 if cpu_count() > 4 else 1
+    else:
+        processes = args.processes
+    pool = Pool(processes=processes)
+
+    tasks = []
+
+    for dirpath, _, filenames in os.walk(args.in_dir):
+        if not os.path.isdir(args.out_dir):
+            os.makedirs(args.out_dir, exist_ok=True)
+        for filename in filenames:
+            if filename.lower().endswith(".wav"):
+                tasks.append((filename, args))
+
+    for _ in tqdm(
+        pool.imap_unordered(process, tasks),
+    ):
+        pass
+
+    pool.close()
+    pool.join()
+
+    print("音频重采样完毕!")

server_fastapi.py

@@ -204,6 +204,8 @@ if __name__ == "__main__":
         auto_split: bool,
         emotion: Optional[Union[int, str]] = None,
         reference_audio=None,
+        style_text: Optional[str] = None,
+        style_weight: float = 0.7,
     ) -> Union[Response, Dict[str, any]]:
         """TTS实现函数"""
         # 检查模型是否存在
@@ -261,6 +263,8 @@ if __name__ == "__main__":
                     device=loaded_models.models[model_id].device,
                     emotion=emotion,
                     reference_audio=ref_audio,
+                    style_text=style_text,
+                    style_weight=style_weight,
                 )
                 audio = gradio.processing_utils.convert_to_16_bit_wav(audio)
         else:
@@ -282,6 +286,8 @@ if __name__ == "__main__":
                             device=loaded_models.models[model_id].device,
                             emotion=emotion,
                             reference_audio=ref_audio,
+                            style_text=style_text,
+                            style_weight=style_weight,
                         )
                     )
                     audios.append(np.zeros(int(44100 * 0.2)))
@@ -312,6 +318,8 @@ if __name__ == "__main__":
         auto_split: bool = Query(False, description="自动切分"),
         emotion: Optional[Union[int, str]] = Query(None, description="emo"),
         reference_audio: UploadFile = File(None),
+        style_text: Optional[str] = Form(None, description="风格文本"),
+        style_weight: float = Query(0.7, description="风格权重"),
     ):
         """语音接口，若需要上传参考音频请仅使用post请求"""
         logger.info(
@@ -331,6 +339,8 @@ if __name__ == "__main__":
             auto_split=auto_split,
             emotion=emotion,
             reference_audio=reference_audio,
+            style_text=style_text,
+            style_weight=style_weight,
         )
 
     @app.get("/voice")
@@ -350,6 +360,8 @@ if __name__ == "__main__":
         auto_translate: bool = Query(False, description="自动翻译"),
         auto_split: bool = Query(False, description="自动切分"),
         emotion: Optional[Union[int, str]] = Query(None, description="emo"),
+        style_text: Optional[str] = Query(None, description="风格文本"),
+        style_weight: float = Query(0.7, description="风格权重"),
     ):
         """语音接口"""
         logger.info(
@@ -368,6 +380,8 @@ if __name__ == "__main__":
             auto_translate=auto_translate,
             auto_split=auto_split,
             emotion=emotion,
+            style_text=style_text,
+            style_weight=style_weight,
         )
 
     @app.get("/models/info")

slm/wavlm-base-plus/.gitattributes

@@ -0,0 +1,27 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bin.* filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text 
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

slm/wavlm-base-plus/README.md

@@ -0,0 +1,65 @@
+---
+language:
+- en
+datasets:
+tags:
+- speech
+inference: false
+---
+
+# WavLM-Base-Plus
+
+[Microsoft's WavLM](https://github.com/microsoft/unilm/tree/master/wavlm)
+
+The base model pretrained on 16kHz sampled speech audio. When using the model, make sure that your speech input is also sampled at 16kHz. 
+
+**Note**: This model does not have a tokenizer as it was pretrained on audio alone. In order to use this model **speech recognition**, a tokenizer should be created and the model should be fine-tuned on labeled text data. Check out [this blog](https://huggingface.co/blog/fine-tune-wav2vec2-english) for more in-detail explanation of how to fine-tune the model.
+
+The model was pre-trained on:
+
+- 60,000 hours of [Libri-Light](https://arxiv.org/abs/1912.07875)
+- 10,000 hours of [GigaSpeech](https://arxiv.org/abs/2106.06909)
+- 24,000 hours of [VoxPopuli](https://arxiv.org/abs/2101.00390)
+
+[Paper: WavLM: Large-Scale Self-Supervised Pre-Training for Full Stack Speech Processing](https://arxiv.org/abs/2110.13900)
+
+Authors: Sanyuan Chen, Chengyi Wang, Zhengyang Chen, Yu Wu, Shujie Liu, Zhuo Chen, Jinyu Li, Naoyuki Kanda, Takuya Yoshioka, Xiong Xiao, Jian Wu, Long Zhou, Shuo Ren, Yanmin Qian, Yao Qian, Jian Wu, Michael Zeng, Furu Wei
+
+**Abstract**
+*Self-supervised learning (SSL) achieves great success in speech recognition, while limited exploration has been attempted for other speech processing tasks. As speech signal contains multi-faceted information including speaker identity, paralinguistics, spoken content, etc., learning universal representations for all speech tasks is challenging. In this paper, we propose a new pre-trained model, WavLM, to solve full-stack downstream speech tasks. WavLM is built based on the HuBERT framework, with an emphasis on both spoken content modeling and speaker identity preservation. We first equip the Transformer structure with gated relative position bias to improve its capability on recognition tasks. For better speaker discrimination, we propose an utterance mixing training strategy, where additional overlapped utterances are created unsupervisely and incorporated during model training. Lastly, we scale up the training dataset from 60k hours to 94k hours. WavLM Large achieves state-of-the-art performance on the SUPERB benchmark, and brings significant improvements for various speech processing tasks on their representative benchmarks.*
+
+The original model can be found under https://github.com/microsoft/unilm/tree/master/wavlm.
+
+# Usage
+
+This is an English pre-trained speech model that has to be fine-tuned on a downstream task like speech recognition or audio classification before it can be 
+used in inference. The model was pre-trained in English and should therefore perform well only in English. The model has been shown to work well on the [SUPERB benchmark](https://superbbenchmark.org/).
+
+**Note**: The model was pre-trained on phonemes rather than characters. This means that one should make sure that the input text is converted to a sequence 
+of phonemes before fine-tuning.
+
+## Speech Recognition
+
+To fine-tune the model for speech recognition, see [the official speech recognition example](https://github.com/huggingface/transformers/tree/master/examples/pytorch/speech-recognition).
+
+## Speech Classification
+
+To fine-tune the model for speech classification, see [the official audio classification example](https://github.com/huggingface/transformers/tree/master/examples/pytorch/audio-classification).
+
+## Speaker Verification
+
+TODO
+
+## Speaker Diarization
+
+TODO
+
+# Contribution
+
+The model was contributed by [cywang](https://huggingface.co/cywang) and [patrickvonplaten](https://huggingface.co/patrickvonplaten).
+
+# License
+
+The official license can be found [here](https://github.com/microsoft/UniSpeech/blob/main/LICENSE)
+
+![design](https://raw.githubusercontent.com/patrickvonplaten/scientific_images/master/wavlm.png)

slm/wavlm-base-plus/config.json

@@ -0,0 +1,99 @@
+{
+  "_name_or_path": "wavlm-base-plus",
+  "activation_dropout": 0.0,
+  "adapter_kernel_size": 3,
+  "adapter_stride": 2,
+  "add_adapter": false,
+  "apply_spec_augment": true,
+  "architectures": [
+    "WavLMModel"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": 1,
+  "classifier_proj_size": 256,
+  "codevector_dim": 256,
+  "contrastive_logits_temperature": 0.1,
+  "conv_bias": false,
+  "conv_dim": [
+    512,
+    512,
+    512,
+    512,
+    512,
+    512,
+    512
+  ],
+  "conv_kernel": [
+    10,
+    3,
+    3,
+    3,
+    3,
+    2,
+    2
+  ],
+  "conv_stride": [
+    5,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2
+  ],
+  "ctc_loss_reduction": "sum",
+  "ctc_zero_infinity": false,
+  "diversity_loss_weight": 0.1,
+  "do_stable_layer_norm": false,
+  "eos_token_id": 2,
+  "feat_extract_activation": "gelu",
+  "feat_extract_norm": "group",
+  "feat_proj_dropout": 0.1,
+  "feat_quantizer_dropout": 0.0,
+  "final_dropout": 0.0,
+  "freeze_feat_extract_train": true,
+  "hidden_act": "gelu",
+  "hidden_dropout": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "layerdrop": 0.05,
+  "mask_channel_length": 10,
+  "mask_channel_min_space": 1,
+  "mask_channel_other": 0.0,
+  "mask_channel_prob": 0.0,
+  "mask_channel_selection": "static",
+  "mask_feature_length": 10,
+  "mask_feature_min_masks": 0,
+  "mask_feature_prob": 0.0,
+  "mask_time_length": 10,
+  "mask_time_min_masks": 2,
+  "mask_time_min_space": 1,
+  "mask_time_other": 0.0,
+  "mask_time_prob": 0.05,
+  "mask_time_selection": "static",
+  "model_type": "wavlm",
+  "no_mask_channel_overlap": false,
+  "no_mask_time_overlap": false,
+  "num_adapter_layers": 3,
+  "num_attention_heads": 12,
+  "num_buckets": 320,
+  "num_codevector_groups": 2,
+  "num_codevectors_per_group": 320,
+  "num_conv_pos_embedding_groups": 16,
+  "num_conv_pos_embeddings": 128,
+  "num_ctc_classes": 80,
+  "num_feat_extract_layers": 7,
+  "num_hidden_layers": 12,
+  "num_negatives": 100,
+  "output_hidden_size": 768,
+  "pad_token_id": 0,
+  "proj_codevector_dim": 256,
+  "replace_prob": 0.5,
+  "torch_dtype": "float32",
+  "transformers_version": "4.13.0.dev0",
+  "use_weighted_layer_sum": false,
+  "vocab_size": 32,
+  "tokenizer_class": "Wav2Vec2CTCTokenizer"
+}

slm/wavlm-base-plus/preprocessor_config.json

@@ -0,0 +1,9 @@
+{
+  "do_normalize": false,
+  "feature_extractor_type": "Wav2Vec2FeatureExtractor",
+  "feature_size": 1,
+  "padding_side": "right",
+  "padding_value": 0.0,
+  "return_attention_mask": true,
+  "sampling_rate": 16000
+}

text/__init__.py

@@ -18,13 +18,15 @@ def cleaned_text_to_sequence(cleaned_text, tones, language):
     return phones, tones, lang_ids
 
 
-def get_bert(norm_text, word2ph, language, device):
+def get_bert(norm_text, word2ph, language, device, style_text=None, style_weight=0.7):
     from .chinese_bert import get_bert_feature as zh_bert
     from .english_bert_mock import get_bert_feature as en_bert
     from .japanese_bert import get_bert_feature as jp_bert
 
     lang_bert_func_map = {"ZH": zh_bert, "EN": en_bert, "JP": jp_bert}
-    bert = lang_bert_func_map[language](norm_text, word2ph, device)
+    bert = lang_bert_func_map[language](
+        norm_text, word2ph, device, style_text, style_weight
+    )
     return bert
 
 

text/chinese_bert.py

@@ -12,7 +12,13 @@ tokenizer = AutoTokenizer.from_pretrained(LOCAL_PATH)
 models = dict()
 
 
-def get_bert_feature(text, word2ph, device=config.bert_gen_config.device):
+def get_bert_feature(
+    text,
+    word2ph,
+    device=config.bert_gen_config.device,
+    style_text=None,
+    style_weight=0.7,
+):
     if (
         sys.platform == "darwin"
         and torch.backends.mps.is_available()
@@ -29,12 +35,24 @@ def get_bert_feature(text, word2ph, device=config.bert_gen_config.device):
             inputs[i] = inputs[i].to(device)
         res = models[device](**inputs, output_hidden_states=True)
         res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
-
+        if style_text:
+            style_inputs = tokenizer(style_text, return_tensors="pt")
+            for i in style_inputs:
+                style_inputs[i] = style_inputs[i].to(device)
+            style_res = models[device](**style_inputs, output_hidden_states=True)
+            style_res = torch.cat(style_res["hidden_states"][-3:-2], -1)[0].cpu()
+            style_res_mean = style_res.mean(0)
     assert len(word2ph) == len(text) + 2
     word2phone = word2ph
     phone_level_feature = []
     for i in range(len(word2phone)):
-        repeat_feature = res[i].repeat(word2phone[i], 1)
+        if style_text:
+            repeat_feature = (
+                res[i].repeat(word2phone[i], 1) * (1 - style_weight)
+                + style_res_mean.repeat(word2phone[i], 1) * style_weight
+            )
+        else:
+            repeat_feature = res[i].repeat(word2phone[i], 1)
         phone_level_feature.append(repeat_feature)
 
     phone_level_feature = torch.cat(phone_level_feature, dim=0)

text/english.py

@@ -5,6 +5,7 @@ from g2p_en import G2p
 from transformers import DebertaV2Tokenizer
 
 from text import symbols
+from text.symbols import punctuation
 
 current_file_path = os.path.dirname(__file__)
 CMU_DICT_PATH = os.path.join(current_file_path, "cmudict.rep")
@@ -217,6 +218,8 @@ def refine_ph(phn):
     if re.search(r"\d$", phn):
         tone = int(phn[-1]) + 1
         phn = phn[:-1]
+    else:
+        tone = 3
     return phn.lower(), tone
 
 
@@ -389,45 +392,84 @@ def sep_text(text):
     return words
 
 
+def text_to_words(text):
+    tokens = tokenizer.tokenize(text)
+    words = []
+    for idx, t in enumerate(tokens):
+        if t.startswith("▁"):
+            words.append([t[1:]])
+        else:
+            if t in punctuation:
+                if idx == len(tokens) - 1:
+                    words.append([f"{t}"])
+                else:
+                    if (
+                        not tokens[idx + 1].startswith("▁")
+                        and tokens[idx + 1] not in punctuation
+                    ):
+                        if idx == 0:
+                            words.append([])
+                        words[-1].append(f"{t}")
+                    else:
+                        words.append([f"{t}"])
+            else:
+                if idx == 0:
+                    words.append([])
+                words[-1].append(f"{t}")
+    return words
+
+
 def g2p(text):
     phones = []
     tones = []
-    # word2ph = []
+    phone_len = []
-    words = sep_text(text)
+    # words = sep_text(text)
-    tokens = [tokenizer.tokenize(i) for i in words]
+    # tokens = [tokenizer.tokenize(i) for i in words]
+    words = text_to_words(text)
+
     for word in words:
-        if word.upper() in eng_dict:
+        temp_phones, temp_tones = [], []
-            phns, tns = refine_syllables(eng_dict[word.upper()])
+        if len(word) > 1:
-            phones.append([post_replace_ph(i) for i in phns])
+            if "'" in word:
-            tones.append(tns)
+                word = ["".join(word)]
-            # word2ph.append(len(phns))
+        for w in word:
-        else:
+            if w in punctuation:
-            phone_list = list(filter(lambda p: p != " ", _g2p(word)))
+                temp_phones.append(w)
-            phns = []
+                temp_tones.append(0)
-            tns = []
+                continue
-            for ph in phone_list:
+            if w.upper() in eng_dict:
-                if ph in arpa:
+                phns, tns = refine_syllables(eng_dict[w.upper()])
-                    ph, tn = refine_ph(ph)
+                temp_phones += [post_replace_ph(i) for i in phns]
-                    phns.append(ph)
+                temp_tones += tns
-                    tns.append(tn)
+                # w2ph.append(len(phns))
-                else:
+            else:
-                    phns.append(ph)
+                phone_list = list(filter(lambda p: p != " ", _g2p(w)))
-                    tns.append(0)
+                phns = []
-            phones.append([post_replace_ph(i) for i in phns])
+                tns = []
-            tones.append(tns)
+                for ph in phone_list:
-            # word2ph.append(len(phns))
+                    if ph in arpa:
-    # phones = [post_replace_ph(i) for i in phones]
+                        ph, tn = refine_ph(ph)
+                        phns.append(ph)
+                        tns.append(tn)
+                    else:
+                        phns.append(ph)
+                        tns.append(0)
+                temp_phones += [post_replace_ph(i) for i in phns]
+                temp_tones += tns
+        phones += temp_phones
+        tones += temp_tones
+        phone_len.append(len(temp_phones))
+        # phones = [post_replace_ph(i) for i in phones]
 
     word2ph = []
-    for token, phoneme in zip(tokens, phones):
+    for token, pl in zip(words, phone_len):
-        phone_len = len(phoneme)
         word_len = len(token)
 
-        aaa = distribute_phone(phone_len, word_len)
+        aaa = distribute_phone(pl, word_len)
         word2ph += aaa
 
-    phones = ["_"] + [j for i in phones for j in i] + ["_"]
+    phones = ["_"] + phones + ["_"]
-    tones = [0] + [j for i in tones for j in i] + [0]
+    tones = [0] + tones + [0]
     word2ph = [1] + word2ph + [1]
     assert len(phones) == len(tones), text
     assert len(phones) == sum(word2ph), text

text/english_bert_mock.py

@@ -13,7 +13,13 @@ tokenizer = DebertaV2Tokenizer.from_pretrained(LOCAL_PATH)
 models = dict()
 
 
-def get_bert_feature(text, word2ph, device=config.bert_gen_config.device):
+def get_bert_feature(
+    text,
+    word2ph,
+    device=config.bert_gen_config.device,
+    style_text=None,
+    style_weight=0.7,
+):
     if (
         sys.platform == "darwin"
         and torch.backends.mps.is_available()
@@ -30,11 +36,24 @@ def get_bert_feature(text, word2ph, device=config.bert_gen_config.device):
             inputs[i] = inputs[i].to(device)
         res = models[device](**inputs, output_hidden_states=True)
         res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
+        if style_text:
+            style_inputs = tokenizer(style_text, return_tensors="pt")
+            for i in style_inputs:
+                style_inputs[i] = style_inputs[i].to(device)
+            style_res = models[device](**style_inputs, output_hidden_states=True)
+            style_res = torch.cat(style_res["hidden_states"][-3:-2], -1)[0].cpu()
+            style_res_mean = style_res.mean(0)
     assert len(word2ph) == res.shape[0], (text, res.shape[0], len(word2ph))
     word2phone = word2ph
     phone_level_feature = []
     for i in range(len(word2phone)):
-        repeat_feature = res[i].repeat(word2phone[i], 1)
+        if style_text:
+            repeat_feature = (
+                res[i].repeat(word2phone[i], 1) * (1 - style_weight)
+                + style_res_mean.repeat(word2phone[i], 1) * style_weight
+            )
+        else:
+            repeat_feature = res[i].repeat(word2phone[i], 1)
         phone_level_feature.append(repeat_feature)
 
     phone_level_feature = torch.cat(phone_level_feature, dim=0)

text/japanese_bert.py

@@ -13,8 +13,16 @@ tokenizer = AutoTokenizer.from_pretrained(LOCAL_PATH)
 models = dict()
 
 
-def get_bert_feature(text, word2ph, device=config.bert_gen_config.device):
+def get_bert_feature(
+    text,
+    word2ph,
+    device=config.bert_gen_config.device,
+    style_text=None,
+    style_weight=0.7,
+):
     text = "".join(text2sep_kata(text)[0])
+    if style_text:
+        style_text = "".join(text2sep_kata(style_text)[0])
     if (
         sys.platform == "darwin"
         and torch.backends.mps.is_available()
@@ -31,12 +39,25 @@ def get_bert_feature(text, word2ph, device=config.bert_gen_config.device):
             inputs[i] = inputs[i].to(device)
         res = models[device](**inputs, output_hidden_states=True)
         res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
+        if style_text:
+            style_inputs = tokenizer(style_text, return_tensors="pt")
+            for i in style_inputs:
+                style_inputs[i] = style_inputs[i].to(device)
+            style_res = models[device](**style_inputs, output_hidden_states=True)
+            style_res = torch.cat(style_res["hidden_states"][-3:-2], -1)[0].cpu()
+            style_res_mean = style_res.mean(0)
 
     assert len(word2ph) == len(text) + 2
     word2phone = word2ph
     phone_level_feature = []
     for i in range(len(word2phone)):
-        repeat_feature = res[i].repeat(word2phone[i], 1)
+        if style_text:
+            repeat_feature = (
+                res[i].repeat(word2phone[i], 1) * (1 - style_weight)
+                + style_res_mean.repeat(word2phone[i], 1) * style_weight
+            )
+        else:
+            repeat_feature = res[i].repeat(word2phone[i], 1)
         phone_level_feature.append(repeat_feature)
 
     phone_level_feature = torch.cat(phone_level_feature, dim=0)

text/tone_sandhi.py

@@ -634,9 +634,11 @@ class ToneSandhi:
     # input seg: [('听', 'v'), ('一', 'm'), ('听', 'v')]
     # output seg: [['听一听', 'v']]
     def _merge_yi(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
-        new_seg = []
+        new_seg = [] * len(seg)
         # function 1
-        for i, (word, pos) in enumerate(seg):
+        i = 0
+        while i < len(seg):
+            word, pos = seg[i]
             if (
                 i - 1 >= 0
                 and word == "一"
@@ -645,6 +647,7 @@ class ToneSandhi:
                 and seg[i - 1][1] == "v"
             ):
                 new_seg[i - 1][0] = new_seg[i - 1][0] + "一" + new_seg[i - 1][0]
+                i += 2
             else:
                 if (
                     i - 2 >= 0
@@ -655,7 +658,8 @@ class ToneSandhi:
                     continue
                 else:
                     new_seg.append([word, pos])
-        seg = new_seg
+                i += 1
+        seg = [i for i in new_seg if len(i) > 0]
         new_seg = []
         # function 2
         for i, (word, pos) in enumerate(seg):

train_ms.py

@@ -27,8 +27,15 @@ from models import (
     SynthesizerTrn,
     MultiPeriodDiscriminator,
     DurationDiscriminator,
+    WavLMDiscriminator,
+)
+from losses import (
+    generator_loss,
+    discriminator_loss,
+    feature_loss,
+    kl_loss,
+    WavLMLoss,
 )
-from losses import generator_loss, discriminator_loss, feature_loss, kl_loss
 from mel_processing import mel_spectrogram_torch, spec_to_mel_torch
 from text.symbols import symbols
 
@@ -42,7 +49,6 @@ torch.backends.cuda.enable_flash_sdp(True)
 torch.backends.cuda.enable_mem_efficient_sdp(
     True
 )  # Not available if torch version is lower than 2.0
-torch.backends.cuda.enable_math_sdp(True)
 global_step = 0
 
 
@@ -173,6 +179,8 @@ def run():
             0.1,
             gin_channels=hps.model.gin_channels if hps.data.n_speakers != 0 else 0,
         ).cuda(local_rank)
+    else:
+        net_dur_disc = None
     if (
         "use_spk_conditioned_encoder" in hps.model.keys()
         and hps.model.use_spk_conditioned_encoder is True
@@ -210,6 +218,9 @@ def run():
             param.requires_grad = False
 
     net_d = MultiPeriodDiscriminator(hps.model.use_spectral_norm).cuda(local_rank)
+    net_wd = WavLMDiscriminator(
+        hps.model.slm.hidden, hps.model.slm.nlayers, hps.model.slm.initial_channel
+    ).cuda(local_rank)
     optim_g = torch.optim.AdamW(
         filter(lambda p: p.requires_grad, net_g.parameters()),
         hps.train.learning_rate,
@@ -222,6 +233,12 @@ def run():
         betas=hps.train.betas,
         eps=hps.train.eps,
     )
+    optim_wd = torch.optim.AdamW(
+        net_wd.parameters(),
+        hps.train.learning_rate,
+        betas=hps.train.betas,
+        eps=hps.train.eps,
+    )
     if net_dur_disc is not None:
         optim_dur_disc = torch.optim.AdamW(
             net_dur_disc.parameters(),
@@ -233,12 +250,11 @@ def run():
         optim_dur_disc = None
     net_g = DDP(net_g, device_ids=[local_rank], bucket_cap_mb=512)
     net_d = DDP(net_d, device_ids=[local_rank], bucket_cap_mb=512)
-    dur_resume_lr = None
+    net_wd = DDP(net_wd, device_ids=[local_rank], bucket_cap_mb=512)
     if net_dur_disc is not None:
         net_dur_disc = DDP(
             net_dur_disc,
             device_ids=[local_rank],
-            find_unused_parameters=True,
             bucket_cap_mb=512,
         )
 
@@ -250,9 +266,10 @@ def run():
             token=config.openi_token,
             mirror=config.mirror,
         )
-
+    dur_resume_lr = hps.train.learning_rate
-    try:
+    wd_resume_lr = hps.train.learning_rate
-        if net_dur_disc is not None:
+    if net_dur_disc is not None:
+        try:
             _, _, dur_resume_lr, epoch_str = utils.load_checkpoint(
                 utils.latest_checkpoint_path(hps.model_dir, "DUR_*.pth"),
                 net_dur_disc,
@@ -261,28 +278,32 @@ def run():
                 if "skip_optimizer" in hps.train
                 else True,
             )
-            _, optim_g, g_resume_lr, epoch_str = utils.load_checkpoint(
-                utils.latest_checkpoint_path(hps.model_dir, "G_*.pth"),
-                net_g,
-                optim_g,
-                skip_optimizer=hps.train.skip_optimizer
-                if "skip_optimizer" in hps.train
-                else True,
-            )
-            _, optim_d, d_resume_lr, epoch_str = utils.load_checkpoint(
-                utils.latest_checkpoint_path(hps.model_dir, "D_*.pth"),
-                net_d,
-                optim_d,
-                skip_optimizer=hps.train.skip_optimizer
-                if "skip_optimizer" in hps.train
-                else True,
-            )
-            if not optim_g.param_groups[0].get("initial_lr"):
-                optim_g.param_groups[0]["initial_lr"] = g_resume_lr
-            if not optim_d.param_groups[0].get("initial_lr"):
-                optim_d.param_groups[0]["initial_lr"] = d_resume_lr
             if not optim_dur_disc.param_groups[0].get("initial_lr"):
                 optim_dur_disc.param_groups[0]["initial_lr"] = dur_resume_lr
+        except:
+            print("Initialize dur_disc")
+
+    try:
+        _, optim_g, g_resume_lr, epoch_str = utils.load_checkpoint(
+            utils.latest_checkpoint_path(hps.model_dir, "G_*.pth"),
+            net_g,
+            optim_g,
+            skip_optimizer=hps.train.skip_optimizer
+            if "skip_optimizer" in hps.train
+            else True,
+        )
+        _, optim_d, d_resume_lr, epoch_str = utils.load_checkpoint(
+            utils.latest_checkpoint_path(hps.model_dir, "D_*.pth"),
+            net_d,
+            optim_d,
+            skip_optimizer=hps.train.skip_optimizer
+            if "skip_optimizer" in hps.train
+            else True,
+        )
+        if not optim_g.param_groups[0].get("initial_lr"):
+            optim_g.param_groups[0]["initial_lr"] = g_resume_lr
+        if not optim_d.param_groups[0].get("initial_lr"):
+            optim_d.param_groups[0]["initial_lr"] = d_resume_lr
 
         epoch_str = max(epoch_str, 1)
         # global_step = (epoch_str - 1) * len(train_loader)
@@ -297,21 +318,36 @@ def run():
         epoch_str = 1
         global_step = 0
 
+    try:
+        _, optim_wd, wd_resume_lr, epoch_str = utils.load_checkpoint(
+            utils.latest_checkpoint_path(hps.model_dir, "WD_*.pth"),
+            net_wd,
+            optim_wd,
+            skip_optimizer=hps.train.skip_optimizer
+            if "skip_optimizer" in hps.train
+            else True,
+        )
+        if not optim_wd.param_groups[0].get("initial_lr"):
+            optim_wd.param_groups[0]["initial_lr"] = wd_resume_lr
+    except Exception as e:
+        print(e)
+
     scheduler_g = torch.optim.lr_scheduler.ExponentialLR(
         optim_g, gamma=hps.train.lr_decay, last_epoch=epoch_str - 2
     )
     scheduler_d = torch.optim.lr_scheduler.ExponentialLR(
         optim_d, gamma=hps.train.lr_decay, last_epoch=epoch_str - 2
     )
+    scheduler_wd = torch.optim.lr_scheduler.ExponentialLR(
+        optim_wd, gamma=hps.train.lr_decay, last_epoch=epoch_str - 2
+    )
     if net_dur_disc is not None:
-        if not optim_dur_disc.param_groups[0].get("initial_lr"):
-            optim_dur_disc.param_groups[0]["initial_lr"] = dur_resume_lr
         scheduler_dur_disc = torch.optim.lr_scheduler.ExponentialLR(
             optim_dur_disc, gamma=hps.train.lr_decay, last_epoch=epoch_str - 2
         )
     else:
         scheduler_dur_disc = None
-    scaler = GradScaler(enabled=hps.train.fp16_run)
+    scaler = GradScaler(enabled=hps.train.bf16_run)
 
     for epoch in range(epoch_str, hps.train.epochs + 1):
         if rank == 0:
@@ -320,9 +356,9 @@ def run():
                 local_rank,
                 epoch,
                 hps,
-                [net_g, net_d, net_dur_disc],
+                [net_g, net_d, net_dur_disc, net_wd],
-                [optim_g, optim_d, optim_dur_disc],
+                [optim_g, optim_d, optim_dur_disc, optim_wd],
-                [scheduler_g, scheduler_d, scheduler_dur_disc],
+                [scheduler_g, scheduler_d, scheduler_dur_disc, scheduler_wd],
                 scaler,
                 [train_loader, eval_loader],
                 logger,
@@ -334,9 +370,9 @@ def run():
                 local_rank,
                 epoch,
                 hps,
-                [net_g, net_d, net_dur_disc],
+                [net_g, net_d, net_dur_disc, net_wd],
-                [optim_g, optim_d, optim_dur_disc],
+                [optim_g, optim_d, optim_dur_disc, optim_wd],
-                [scheduler_g, scheduler_d, scheduler_dur_disc],
+                [scheduler_g, scheduler_d, scheduler_dur_disc, scheduler_wd],
                 scaler,
                 [train_loader, None],
                 None,
@@ -361,18 +397,25 @@ def train_and_evaluate(
     logger,
     writers,
 ):
-    net_g, net_d, net_dur_disc = nets
+    net_g, net_d, net_dur_disc, net_wd = nets
-    optim_g, optim_d, optim_dur_disc = optims
+    optim_g, optim_d, optim_dur_disc, optim_wd = optims
-    scheduler_g, scheduler_d, scheduler_dur_disc = schedulers
+    scheduler_g, scheduler_d, scheduler_dur_disc, scheduler_wd = schedulers
     train_loader, eval_loader = loaders
     if writers is not None:
         writer, writer_eval = writers
+    wl = WavLMLoss(
+        hps.model.slm.model,
+        net_wd,
+        hps.data.sampling_rate,
+        hps.model.slm.sr,
+    ).to(local_rank)
 
     train_loader.batch_sampler.set_epoch(epoch)
     global global_step
 
     net_g.train()
     net_d.train()
+    net_wd.train()
     if net_dur_disc is not None:
         net_dur_disc.train()
     for batch_idx, (
@@ -388,7 +431,6 @@ def train_and_evaluate(
         bert,
         ja_bert,
         en_bert,
-        emo,
     ) in enumerate(tqdm(train_loader)):
         if net_g.module.use_noise_scaled_mas:
             current_mas_noise_scale = (
@@ -411,9 +453,8 @@ def train_and_evaluate(
         bert = bert.cuda(local_rank, non_blocking=True)
         ja_bert = ja_bert.cuda(local_rank, non_blocking=True)
         en_bert = en_bert.cuda(local_rank, non_blocking=True)
-        emo = emo.cuda(local_rank, non_blocking=True)
 
-        with autocast(enabled=hps.train.fp16_run):
+        with autocast(enabled=hps.train.bf16_run, dtype=torch.bfloat16):
             (
                 y_hat,
                 l_length,
@@ -422,9 +463,8 @@ def train_and_evaluate(
                 x_mask,
                 z_mask,
                 (z, z_p, m_p, logs_p, m_q, logs_q),
-                (hidden_x, logw, logw_),
+                (hidden_x, logw, logw_, logw_sdp),
                 g,
-                loss_commit,
             ) = net_g(
                 x,
                 x_lengths,
@@ -436,7 +476,6 @@ def train_and_evaluate(
                 bert,
                 ja_bert,
                 en_bert,
-                emo,
             )
             mel = spec_to_mel_torch(
                 spec,
@@ -450,7 +489,7 @@ def train_and_evaluate(
                 mel, ids_slice, hps.train.segment_size // hps.data.hop_length
             )
             y_hat_mel = mel_spectrogram_torch(
-                y_hat.squeeze(1),
+                y_hat.squeeze(1).float(),
                 hps.data.filter_length,
                 hps.data.n_mel_channels,
                 hps.data.sampling_rate,
@@ -466,7 +505,7 @@ def train_and_evaluate(
 
             # Discriminator
             y_d_hat_r, y_d_hat_g, _, _ = net_d(y, y_hat.detach())
-            with autocast(enabled=False):
+            with autocast(enabled=hps.train.bf16_run, dtype=torch.bfloat16):
                 loss_disc, losses_disc_r, losses_disc_g = discriminator_loss(
                     y_d_hat_r, y_d_hat_g
                 )
@@ -475,11 +514,20 @@ def train_and_evaluate(
                 y_dur_hat_r, y_dur_hat_g = net_dur_disc(
                     hidden_x.detach(),
                     x_mask.detach(),
-                    logw.detach(),
                     logw_.detach(),
+                    logw.detach(),
                     g.detach(),
                 )
-                with autocast(enabled=False):
+                y_dur_hat_r_sdp, y_dur_hat_g_sdp = net_dur_disc(
+                    hidden_x.detach(),
+                    x_mask.detach(),
+                    logw_.detach(),
+                    logw_sdp.detach(),
+                    g.detach(),
+                )
+                y_dur_hat_r = y_dur_hat_r + y_dur_hat_r_sdp
+                y_dur_hat_g = y_dur_hat_g + y_dur_hat_g_sdp
+                with autocast(enabled=hps.train.bf16_run, dtype=torch.bfloat16):
                     # TODO: I think need to mean using the mask, but for now, just mean all
                     (
                         loss_dur_disc,
@@ -490,31 +538,60 @@ def train_and_evaluate(
                 optim_dur_disc.zero_grad()
                 scaler.scale(loss_dur_disc_all).backward()
                 scaler.unscale_(optim_dur_disc)
-                commons.clip_grad_value_(net_dur_disc.parameters(), None)
+                # torch.nn.utils.clip_grad_norm_(
+                #     parameters=net_dur_disc.parameters(), max_norm=100
+                # )
+                grad_norm_dur = commons.clip_grad_value_(
+                    net_dur_disc.parameters(), None
+                )
                 scaler.step(optim_dur_disc)
 
         optim_d.zero_grad()
         scaler.scale(loss_disc_all).backward()
         scaler.unscale_(optim_d)
+        if getattr(hps.train, "bf16_run", False):
+            torch.nn.utils.clip_grad_norm_(parameters=net_d.parameters(), max_norm=200)
         grad_norm_d = commons.clip_grad_value_(net_d.parameters(), None)
         scaler.step(optim_d)
 
-        with autocast(enabled=hps.train.fp16_run):
+        with autocast(enabled=hps.train.bf16_run, dtype=torch.bfloat16):
+            loss_slm = wl.discriminator(
+                y.detach().squeeze(), y_hat.detach().squeeze()
+            ).mean()
+
+        optim_wd.zero_grad()
+        scaler.scale(loss_slm).backward()
+        scaler.unscale_(optim_wd)
+        # torch.nn.utils.clip_grad_norm_(parameters=net_wd.parameters(), max_norm=200)
+        grad_norm_wd = commons.clip_grad_value_(net_wd.parameters(), None)
+        scaler.step(optim_wd)
+
+        with autocast(enabled=hps.train.bf16_run, dtype=torch.bfloat16):
             # Generator
             y_d_hat_r, y_d_hat_g, fmap_r, fmap_g = net_d(y, y_hat)
             if net_dur_disc is not None:
-                y_dur_hat_r, y_dur_hat_g = net_dur_disc(
+                _, y_dur_hat_g = net_dur_disc(hidden_x, x_mask, logw_, logw, g)
-                    hidden_x, x_mask, logw, logw_, g
+                _, y_dur_hat_g_sdp = net_dur_disc(hidden_x, x_mask, logw_, logw_sdp, g)
-                )
+                y_dur_hat_g = y_dur_hat_g + y_dur_hat_g_sdp
-            with autocast(enabled=False):
+            with autocast(enabled=hps.train.bf16_run, dtype=torch.bfloat16):
                 loss_dur = torch.sum(l_length.float())
                 loss_mel = F.l1_loss(y_mel, y_hat_mel) * hps.train.c_mel
                 loss_kl = kl_loss(z_p, logs_q, m_p, logs_p, z_mask) * hps.train.c_kl
 
                 loss_fm = feature_loss(fmap_r, fmap_g)
                 loss_gen, losses_gen = generator_loss(y_d_hat_g)
+
+                loss_lm = wl(y.detach().squeeze(), y_hat.squeeze()).mean()
+                loss_lm_gen = wl.generator(y_hat.squeeze())
+
                 loss_gen_all = (
-                    loss_gen + loss_fm + loss_mel + loss_dur + loss_kl + loss_commit
+                    loss_gen
+                    + loss_fm
+                    + loss_mel
+                    + loss_dur
+                    + loss_kl
+                    + loss_lm
+                    + loss_lm_gen
                 )
                 if net_dur_disc is not None:
                     loss_dur_gen, losses_dur_gen = generator_loss(y_dur_hat_g)
@@ -522,6 +599,8 @@ def train_and_evaluate(
         optim_g.zero_grad()
         scaler.scale(loss_gen_all).backward()
         scaler.unscale_(optim_g)
+        if getattr(hps.train, "bf16_run", False):
+            torch.nn.utils.clip_grad_norm_(parameters=net_g.parameters(), max_norm=500)
         grad_norm_g = commons.clip_grad_value_(net_g.parameters(), None)
         scaler.step(optim_g)
         scaler.update()
@@ -540,9 +619,12 @@ def train_and_evaluate(
                 scalar_dict = {
                     "loss/g/total": loss_gen_all,
                     "loss/d/total": loss_disc_all,
+                    "loss/wd/total": loss_slm,
                     "learning_rate": lr,
                     "grad_norm_d": grad_norm_d,
                     "grad_norm_g": grad_norm_g,
+                    "grad_norm_dur": grad_norm_dur,
+                    "grad_norm_wd": grad_norm_wd,
                 }
                 scalar_dict.update(
                     {
@@ -550,6 +632,8 @@ def train_and_evaluate(
                         "loss/g/mel": loss_mel,
                         "loss/g/dur": loss_dur,
                         "loss/g/kl": loss_kl,
+                        "loss/g/lm": loss_lm,
+                        "loss/g/lm_gen": loss_lm_gen,
                     }
                 )
                 scalar_dict.update(
@@ -562,6 +646,30 @@ def train_and_evaluate(
                     {"loss/d_g/{}".format(i): v for i, v in enumerate(losses_disc_g)}
                 )
 
+                if net_dur_disc is not None:
+                    scalar_dict.update({"loss/dur_disc/total": loss_dur_disc_all})
+
+                    scalar_dict.update(
+                        {
+                            "loss/dur_disc_g/{}".format(i): v
+                            for i, v in enumerate(losses_dur_disc_g)
+                        }
+                    )
+                    scalar_dict.update(
+                        {
+                            "loss/dur_disc_r/{}".format(i): v
+                            for i, v in enumerate(losses_dur_disc_r)
+                        }
+                    )
+
+                    scalar_dict.update({"loss/g/dur_gen": loss_dur_gen})
+                    scalar_dict.update(
+                        {
+                            "loss/g/dur_gen_{}".format(i): v
+                            for i, v in enumerate(losses_dur_gen)
+                        }
+                    )
+
                 image_dict = {
                     "slice/mel_org": utils.plot_spectrogram_to_numpy(
                         y_mel[0].data.cpu().numpy()
@@ -599,6 +707,13 @@ def train_and_evaluate(
                     epoch,
                     os.path.join(hps.model_dir, "D_{}.pth".format(global_step)),
                 )
+                utils.save_checkpoint(
+                    net_wd,
+                    optim_wd,
+                    hps.train.learning_rate,
+                    epoch,
+                    os.path.join(hps.model_dir, "WD_{}.pth".format(global_step)),
+                )
                 if net_dur_disc is not None:
                     utils.save_checkpoint(
                         net_dur_disc,
@@ -642,7 +757,6 @@ def evaluate(hps, generator, eval_loader, writer_eval):
             bert,
             ja_bert,
             en_bert,
-            emo,
         ) in enumerate(eval_loader):
             x, x_lengths = x.cuda(), x_lengths.cuda()
             spec, spec_lengths = spec.cuda(), spec_lengths.cuda()
@@ -653,7 +767,6 @@ def evaluate(hps, generator, eval_loader, writer_eval):
             en_bert = en_bert.cuda()
             tone = tone.cuda()
             language = language.cuda()
-            emo = emo.cuda()
             for use_sdp in [True, False]:
                 y_hat, attn, mask, *_ = generator.module.infer(
                     x,
@@ -664,7 +777,6 @@ def evaluate(hps, generator, eval_loader, writer_eval):
                     bert,
                     ja_bert,
                     en_bert,
-                    emo,
                     y=spec,
                     max_len=1000,
                     sdp_ratio=0.0 if not use_sdp else 1.0,

utils.py

@@ -301,7 +301,11 @@ def clean_checkpoints(path_to_models="logs/44k/", n_ckpts_to_keep=2, sort_by_tim
 
     to_del = [
         os.path.join(path_to_models, fn)
-        for fn in (x_sorted("G")[:-n_ckpts_to_keep] + x_sorted("D")[:-n_ckpts_to_keep])
+        for fn in (
+            x_sorted("G")[:-n_ckpts_to_keep]
+            + x_sorted("D")[:-n_ckpts_to_keep]
+            + x_sorted("WD")[:-n_ckpts_to_keep]
+        )
     ]
 
     def del_info(fn):

webui.py

@@ -42,6 +42,8 @@ def generate_audio(
     language,
     reference_audio,
     emotion,
+    style_text,
+    style_weight,
     skip_start=False,
     skip_end=False,
 ):
@@ -49,8 +51,8 @@ def generate_audio(
     # silence = np.zeros(hps.data.sampling_rate // 2, dtype=np.int16)
     with torch.no_grad():
         for idx, piece in enumerate(slices):
-            skip_start = (idx != 0) and skip_start
+            skip_start = idx != 0
-            skip_end = (idx != len(slices) - 1) and skip_end
+            skip_end = idx != len(slices) - 1
             audio = infer(
                 piece,
                 reference_audio=reference_audio,
@@ -66,10 +68,11 @@ def generate_audio(
                 device=device,
                 skip_start=skip_start,
                 skip_end=skip_end,
+                style_text=style_text,
+                style_weight=style_weight,
             )
             audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
             audio_list.append(audio16bit)
-            # audio_list.append(silence)  # 将静音添加到列表中
     return audio_list
 
 
@@ -90,8 +93,8 @@ def generate_audio_multilang(
     # silence = np.zeros(hps.data.sampling_rate // 2, dtype=np.int16)
     with torch.no_grad():
         for idx, piece in enumerate(slices):
-            skip_start = (idx != 0) and skip_start
+            skip_start = idx != 0
-            skip_end = (idx != len(slices) - 1) and skip_end
+            skip_end = idx != len(slices) - 1
             audio = infer_multilang(
                 piece,
                 reference_audio=reference_audio,
@@ -110,7 +113,6 @@ def generate_audio_multilang(
             )
             audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
             audio_list.append(audio16bit)
-            # audio_list.append(silence)  # 将静音添加到列表中
     return audio_list
 
 
@@ -127,63 +129,50 @@ def tts_split(
     interval_between_sent,
     reference_audio,
     emotion,
+    style_text,
+    style_weight,
 ):
-    if language == "mix":
-        return ("invalid", None)
     while text.find("\n\n") != -1:
         text = text.replace("\n\n", "\n")
+    text = text.replace("|", "")
     para_list = re_matching.cut_para(text)
+    para_list = [p for p in para_list if p != ""]
     audio_list = []
-    if not cut_by_sent:
+    for p in para_list:
-        for idx, p in enumerate(para_list):
+        if not cut_by_sent:
-            skip_start = idx != 0
+            audio_list += process_text(
-            skip_end = idx != len(para_list) - 1
-            audio = infer(
                 p,
-                reference_audio=reference_audio,
+                speaker,
-                emotion=emotion,
+                sdp_ratio,
-                sdp_ratio=sdp_ratio,
+                noise_scale,
-                noise_scale=noise_scale,
+                noise_scale_w,
-                noise_scale_w=noise_scale_w,
+                length_scale,
-                length_scale=length_scale,
+                language,
-                sid=speaker,
+                reference_audio,
-                language=language,
+                emotion,
-                hps=hps,
+                style_text,
-                net_g=net_g,
+                style_weight,
-                device=device,
-                skip_start=skip_start,
-                skip_end=skip_end,
             )
-            audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
-            audio_list.append(audio16bit)
             silence = np.zeros((int)(44100 * interval_between_para), dtype=np.int16)
             audio_list.append(silence)
-    else:
+        else:
-        for idx, p in enumerate(para_list):
-            skip_start = idx != 0
-            skip_end = idx != len(para_list) - 1
             audio_list_sent = []
             sent_list = re_matching.cut_sent(p)
-            for idx, s in enumerate(sent_list):
+            sent_list = [s for s in sent_list if s != ""]
-                skip_start = (idx != 0) and skip_start
+            for s in sent_list:
-                skip_end = (idx != len(sent_list) - 1) and skip_end
+                audio_list_sent += process_text(
-                audio = infer(
                     s,
-                    reference_audio=reference_audio,
+                    speaker,
-                    emotion=emotion,
+                    sdp_ratio,
-                    sdp_ratio=sdp_ratio,
+                    noise_scale,
-                    noise_scale=noise_scale,
+                    noise_scale_w,
-                    noise_scale_w=noise_scale_w,
+                    length_scale,
-                    length_scale=length_scale,
+                    language,
-                    sid=speaker,
+                    reference_audio,
-                    language=language,
+                    emotion,
-                    hps=hps,
+                    style_text,
-                    net_g=net_g,
+                    style_weight,
-                    device=device,
-                    skip_start=skip_start,
-                    skip_end=skip_end,
                 )
-                audio_list_sent.append(audio)
                 silence = np.zeros((int)(44100 * interval_between_sent))
                 audio_list_sent.append(silence)
             if (interval_between_para - interval_between_sent) > 0:
@@ -196,7 +185,118 @@ def tts_split(
             )  # 对完整句子做音量归一
             audio_list.append(audio16bit)
     audio_concat = np.concatenate(audio_list)
-    return ("Success", (44100, audio_concat))
+    return ("Success", (hps.data.sampling_rate, audio_concat))
+
+
+def process_mix(slice):
+    _speaker = slice.pop()
+    _text, _lang = [], []
+    for lang, content in slice:
+        content = content.split("|")
+        content = [part for part in content if part != ""]
+        if len(content) == 0:
+            continue
+        if len(_text) == 0:
+            _text = [[part] for part in content]
+            _lang = [[lang] for part in content]
+        else:
+            _text[-1].append(content[0])
+            _lang[-1].append(lang)
+            if len(content) > 1:
+                _text += [[part] for part in content[1:]]
+                _lang += [[lang] for part in content[1:]]
+    return _text, _lang, _speaker
+
+
+def process_auto(text):
+    _text, _lang = [], []
+    for slice in text.split("|"):
+        if slice == "":
+            continue
+        temp_text, temp_lang = [], []
+        sentences_list = split_by_language(slice, target_languages=["zh", "ja", "en"])
+        for sentence, lang in sentences_list:
+            if sentence == "":
+                continue
+            temp_text.append(sentence)
+            temp_lang.append(lang.upper())
+        _text.append(temp_text)
+        _lang.append(temp_lang)
+    return _text, _lang
+
+
+def process_text(
+    text: str,
+    speaker,
+    sdp_ratio,
+    noise_scale,
+    noise_scale_w,
+    length_scale,
+    language,
+    reference_audio,
+    emotion,
+    style_text=None,
+    style_weight=0,
+):
+    audio_list = []
+    if language == "mix":
+        bool_valid, str_valid = re_matching.validate_text(text)
+        if not bool_valid:
+            return str_valid, (
+                hps.data.sampling_rate,
+                np.concatenate([np.zeros(hps.data.sampling_rate // 2)]),
+            )
+        for slice in re_matching.text_matching(text):
+            _text, _lang, _speaker = process_mix(slice)
+            if _speaker is None:
+                continue
+            print(f"Text: {_text}\nLang: {_lang}")
+            audio_list.extend(
+                generate_audio_multilang(
+                    _text,
+                    sdp_ratio,
+                    noise_scale,
+                    noise_scale_w,
+                    length_scale,
+                    _speaker,
+                    _lang,
+                    reference_audio,
+                    emotion,
+                )
+            )
+    elif language.lower() == "auto":
+        _text, _lang = process_auto(text)
+        print(f"Text: {_text}\nLang: {_lang}")
+        audio_list.extend(
+            generate_audio_multilang(
+                _text,
+                sdp_ratio,
+                noise_scale,
+                noise_scale_w,
+                length_scale,
+                speaker,
+                _lang,
+                reference_audio,
+                emotion,
+            )
+        )
+    else:
+        audio_list.extend(
+            generate_audio(
+                text.split("|"),
+                sdp_ratio,
+                noise_scale,
+                noise_scale_w,
+                length_scale,
+                speaker,
+                language,
+                reference_audio,
+                emotion,
+                style_text,
+                style_weight,
+            )
+        )
+    return audio_list
 
 
 def tts_fn(
@@ -210,7 +310,11 @@ def tts_fn(
     reference_audio,
     emotion,
     prompt_mode,
+    style_text=None,
+    style_weight=0,
 ):
+    if style_text == "":
+        style_text = None
     if prompt_mode == "Audio prompt":
         if reference_audio == None:
             return ("Invalid audio prompt", None)
@@ -218,147 +322,35 @@ def tts_fn(
             reference_audio = load_audio(reference_audio)[1]
     else:
         reference_audio = None
-    audio_list = []
+
-    if language == "mix":
+    audio_list = process_text(
-        bool_valid, str_valid = re_matching.validate_text(text)
+        text,
-        if not bool_valid:
+        speaker,
-            return str_valid, (
+        sdp_ratio,
-                hps.data.sampling_rate,
+        noise_scale,
-                np.concatenate([np.zeros(hps.data.sampling_rate // 2)]),
+        noise_scale_w,
-            )
+        length_scale,
-        result = []
+        language,
-        for slice in re_matching.text_matching(text):
+        reference_audio,
-            _speaker = slice.pop()
+        emotion,
-            temp_contant = []
+        style_text,
-            temp_lang = []
+        style_weight,
-            for lang, content in slice:
+    )
-                if "|" in content:
-                    temp = []
-                    temp_ = []
-                    for i in content.split("|"):
-                        if i != "":
-                            temp.append([i])
-                            temp_.append([lang])
-                        else:
-                            temp.append([])
-                            temp_.append([])
-                    temp_contant += temp
-                    temp_lang += temp_
-                else:
-                    if len(temp_contant) == 0:
-                        temp_contant.append([])
-                        temp_lang.append([])
-                    temp_contant[-1].append(content)
-                    temp_lang[-1].append(lang)
-            for i, j in zip(temp_lang, temp_contant):
-                result.append([*zip(i, j), _speaker])
-        for i, one in enumerate(result):
-            skip_start = i != 0
-            skip_end = i != len(result) - 1
-            _speaker = one.pop()
-            idx = 0
-            while idx < len(one):
-                text_to_generate = []
-                lang_to_generate = []
-                while True:
-                    lang, content = one[idx]
-                    temp_text = [content]
-                    if len(text_to_generate) > 0:
-                        text_to_generate[-1] += [temp_text.pop(0)]
-                        lang_to_generate[-1] += [lang]
-                    if len(temp_text) > 0:
-                        text_to_generate += [[i] for i in temp_text]
-                        lang_to_generate += [[lang]] * len(temp_text)
-                    if idx + 1 < len(one):
-                        idx += 1
-                    else:
-                        break
-                skip_start = (idx != 0) and skip_start
-                skip_end = (idx != len(one) - 1) and skip_end
-                print(text_to_generate, lang_to_generate)
-                audio_list.extend(
-                    generate_audio_multilang(
-                        text_to_generate,
-                        sdp_ratio,
-                        noise_scale,
-                        noise_scale_w,
-                        length_scale,
-                        _speaker,
-                        lang_to_generate,
-                        reference_audio,
-                        emotion,
-                        skip_start,
-                        skip_end,
-                    )
-                )
-                idx += 1
-    elif language.lower() == "auto":
-        for idx, slice in enumerate(text.split("|")):
-            if slice == "":
-                continue
-            skip_start = idx != 0
-            skip_end = idx != len(text.split("|")) - 1
-            sentences_list = split_by_language(
-                slice, target_languages=["zh", "ja", "en"]
-            )
-            idx = 0
-            while idx < len(sentences_list):
-                text_to_generate = []
-                lang_to_generate = []
-                while True:
-                    content, lang = sentences_list[idx]
-                    temp_text = [content]
-                    lang = lang.upper()
-                    if lang == "JA":
-                        lang = "JP"
-                    if len(text_to_generate) > 0:
-                        text_to_generate[-1] += [temp_text.pop(0)]
-                        lang_to_generate[-1] += [lang]
-                    if len(temp_text) > 0:
-                        text_to_generate += [[i] for i in temp_text]
-                        lang_to_generate += [[lang]] * len(temp_text)
-                    if idx + 1 < len(sentences_list):
-                        idx += 1
-                    else:
-                        break
-                skip_start = (idx != 0) and skip_start
-                skip_end = (idx != len(sentences_list) - 1) and skip_end
-                print(text_to_generate, lang_to_generate)
-                audio_list.extend(
-                    generate_audio_multilang(
-                        text_to_generate,
-                        sdp_ratio,
-                        noise_scale,
-                        noise_scale_w,
-                        length_scale,
-                        speaker,
-                        lang_to_generate,
-                        reference_audio,
-                        emotion,
-                        skip_start,
-                        skip_end,
-                    )
-                )
-                idx += 1
-    else:
-        audio_list.extend(
-            generate_audio(
-                text.split("|"),
-                sdp_ratio,
-                noise_scale,
-                noise_scale_w,
-                length_scale,
-                speaker,
-                language,
-                reference_audio,
-                emotion,
-            )
-        )
 
     audio_concat = np.concatenate(audio_list)
     return "Success", (hps.data.sampling_rate, audio_concat)
 
 
+def format_utils(text, speaker):
+    _text, _lang = process_auto(text)
+    res = f"[{speaker}]"
+    for lang_s, content_s in zip(_lang, _text):
+        for lang, content in zip(lang_s, content_s):
+            res += f"<{lang.lower()}>{content}"
+        res += "|"
+    return "mix", res[:-1]
+
+
 def load_audio(path):
     audio, sr = librosa.load(path, 48000)
     # audio = librosa.resample(audio, 44100, 48000)
@@ -408,34 +400,37 @@ if __name__ == "__main__":
                 )
                 trans = gr.Button("中翻日", variant="primary")
                 slicer = gr.Button("快速切分", variant="primary")
+                formatter = gr.Button("检测语言，并整理为 MIX 格式", variant="primary")
                 speaker = gr.Dropdown(
                     choices=speakers, value=speakers[0], label="Speaker"
                 )
                 _ = gr.Markdown(
-                    value="提示模式（Prompt mode）：可选文字提示或音频提示，用于生成文字或音频指定风格的声音。\n"
+                    value="提示模式（Prompt mode）：可选文字提示或音频提示，用于生成文字或音频指定风格的声音。\n",
+                    visible=False,
                 )
                 prompt_mode = gr.Radio(
                     ["Text prompt", "Audio prompt"],
                     label="Prompt Mode",
                     value="Text prompt",
+                    visible=False,
                 )
                 text_prompt = gr.Textbox(
                     label="Text prompt",
                     placeholder="用文字描述生成风格。如：Happy",
                     value="Happy",
-                    visible=True,
+                    visible=False,
                 )
                 audio_prompt = gr.Audio(
                     label="Audio prompt", type="filepath", visible=False
                 )
                 sdp_ratio = gr.Slider(
-                    minimum=0, maximum=1, value=0.2, step=0.1, label="SDP Ratio"
+                    minimum=0, maximum=1, value=0.5, step=0.1, label="SDP Ratio"
                 )
                 noise_scale = gr.Slider(
                     minimum=0.1, maximum=2, value=0.6, step=0.1, label="Noise"
                 )
                 noise_scale_w = gr.Slider(
-                    minimum=0.1, maximum=2, value=0.8, step=0.1, label="Noise_W"
+                    minimum=0.1, maximum=2, value=0.9, step=0.1, label="Noise_W"
                 )
                 length_scale = gr.Slider(
                     minimum=0.1, maximum=2, value=1.0, step=0.1, label="Length"
@@ -445,6 +440,21 @@ if __name__ == "__main__":
                 )
                 btn = gr.Button("生成音频！", variant="primary")
             with gr.Column():
+                with gr.Accordion("融合文本语义", open=False):
+                    gr.Markdown(
+                        value="使用辅助文本的语意来辅助生成对话（语言保持与主文本相同）\n\n"
+                        "**注意**：不要使用**指令式文本**（如：开心），要使用**带有强烈情感的文本**（如：我好快乐！！！）\n\n"
+                        "效果较不明确，留空即为不使用该功能"
+                    )
+                    style_text = gr.Textbox(label="辅助文本")
+                    style_weight = gr.Slider(
+                        minimum=0,
+                        maximum=1,
+                        value=0.7,
+                        step=0.1,
+                        label="Weight",
+                        info="主文本和辅助文本的bert混合比率，0表示仅主文本，1表示仅辅助文本",
+                    )
                 with gr.Row():
                     with gr.Column():
                         interval_between_sent = gr.Slider(
@@ -487,6 +497,8 @@ if __name__ == "__main__":
                 audio_prompt,
                 text_prompt,
                 prompt_mode,
+                style_text,
+                style_weight,
             ],
             outputs=[text_output, audio_output],
         )
@@ -511,6 +523,8 @@ if __name__ == "__main__":
                 interval_between_sent,
                 audio_prompt,
                 text_prompt,
+                style_text,
+                style_weight,
             ],
             outputs=[text_output, audio_output],
         )
@@ -527,6 +541,12 @@ if __name__ == "__main__":
             outputs=[audio_prompt],
         )
 
+        formatter.click(
+            format_utils,
+            inputs=[text, speaker],
+            outputs=[language, text],
+        )
+
     print("推理页面已开启!")
     webbrowser.open(f"http://127.0.0.1:{config.webui_config.port}")
     app.launch(share=config.webui_config.share, server_port=config.webui_config.port)

webui_preprocess.py

@@ -19,9 +19,9 @@ def generate_config(data_dir, batch_size):
     assert data_dir != "", "数据集名称不能为空"
     start_path, _, train_path, val_path, config_path = get_path(data_dir)
     if os.path.isfile(config_path):
-        config = json.load(open(config_path))
+        config = json.load(open(config_path, "r", encoding="utf-8"))
     else:
-        config = json.load(open("configs/config.json"))
+        config = json.load(open("configs/config.json", "r", encoding="utf-8"))
     config["data"]["training_files"] = train_path
     config["data"]["validation_files"] = val_path
     config["train"]["batch_size"] = batch_size
@@ -44,7 +44,7 @@ def resample(data_dir):
     in_dir = os.path.join(start_path, "raw")
     out_dir = os.path.join(start_path, "wavs")
     subprocess.run(
-        f"python resample.py "
+        f"python resample_legacy.py "
         f"--sr 44100 "
         f"--in_dir {in_dir} "
         f"--out_dir {out_dir} ",
@@ -60,7 +60,9 @@ def preprocess_text(data_dir):
     with open(lbl_path, "w", encoding="utf-8") as f:
         for line in lines:
             path, spk, language, text = line.strip().split("|")
-            path = os.path.join(start_path, "wavs", os.path.basename(path))
+            path = os.path.join(start_path, "wavs", os.path.basename(path)).replace(
+                "\\", "/"
+            )
             f.writelines(f"{path}|{spk}|{language}|{text}\n")
     subprocess.run(
         f"python preprocess_text.py "
@@ -83,16 +85,6 @@ def bert_gen(data_dir):
     return "BERT 特征文件生成完成"
 
 
-def clap_gen(data_dir):
-    assert data_dir != "", "数据集名称不能为空"
-    _, _, _, _, config_path = get_path(data_dir)
-    subprocess.run(
-        f"python clap_gen.py " f"--config {config_path}",
-        shell=True,
-    )
-    return "CLAP 特征文件生成完成"
-
-
 if __name__ == "__main__":
     with gr.Blocks() as app:
         with gr.Row():
@@ -104,9 +96,9 @@ if __name__ == "__main__":
                     "- [中文 RoBERTa](https://huggingface.co/hfl/chinese-roberta-wwm-ext-large)\n"
                     "- [日文 DeBERTa](https://huggingface.co/ku-nlp/deberta-v2-large-japanese-char-wwm)\n"
                     "- [英文 DeBERTa](https://huggingface.co/microsoft/deberta-v3-large)\n"
-                    "- [CLAP](https://huggingface.co/laion/clap-htsat-fused)\n"
+                    "- [WavLM](https://huggingface.co/microsoft/wavlm-base-plus)\n"
                     "\n"
-                    "将 BERT 模型放置到 `bert` 文件夹下，CLAP 模型放置到 `emotional` 文件夹下，覆盖同名文件夹。\n"
+                    "将 BERT 模型放置到 `bert` 文件夹下，WavLM 模型放置到 `slm` 文件夹下，覆盖同名文件夹。\n"
                     "\n"
                     "数据准备：\n"
                     "将数据放置在 data 文件夹下，按照如下结构组织：\n"
@@ -156,12 +148,10 @@ if __name__ == "__main__":
                 preprocess_text_btn = gr.Button(value="执行", variant="primary")
                 _ = gr.Markdown(value="## 第四步：生成 BERT 特征文件")
                 bert_gen_btn = gr.Button(value="执行", variant="primary")
-                _ = gr.Markdown(value="## 第五步：生成 CLAP 特征文件")
-                clap_gen_btn = gr.Button(value="执行", variant="primary")
                 _ = gr.Markdown(
                     value="## 训练模型及部署：\n"
                     "修改根目录下的 `config.yml` 中 `dataset_path` 一项为 `data/{你的数据集名称}`\n"
-                    "- 训练：将[预训练模型文件](https://openi.pcl.ac.cn/Stardust_minus/Bert-VITS2/modelmanage/show_model)（`D_0.pth`、`DUR_0.pth` 和 `G_0.pth`）放到 `data/{你的数据集名称}/models` 文件夹下，执行 `torchrun --nproc_per_node=1 train_ms.py` 命令（多卡运行可参考 `run_MnodesAndMgpus.sh` 中的命令。\n"
+                    "- 训练：将[预训练模型文件](https://openi.pcl.ac.cn/Stardust_minus/Bert-VITS2/modelmanage/show_model)（`D_0.pth`、`DUR_0.pth`、`WD_0.pth` 和 `G_0.pth`）放到 `data/{你的数据集名称}/models` 文件夹下，执行 `torchrun --nproc_per_node=1 train_ms.py` 命令（多卡运行可参考 `run_MnodesAndMgpus.sh` 中的命令。\n"
                     "- 部署：修改根目录下的 `config.yml` 中 `webui` 下 `model` 一项为 `models/{权重文件名}.pth` （如 G_10000.pth），然后执行 `python webui.py`"
                 )
 
@@ -171,7 +161,6 @@ if __name__ == "__main__":
         resample_btn.click(resample, inputs=[data_dir], outputs=[info])
         preprocess_text_btn.click(preprocess_text, inputs=[data_dir], outputs=[info])
         bert_gen_btn.click(bert_gen, inputs=[data_dir], outputs=[info])
-        clap_gen_btn.click(clap_gen, inputs=[data_dir], outputs=[info])
 
     webbrowser.open("http://127.0.0.1:7860")
     app.launch(share=False, server_port=7860)