tuna2134/sbv2-v2
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Dev 2.3. (#242)

Browse Source 
* Fix inputs of duration discriminator

* Add LSTM

* Update models.py

* Update tensorboard scalar

* Noise injection for minimizing modality gap

* Update infer.py

* support bf16 run

* del unused_para flag

* support bf16 config

* add grad clip

* fix(logger and grad):add dur grad,fix grad clip

* Update webui_preprocess.py

* Fix English G2P

* fix(bert_gen):add pass

* Pass SDP to DD

* Update webui_preprocess.py

* Update config.json

* Update webui.py

* Update chinese_bert.py

* Upload webui for deploy

* Update webui.py

* torch.save as pt not npy

* Update config.json

* add freeze emo vq

* Update webui_preprocess.py

* Fix tone_sandhi.py

* Comment up grad clip

* Fix in-place addition

* Add SLM discriminator

* Add DDP for WD

* Feat: Style text: make emotions and style similar to the style text by mixing bert (#240) (#241)

* fix:(oldVersion210) Load on demand Emotion model

* feat: update fastapi.py. 添加更多错误日志信息

* Switch pyopenjtalk to pyopenjtalk-prebuilt

* fix: update fastapi.py. 2.2 reference适配

* Update resample.py

* 修复Onnx导出的BUG (#237)

* Add files via upload

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Add files via upload

* Add files via upload

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Delete attentions_onnx.py

* Delete models_onnx.py

* Add files via upload

* Add files via upload

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Update __init__.py

* Update __init__.py

* Update __init__.py

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------



* Fix onnx

* Format export

* Feat: style-text and bert mixing (JA only)

* Ensure the same tensor shape

* Update

* update gradio version

* Fix

* Style text for chinese and english (ver 2.2)

* Style text for chinese and english (ver 2.1)

* Style text in FastAPI

* Translate style text desc in chinese

---------

Co-authored-by: litagin02 <139731664+litagin02@users.noreply.github.com>
Co-authored-by: Sora <654163754@qq.com>
Co-authored-by: Sihan Wang <wangsihan1995@gmail.com>
Co-authored-by: Ναρουσέ·μ·γιουμεμί·Χινακάννα <40709280+NaruseMioShirakana@users.noreply.github.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

* Remove CLAP

* Revert "Remove CLAP"

This reverts commit 62fd59bc837c580239840a2bc84b15e0663730fc.

Revert

* Remove CLAP

* bf16 audo grad cilp

* Update webui and infer utils

* Update webui.py

* Update webui.py

* Update webui-preprocess.py

* Update webui_preprocess.py

---------

Co-authored-by: Sihan Wang <wangsihan1995@gmail.com>
Co-authored-by: OedoSoldier <31711261+OedoSoldier@users.noreply.github.com>
Co-authored-by: litagin02 <139731664+litagin02@users.noreply.github.com>
Co-authored-by: Sora <654163754@qq.com>
Co-authored-by: Ναρουσέ·μ·γιουμεμί·Χινακάννα <40709280+NaruseMioShirakana@users.noreply.github.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Stardust·减
2023-12-19 19:12:26 +08:00
committed by GitHub
parent 5479e9039d
commit 76653b5b6d
34 changed files with 2978 additions and 1226 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
oldVersion/V210/__init__.py
View File

@@ -7,7 +7,7 @@ from .text import cleaned_text_to_sequence, get_bert
from .text.cleaner import clean_text
def get_text(text, language_str, hps, device):
def get_text(text, language_str, hps, device, style_text=None, style_weight=0.7):
# 在此处实现当前版本的get_text
norm_text, phone, tone, word2ph = clean_text(text, language_str)
phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
@@ -19,7 +19,9 @@ def get_text(text, language_str, hps, device):
for i in range(len(word2ph)):
word2ph[i] = word2ph[i] * 2
word2ph[0] += 1
bert_ori = get_bert(norm_text, word2ph, language_str, device)
bert_ori = get_bert(
norm_text, word2ph, language_str, device, style_text, style_weight
)
del word2ph
assert bert_ori.shape[-1] == len(phone), phone
@@ -74,9 +76,11 @@ def infer(
emotion=None,
skip_start=False,
skip_end=False,
style_text=None,
style_weight=0.7,
):
bert, ja_bert, en_bert, phones, tones, lang_ids = get_text(
text, language, hps, device
text, language, hps, device, style_text, style_weight
)
emo = get_emo_(reference_audio, emotion)
if skip_start:

6
oldVersion/V210/text/__init__.py
View File

@@ -18,13 +18,15 @@ def cleaned_text_to_sequence(cleaned_text, tones, language):
return phones, tones, lang_ids
def get_bert(norm_text, word2ph, language, device):
def get_bert(norm_text, word2ph, language, device, style_text, style_weight):
from .chinese_bert import get_bert_feature as zh_bert
from .english_bert_mock import get_bert_feature as en_bert
from .japanese_bert import get_bert_feature as jp_bert
lang_bert_func_map = {"ZH": zh_bert, "EN": en_bert, "JP": jp_bert}
bert = lang_bert_func_map[language](norm_text, word2ph, device)
bert = lang_bert_func_map[language](
norm_text, word2ph, device, style_text, style_weight
)
return bert

23
oldVersion/V210/text/chinese_bert.py
View File

@@ -12,7 +12,13 @@ tokenizer = AutoTokenizer.from_pretrained(LOCAL_PATH)
models = dict()
def get_bert_feature(text, word2ph, device=config.bert_gen_config.device):
def get_bert_feature(
text,
word2ph,
device=config.bert_gen_config.device,
style_text=None,
style_weight=0.7,
):
if (
sys.platform == "darwin"
and torch.backends.mps.is_available()
@@ -29,12 +35,25 @@ def get_bert_feature(text, word2ph, device=config.bert_gen_config.device):
inputs[i] = inputs[i].to(device)
res = models[device](**inputs, output_hidden_states=True)
res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
if style_text:
style_inputs = tokenizer(style_text, return_tensors="pt")
for i in style_inputs:
style_inputs[i] = style_inputs[i].to(device)
style_res = models[device](**style_inputs, output_hidden_states=True)
style_res = torch.cat(style_res["hidden_states"][-3:-2], -1)[0].cpu()
style_res_mean = style_res.mean(0)
assert len(word2ph) == len(text) + 2
word2phone = word2ph
phone_level_feature = []
for i in range(len(word2phone)):
repeat_feature = res[i].repeat(word2phone[i], 1)
if style_text:
repeat_feature = (
res[i].repeat(word2phone[i], 1) * (1 - style_weight)
+ style_res_mean.repeat(word2phone[i], 1) * style_weight
)
else:
repeat_feature = res[i].repeat(word2phone[i], 1)
phone_level_feature.append(repeat_feature)
phone_level_feature = torch.cat(phone_level_feature, dim=0)

23
oldVersion/V210/text/english_bert_mock.py
View File

@@ -13,7 +13,13 @@ tokenizer = DebertaV2Tokenizer.from_pretrained(LOCAL_PATH)
models = dict()
def get_bert_feature(text, word2ph, device=config.bert_gen_config.device):
def get_bert_feature(
text,
word2ph,
device=config.bert_gen_config.device,
style_text=None,
style_weight=0.7,
):
if (
sys.platform == "darwin"
and torch.backends.mps.is_available()
@@ -30,11 +36,24 @@ def get_bert_feature(text, word2ph, device=config.bert_gen_config.device):
inputs[i] = inputs[i].to(device)
res = models[device](**inputs, output_hidden_states=True)
res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
if style_text:
style_inputs = tokenizer(style_text, return_tensors="pt")
for i in style_inputs:
style_inputs[i] = style_inputs[i].to(device)
style_res = models[device](**style_inputs, output_hidden_states=True)
style_res = torch.cat(style_res["hidden_states"][-3:-2], -1)[0].cpu()
style_res_mean = style_res.mean(0)
assert len(word2ph) == res.shape[0], (text, res.shape[0], len(word2ph))
word2phone = word2ph
phone_level_feature = []
for i in range(len(word2phone)):
repeat_feature = res[i].repeat(word2phone[i], 1)
if style_text:
repeat_feature = (
res[i].repeat(word2phone[i], 1) * (1 - style_weight)
+ style_res_mean.repeat(word2phone[i], 1) * style_weight
)
else:
repeat_feature = res[i].repeat(word2phone[i], 1)
phone_level_feature.append(repeat_feature)
phone_level_feature = torch.cat(phone_level_feature, dim=0)

25
oldVersion/V210/text/japanese_bert.py
View File

@@ -13,8 +13,16 @@ tokenizer = AutoTokenizer.from_pretrained(LOCAL_PATH)
models = dict()
def get_bert_feature(text, word2ph, device=config.bert_gen_config.device):
def get_bert_feature(
text,
word2ph,
device=config.bert_gen_config.device,
style_text=None,
style_weight=0.7,
):
text = "".join(text2sep_kata(text)[0])
if style_text:
style_text = "".join(text2sep_kata(style_text)[0])
if (
sys.platform == "darwin"
and torch.backends.mps.is_available()
@@ -31,12 +39,25 @@ def get_bert_feature(text, word2ph, device=config.bert_gen_config.device):
inputs[i] = inputs[i].to(device)
res = models[device](**inputs, output_hidden_states=True)
res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
if style_text:
style_inputs = tokenizer(style_text, return_tensors="pt")
for i in style_inputs:
style_inputs[i] = style_inputs[i].to(device)
style_res = models[device](**style_inputs, output_hidden_states=True)
style_res = torch.cat(style_res["hidden_states"][-3:-2], -1)[0].cpu()
style_res_mean = style_res.mean(0)
assert len(word2ph) == len(text) + 2
word2phone = word2ph
phone_level_feature = []
for i in range(len(word2phone)):
repeat_feature = res[i].repeat(word2phone[i], 1)
if style_text:
repeat_feature = (
res[i].repeat(word2phone[i], 1) * (1 - style_weight)
+ style_res_mean.repeat(word2phone[i], 1) * style_weight
)
else:
repeat_feature = res[i].repeat(word2phone[i], 1)
phone_level_feature.append(repeat_feature)
phone_level_feature = torch.cat(phone_level_feature, dim=0)