Dev 2.3. (#242)
* Fix inputs of duration discriminator * Add LSTM * Update models.py * Update tensorboard scalar * Noise injection for minimizing modality gap * Update infer.py * support bf16 run * del unused_para flag * support bf16 config * add grad clip * fix(logger and grad):add dur grad,fix grad clip * Update webui_preprocess.py * Fix English G2P * fix(bert_gen):add pass * Pass SDP to DD * Update webui_preprocess.py * Update config.json * Update webui.py * Update chinese_bert.py * Upload webui for deploy * Update webui.py * torch.save as pt not npy * Update config.json * add freeze emo vq * Update webui_preprocess.py * Fix tone_sandhi.py * Comment up grad clip * Fix in-place addition * Add SLM discriminator * Add DDP for WD * Feat: Style text: make emotions and style similar to the style text by mixing bert (#240) (#241) * fix:(oldVersion210) Load on demand Emotion model * feat: update fastapi.py. 添加更多错误日志信息 * Switch pyopenjtalk to pyopenjtalk-prebuilt * fix: update fastapi.py. 2.2 reference适配 * Update resample.py * 修复Onnx导出的BUG (#237) * Add files via upload * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add files via upload * Add files via upload * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Delete attentions_onnx.py * Delete models_onnx.py * Add files via upload * Add files via upload * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update __init__.py * Update __init__.py * Update __init__.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- * Fix onnx * Format export * Feat: style-text and bert mixing (JA only) * Ensure the same tensor shape * Update * update gradio version * Fix * Style text for chinese and english (ver 2.2) * Style text for chinese and english (ver 2.1) * Style text in FastAPI * Translate style text desc in chinese --------- Co-authored-by: litagin02 <139731664+litagin02@users.noreply.github.com> Co-authored-by: Sora <654163754@qq.com> Co-authored-by: Sihan Wang <wangsihan1995@gmail.com> Co-authored-by: Ναρουσέ·μ·γιουμεμί·Χινακάννα <40709280+NaruseMioShirakana@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Remove CLAP * Revert "Remove CLAP" This reverts commit 62fd59bc837c580239840a2bc84b15e0663730fc. Revert * Remove CLAP * bf16 audo grad cilp * Update webui and infer utils * Update webui.py * Update webui.py * Update webui-preprocess.py * Update webui_preprocess.py --------- Co-authored-by: Sihan Wang <wangsihan1995@gmail.com> Co-authored-by: OedoSoldier <31711261+OedoSoldier@users.noreply.github.com> Co-authored-by: litagin02 <139731664+litagin02@users.noreply.github.com> Co-authored-by: Sora <654163754@qq.com> Co-authored-by: Ναρουσέ·μ·γιουμεμί·Χινακάννα <40709280+NaruseMioShirakana@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Stardust·减
92
losses.py
92
losses.py
@@ -1,4 +1,6 @@
|
||||
import torch
|
||||
import torchaudio
|
||||
from transformers import AutoModel
|
||||
|
||||
|
||||
def feature_loss(fmap_r, fmap_g):
|
||||
@@ -56,3 +58,93 @@ def kl_loss(z_p, logs_q, m_p, logs_p, z_mask):
|
||||
kl = torch.sum(kl * z_mask)
|
||||
l = kl / torch.sum(z_mask)
|
||||
return l
|
||||
|
||||
|
||||
class WavLMLoss(torch.nn.Module):
|
||||
def __init__(self, model, wd, model_sr, slm_sr=16000):
|
||||
super(WavLMLoss, self).__init__()
|
||||
self.wavlm = AutoModel.from_pretrained(model)
|
||||
self.wd = wd
|
||||
self.resample = torchaudio.transforms.Resample(model_sr, slm_sr)
|
||||
|
||||
def forward(self, wav, y_rec):
|
||||
with torch.no_grad():
|
||||
wav_16 = self.resample(wav)
|
||||
wav_embeddings = self.wavlm(
|
||||
input_values=wav_16, output_hidden_states=True
|
||||
).hidden_states
|
||||
y_rec_16 = self.resample(y_rec)
|
||||
y_rec_embeddings = self.wavlm(
|
||||
input_values=y_rec_16.squeeze(), output_hidden_states=True
|
||||
).hidden_states
|
||||
|
||||
floss = 0
|
||||
for er, eg in zip(wav_embeddings, y_rec_embeddings):
|
||||
floss += torch.mean(torch.abs(er - eg))
|
||||
|
||||
return floss.mean()
|
||||
|
||||
def generator(self, y_rec):
|
||||
y_rec_16 = self.resample(y_rec)
|
||||
y_rec_embeddings = self.wavlm(
|
||||
input_values=y_rec_16, output_hidden_states=True
|
||||
).hidden_states
|
||||
y_rec_embeddings = (
|
||||
torch.stack(y_rec_embeddings, dim=1)
|
||||
.transpose(-1, -2)
|
||||
.flatten(start_dim=1, end_dim=2)
|
||||
)
|
||||
y_df_hat_g = self.wd(y_rec_embeddings)
|
||||
loss_gen = torch.mean((1 - y_df_hat_g) ** 2)
|
||||
|
||||
return loss_gen
|
||||
|
||||
def discriminator(self, wav, y_rec):
|
||||
with torch.no_grad():
|
||||
wav_16 = self.resample(wav)
|
||||
wav_embeddings = self.wavlm(
|
||||
input_values=wav_16, output_hidden_states=True
|
||||
).hidden_states
|
||||
y_rec_16 = self.resample(y_rec)
|
||||
y_rec_embeddings = self.wavlm(
|
||||
input_values=y_rec_16, output_hidden_states=True
|
||||
).hidden_states
|
||||
|
||||
y_embeddings = (
|
||||
torch.stack(wav_embeddings, dim=1)
|
||||
.transpose(-1, -2)
|
||||
.flatten(start_dim=1, end_dim=2)
|
||||
)
|
||||
y_rec_embeddings = (
|
||||
torch.stack(y_rec_embeddings, dim=1)
|
||||
.transpose(-1, -2)
|
||||
.flatten(start_dim=1, end_dim=2)
|
||||
)
|
||||
|
||||
y_d_rs = self.wd(y_embeddings)
|
||||
y_d_gs = self.wd(y_rec_embeddings)
|
||||
|
||||
y_df_hat_r, y_df_hat_g = y_d_rs, y_d_gs
|
||||
|
||||
r_loss = torch.mean((1 - y_df_hat_r) ** 2)
|
||||
g_loss = torch.mean((y_df_hat_g) ** 2)
|
||||
|
||||
loss_disc_f = r_loss + g_loss
|
||||
|
||||
return loss_disc_f.mean()
|
||||
|
||||
def discriminator_forward(self, wav):
|
||||
with torch.no_grad():
|
||||
wav_16 = self.resample(wav)
|
||||
wav_embeddings = self.wavlm(
|
||||
input_values=wav_16, output_hidden_states=True
|
||||
).hidden_states
|
||||
y_embeddings = (
|
||||
torch.stack(wav_embeddings, dim=1)
|
||||
.transpose(-1, -2)
|
||||
.flatten(start_dim=1, end_dim=2)
|
||||
)
|
||||
|
||||
y_d_rs = self.wd(y_embeddings)
|
||||
|
||||
return y_d_rs
|
||||
|
||||
Reference in New Issue
Block a user