Dev 2.3. (#242)
* Fix inputs of duration discriminator * Add LSTM * Update models.py * Update tensorboard scalar * Noise injection for minimizing modality gap * Update infer.py * support bf16 run * del unused_para flag * support bf16 config * add grad clip * fix(logger and grad):add dur grad,fix grad clip * Update webui_preprocess.py * Fix English G2P * fix(bert_gen):add pass * Pass SDP to DD * Update webui_preprocess.py * Update config.json * Update webui.py * Update chinese_bert.py * Upload webui for deploy * Update webui.py * torch.save as pt not npy * Update config.json * add freeze emo vq * Update webui_preprocess.py * Fix tone_sandhi.py * Comment up grad clip * Fix in-place addition * Add SLM discriminator * Add DDP for WD * Feat: Style text: make emotions and style similar to the style text by mixing bert (#240) (#241) * fix:(oldVersion210) Load on demand Emotion model * feat: update fastapi.py. 添加更多错误日志信息 * Switch pyopenjtalk to pyopenjtalk-prebuilt * fix: update fastapi.py. 2.2 reference适配 * Update resample.py * 修复Onnx导出的BUG (#237) * Add files via upload * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add files via upload * Add files via upload * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Delete attentions_onnx.py * Delete models_onnx.py * Add files via upload * Add files via upload * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update __init__.py * Update __init__.py * Update __init__.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- * Fix onnx * Format export * Feat: style-text and bert mixing (JA only) * Ensure the same tensor shape * Update * update gradio version * Fix * Style text for chinese and english (ver 2.2) * Style text for chinese and english (ver 2.1) * Style text in FastAPI * Translate style text desc in chinese --------- Co-authored-by: litagin02 <139731664+litagin02@users.noreply.github.com> Co-authored-by: Sora <654163754@qq.com> Co-authored-by: Sihan Wang <wangsihan1995@gmail.com> Co-authored-by: Ναρουσέ·μ·γιουμεμί·Χινακάννα <40709280+NaruseMioShirakana@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Remove CLAP * Revert "Remove CLAP" This reverts commit 62fd59bc837c580239840a2bc84b15e0663730fc. Revert * Remove CLAP * bf16 audo grad cilp * Update webui and infer utils * Update webui.py * Update webui.py * Update webui-preprocess.py * Update webui_preprocess.py --------- Co-authored-by: Sihan Wang <wangsihan1995@gmail.com> Co-authored-by: OedoSoldier <31711261+OedoSoldier@users.noreply.github.com> Co-authored-by: litagin02 <139731664+litagin02@users.noreply.github.com> Co-authored-by: Sora <654163754@qq.com> Co-authored-by: Ναρουσέ·μ·γιουμεμί·Χινακάννα <40709280+NaruseMioShirakana@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Stardust·减
131
models.py
131
models.py
@@ -40,33 +40,22 @@ class DurationDiscriminator(nn.Module): # vits2
|
||||
self.norm_2 = modules.LayerNorm(filter_channels)
|
||||
self.dur_proj = nn.Conv1d(1, filter_channels, 1)
|
||||
|
||||
self.pre_out_conv_1 = nn.Conv1d(
|
||||
2 * filter_channels, filter_channels, kernel_size, padding=kernel_size // 2
|
||||
self.LSTM = nn.LSTM(
|
||||
2 * filter_channels, filter_channels, batch_first=True, bidirectional=True
|
||||
)
|
||||
self.pre_out_norm_1 = modules.LayerNorm(filter_channels)
|
||||
self.pre_out_conv_2 = nn.Conv1d(
|
||||
filter_channels, filter_channels, kernel_size, padding=kernel_size // 2
|
||||
)
|
||||
self.pre_out_norm_2 = modules.LayerNorm(filter_channels)
|
||||
|
||||
if gin_channels != 0:
|
||||
self.cond = nn.Conv1d(gin_channels, in_channels, 1)
|
||||
|
||||
self.output_layer = nn.Sequential(nn.Linear(filter_channels, 1), nn.Sigmoid())
|
||||
self.output_layer = nn.Sequential(
|
||||
nn.Linear(2 * filter_channels, 1), nn.Sigmoid()
|
||||
)
|
||||
|
||||
def forward_probability(self, x, x_mask, dur, g=None):
|
||||
def forward_probability(self, x, dur):
|
||||
dur = self.dur_proj(dur)
|
||||
x = torch.cat([x, dur], dim=1)
|
||||
x = self.pre_out_conv_1(x * x_mask)
|
||||
x = torch.relu(x)
|
||||
x = self.pre_out_norm_1(x)
|
||||
x = self.drop(x)
|
||||
x = self.pre_out_conv_2(x * x_mask)
|
||||
x = torch.relu(x)
|
||||
x = self.pre_out_norm_2(x)
|
||||
x = self.drop(x)
|
||||
x = x * x_mask
|
||||
x = x.transpose(1, 2)
|
||||
x, _ = self.LSTM(x)
|
||||
output_prob = self.output_layer(x)
|
||||
return output_prob
|
||||
|
||||
@@ -86,7 +75,7 @@ class DurationDiscriminator(nn.Module): # vits2
|
||||
|
||||
output_probs = []
|
||||
for dur in [dur_r, dur_hat]:
|
||||
output_prob = self.forward_probability(x, x_mask, dur, g)
|
||||
output_prob = self.forward_probability(x, dur)
|
||||
output_probs.append(output_prob)
|
||||
|
||||
return output_probs
|
||||
@@ -354,7 +343,6 @@ class TextEncoder(nn.Module):
|
||||
n_layers,
|
||||
kernel_size,
|
||||
p_dropout,
|
||||
n_speakers,
|
||||
gin_channels=0,
|
||||
):
|
||||
super().__init__()
|
||||
@@ -376,31 +364,6 @@ class TextEncoder(nn.Module):
|
||||
self.bert_proj = nn.Conv1d(1024, hidden_channels, 1)
|
||||
self.ja_bert_proj = nn.Conv1d(1024, hidden_channels, 1)
|
||||
self.en_bert_proj = nn.Conv1d(1024, hidden_channels, 1)
|
||||
# self.emo_proj = nn.Linear(512, hidden_channels)
|
||||
self.in_feature_net = nn.Sequential(
|
||||
# input is assumed to an already normalized embedding
|
||||
nn.Linear(512, 1028, bias=False),
|
||||
nn.GELU(),
|
||||
nn.LayerNorm(1028),
|
||||
*[Block(1028, 512) for _ in range(1)],
|
||||
nn.Linear(1028, 512, bias=False),
|
||||
# normalize before passing to VQ?
|
||||
# nn.GELU(),
|
||||
# nn.LayerNorm(512),
|
||||
)
|
||||
self.emo_vq = VectorQuantize(
|
||||
dim=512,
|
||||
codebook_size=64,
|
||||
codebook_dim=32,
|
||||
commitment_weight=0.1,
|
||||
decay=0.85,
|
||||
heads=32,
|
||||
kmeans_iters=20,
|
||||
separate_codebook_per_head=True,
|
||||
stochastic_sample_codes=True,
|
||||
threshold_ema_dead_code=2,
|
||||
)
|
||||
self.out_feature_net = nn.Linear(512, hidden_channels)
|
||||
|
||||
self.encoder = attentions.Encoder(
|
||||
hidden_channels,
|
||||
@@ -413,18 +376,10 @@ class TextEncoder(nn.Module):
|
||||
)
|
||||
self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
|
||||
|
||||
def forward(
|
||||
self, x, x_lengths, tone, language, bert, ja_bert, en_bert, emo, sid, g=None
|
||||
):
|
||||
sid = sid.cpu()
|
||||
def forward(self, x, x_lengths, tone, language, bert, ja_bert, en_bert, g=None):
|
||||
bert_emb = self.bert_proj(bert).transpose(1, 2)
|
||||
ja_bert_emb = self.ja_bert_proj(ja_bert).transpose(1, 2)
|
||||
en_bert_emb = self.en_bert_proj(en_bert).transpose(1, 2)
|
||||
emo_emb = self.in_feature_net(emo)
|
||||
emo_emb, _, loss_commit = self.emo_vq(emo_emb.unsqueeze(1))
|
||||
loss_commit = loss_commit.mean()
|
||||
emo_emb = self.out_feature_net(emo_emb)
|
||||
# emo_emb = self.emo_proj(emo.unsqueeze(1))
|
||||
x = (
|
||||
self.emb(x)
|
||||
+ self.tone_emb(tone)
|
||||
@@ -432,7 +387,6 @@ class TextEncoder(nn.Module):
|
||||
+ bert_emb
|
||||
+ ja_bert_emb
|
||||
+ en_bert_emb
|
||||
+ emo_emb
|
||||
) * math.sqrt(
|
||||
self.hidden_channels
|
||||
) # [b, t, h]
|
||||
@@ -445,7 +399,7 @@ class TextEncoder(nn.Module):
|
||||
stats = self.proj(x) * x_mask
|
||||
|
||||
m, logs = torch.split(stats, self.out_channels, dim=1)
|
||||
return x, m, logs, x_mask, loss_commit
|
||||
return x, m, logs, x_mask
|
||||
|
||||
|
||||
class ResidualCouplingBlock(nn.Module):
|
||||
@@ -748,6 +702,55 @@ class MultiPeriodDiscriminator(torch.nn.Module):
|
||||
return y_d_rs, y_d_gs, fmap_rs, fmap_gs
|
||||
|
||||
|
||||
class WavLMDiscriminator(nn.Module):
|
||||
"""docstring for Discriminator."""
|
||||
|
||||
def __init__(
|
||||
self, slm_hidden=768, slm_layers=13, initial_channel=64, use_spectral_norm=False
|
||||
):
|
||||
super(WavLMDiscriminator, self).__init__()
|
||||
norm_f = weight_norm if use_spectral_norm == False else spectral_norm
|
||||
self.pre = norm_f(
|
||||
Conv1d(slm_hidden * slm_layers, initial_channel, 1, 1, padding=0)
|
||||
)
|
||||
|
||||
self.convs = nn.ModuleList(
|
||||
[
|
||||
norm_f(
|
||||
nn.Conv1d(
|
||||
initial_channel, initial_channel * 2, kernel_size=5, padding=2
|
||||
)
|
||||
),
|
||||
norm_f(
|
||||
nn.Conv1d(
|
||||
initial_channel * 2,
|
||||
initial_channel * 4,
|
||||
kernel_size=5,
|
||||
padding=2,
|
||||
)
|
||||
),
|
||||
norm_f(
|
||||
nn.Conv1d(initial_channel * 4, initial_channel * 4, 5, 1, padding=2)
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
self.conv_post = norm_f(Conv1d(initial_channel * 4, 1, 3, 1, padding=1))
|
||||
|
||||
def forward(self, x):
|
||||
x = self.pre(x)
|
||||
|
||||
fmap = []
|
||||
for l in self.convs:
|
||||
x = l(x)
|
||||
x = F.leaky_relu(x, modules.LRELU_SLOPE)
|
||||
fmap.append(x)
|
||||
x = self.conv_post(x)
|
||||
x = torch.flatten(x, 1, -1)
|
||||
|
||||
return x
|
||||
|
||||
|
||||
class ReferenceEncoder(nn.Module):
|
||||
"""
|
||||
inputs --- [N, Ty/r, n_mels*r] mels
|
||||
@@ -878,7 +881,6 @@ class SynthesizerTrn(nn.Module):
|
||||
n_layers,
|
||||
kernel_size,
|
||||
p_dropout,
|
||||
self.n_speakers,
|
||||
gin_channels=self.enc_gin_channels,
|
||||
)
|
||||
self.dec = Generator(
|
||||
@@ -946,14 +948,13 @@ class SynthesizerTrn(nn.Module):
|
||||
bert,
|
||||
ja_bert,
|
||||
en_bert,
|
||||
emo=None,
|
||||
):
|
||||
if self.n_speakers > 0:
|
||||
g = self.emb_g(sid).unsqueeze(-1) # [b, h, 1]
|
||||
else:
|
||||
g = self.ref_enc(y.transpose(1, 2)).unsqueeze(-1)
|
||||
x, m_p, logs_p, x_mask, loss_commit = self.enc_p(
|
||||
x, x_lengths, tone, language, bert, ja_bert, en_bert, emo, sid, g=g
|
||||
x, m_p, logs_p, x_mask = self.enc_p(
|
||||
x, x_lengths, tone, language, bert, ja_bert, en_bert, g=g
|
||||
)
|
||||
z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g)
|
||||
z_p = self.flow(z, y_mask, g=g)
|
||||
@@ -996,9 +997,11 @@ class SynthesizerTrn(nn.Module):
|
||||
|
||||
logw_ = torch.log(w + 1e-6) * x_mask
|
||||
logw = self.dp(x, x_mask, g=g)
|
||||
logw_sdp = self.sdp(x, x_mask, g=g, reverse=True, noise_scale=1.0)
|
||||
l_length_dp = torch.sum((logw - logw_) ** 2, [1, 2]) / torch.sum(
|
||||
x_mask
|
||||
) # for averaging
|
||||
l_length_sdp += torch.sum((logw_sdp - logw_) ** 2, [1, 2]) / torch.sum(x_mask)
|
||||
|
||||
l_length = l_length_dp + l_length_sdp
|
||||
|
||||
@@ -1018,9 +1021,8 @@ class SynthesizerTrn(nn.Module):
|
||||
x_mask,
|
||||
y_mask,
|
||||
(z, z_p, m_p, logs_p, m_q, logs_q),
|
||||
(x, logw, logw_),
|
||||
(x, logw, logw_, logw_sdp),
|
||||
g,
|
||||
loss_commit,
|
||||
)
|
||||
|
||||
def infer(
|
||||
@@ -1033,7 +1035,6 @@ class SynthesizerTrn(nn.Module):
|
||||
bert,
|
||||
ja_bert,
|
||||
en_bert,
|
||||
emo=None,
|
||||
noise_scale=0.667,
|
||||
length_scale=1,
|
||||
noise_scale_w=0.8,
|
||||
@@ -1047,8 +1048,8 @@ class SynthesizerTrn(nn.Module):
|
||||
g = self.emb_g(sid).unsqueeze(-1) # [b, h, 1]
|
||||
else:
|
||||
g = self.ref_enc(y.transpose(1, 2)).unsqueeze(-1)
|
||||
x, m_p, logs_p, x_mask, _ = self.enc_p(
|
||||
x, x_lengths, tone, language, bert, ja_bert, en_bert, emo, sid, g=g
|
||||
x, m_p, logs_p, x_mask = self.enc_p(
|
||||
x, x_lengths, tone, language, bert, ja_bert, en_bert, g=g
|
||||
)
|
||||
logw = self.sdp(x, x_mask, g=g, reverse=True, noise_scale=noise_scale_w) * (
|
||||
sdp_ratio
|
||||
|
||||
Reference in New Issue
Block a user