tuna2134/sbv2-v2
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Format code

Browse Source
This commit is contained in:
github-actions[bot]
2023-09-06 13:42:08 +00:00
parent d82ba3457a
commit 92f3fdd11f
26 changed files with 2938 additions and 1453 deletions
Download Patch File Download Diff File Expand all files Collapse all files

707
attentions.py
View File

@@ -9,336 +9,457 @@ import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class LayerNorm(nn.Module): class LayerNorm(nn.Module):
def __init__(self, channels, eps=1e-5): def __init__(self, channels, eps=1e-5):
super().__init__() super().__init__()
self.channels = channels self.channels = channels
self.eps = eps self.eps = eps
self.gamma = nn.Parameter(torch.ones(channels)) self.gamma = nn.Parameter(torch.ones(channels))
self.beta = nn.Parameter(torch.zeros(channels)) self.beta = nn.Parameter(torch.zeros(channels))
def forward(self, x):
x = x.transpose(1, -1)
x = F.layer_norm(x, (self.channels,), self.gamma, self.beta, self.eps)
return x.transpose(1, -1)
def forward(self, x):
x = x.transpose(1, -1)
x = F.layer_norm(x, (self.channels,), self.gamma, self.beta, self.eps)
return x.transpose(1, -1)
@torch.jit.script @torch.jit.script
def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels): def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
n_channels_int = n_channels[0] n_channels_int = n_channels[0]
in_act = input_a + input_b in_act = input_a + input_b
t_act = torch.tanh(in_act[:, :n_channels_int, :]) t_act = torch.tanh(in_act[:, :n_channels_int, :])
s_act = torch.sigmoid(in_act[:, n_channels_int:, :]) s_act = torch.sigmoid(in_act[:, n_channels_int:, :])
acts = t_act * s_act acts = t_act * s_act
return acts return acts
class Encoder(nn.Module): class Encoder(nn.Module):
def __init__(self, hidden_channels, filter_channels, n_heads, n_layers, kernel_size=1, p_dropout=0., window_size=4, isflow = True, **kwargs): def __init__(
super().__init__() self,
self.hidden_channels = hidden_channels hidden_channels,
self.filter_channels = filter_channels filter_channels,
self.n_heads = n_heads n_heads,
self.n_layers = n_layers n_layers,
self.kernel_size = kernel_size kernel_size=1,
self.p_dropout = p_dropout p_dropout=0.0,
self.window_size = window_size window_size=4,
#if isflow: isflow=True,
# cond_layer = torch.nn.Conv1d(256, 2*hidden_channels*n_layers, 1) **kwargs
# self.cond_pre = torch.nn.Conv1d(hidden_channels, 2*hidden_channels, 1) ):
# self.cond_layer = weight_norm(cond_layer, name='weight') super().__init__()
# self.gin_channels = 256 self.hidden_channels = hidden_channels
self.cond_layer_idx = self.n_layers self.filter_channels = filter_channels
if 'gin_channels' in kwargs: self.n_heads = n_heads
self.gin_channels = kwargs['gin_channels'] self.n_layers = n_layers
if self.gin_channels != 0: self.kernel_size = kernel_size
self.spk_emb_linear = nn.Linear(self.gin_channels, self.hidden_channels) self.p_dropout = p_dropout
# vits2 says 3rd block, so idx is 2 by default self.window_size = window_size
self.cond_layer_idx = kwargs['cond_layer_idx'] if 'cond_layer_idx' in kwargs else 2 # if isflow:
logging.debug(self.gin_channels, self.cond_layer_idx) # cond_layer = torch.nn.Conv1d(256, 2*hidden_channels*n_layers, 1)
assert self.cond_layer_idx < self.n_layers, 'cond_layer_idx should be less than n_layers' # self.cond_pre = torch.nn.Conv1d(hidden_channels, 2*hidden_channels, 1)
self.drop = nn.Dropout(p_dropout) # self.cond_layer = weight_norm(cond_layer, name='weight')
self.attn_layers = nn.ModuleList() # self.gin_channels = 256
self.norm_layers_1 = nn.ModuleList() self.cond_layer_idx = self.n_layers
self.ffn_layers = nn.ModuleList() if "gin_channels" in kwargs:
self.norm_layers_2 = nn.ModuleList() self.gin_channels = kwargs["gin_channels"]
for i in range(self.n_layers): if self.gin_channels != 0:
self.attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout, window_size=window_size)) self.spk_emb_linear = nn.Linear(self.gin_channels, self.hidden_channels)
self.norm_layers_1.append(LayerNorm(hidden_channels)) # vits2 says 3rd block, so idx is 2 by default
self.ffn_layers.append(FFN(hidden_channels, hidden_channels, filter_channels, kernel_size, p_dropout=p_dropout)) self.cond_layer_idx = (
self.norm_layers_2.append(LayerNorm(hidden_channels)) kwargs["cond_layer_idx"] if "cond_layer_idx" in kwargs else 2
def forward(self, x, x_mask, g=None): )
attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1) logging.debug(self.gin_channels, self.cond_layer_idx)
x = x * x_mask assert (
for i in range(self.n_layers): self.cond_layer_idx < self.n_layers
if i == self.cond_layer_idx and g is not None: ), "cond_layer_idx should be less than n_layers"
g = self.spk_emb_linear(g.transpose(1, 2)) self.drop = nn.Dropout(p_dropout)
g = g.transpose(1, 2) self.attn_layers = nn.ModuleList()
x = x + g self.norm_layers_1 = nn.ModuleList()
x = x * x_mask self.ffn_layers = nn.ModuleList()
y = self.attn_layers[i](x, x, attn_mask) self.norm_layers_2 = nn.ModuleList()
y = self.drop(y) for i in range(self.n_layers):
x = self.norm_layers_1[i](x + y) self.attn_layers.append(
MultiHeadAttention(
hidden_channels,
hidden_channels,
n_heads,
p_dropout=p_dropout,
window_size=window_size,
)
)
self.norm_layers_1.append(LayerNorm(hidden_channels))
self.ffn_layers.append(
FFN(
hidden_channels,
hidden_channels,
filter_channels,
kernel_size,
p_dropout=p_dropout,
)
)
self.norm_layers_2.append(LayerNorm(hidden_channels))
y = self.ffn_layers[i](x, x_mask) def forward(self, x, x_mask, g=None):
y = self.drop(y) attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
x = self.norm_layers_2[i](x + y) x = x * x_mask
x = x * x_mask for i in range(self.n_layers):
return x if i == self.cond_layer_idx and g is not None:
g = self.spk_emb_linear(g.transpose(1, 2))
g = g.transpose(1, 2)
x = x + g
x = x * x_mask
y = self.attn_layers[i](x, x, attn_mask)
y = self.drop(y)
x = self.norm_layers_1[i](x + y)
y = self.ffn_layers[i](x, x_mask)
y = self.drop(y)
x = self.norm_layers_2[i](x + y)
x = x * x_mask
return x
class Decoder(nn.Module): class Decoder(nn.Module):
def __init__(self, hidden_channels, filter_channels, n_heads, n_layers, kernel_size=1, p_dropout=0., proximal_bias=False, proximal_init=True, **kwargs): def __init__(
super().__init__() self,
self.hidden_channels = hidden_channels hidden_channels,
self.filter_channels = filter_channels filter_channels,
self.n_heads = n_heads n_heads,
self.n_layers = n_layers n_layers,
self.kernel_size = kernel_size kernel_size=1,
self.p_dropout = p_dropout p_dropout=0.0,
self.proximal_bias = proximal_bias proximal_bias=False,
self.proximal_init = proximal_init proximal_init=True,
**kwargs
):
super().__init__()
self.hidden_channels = hidden_channels
self.filter_channels = filter_channels
self.n_heads = n_heads
self.n_layers = n_layers
self.kernel_size = kernel_size
self.p_dropout = p_dropout
self.proximal_bias = proximal_bias
self.proximal_init = proximal_init
self.drop = nn.Dropout(p_dropout) self.drop = nn.Dropout(p_dropout)
self.self_attn_layers = nn.ModuleList() self.self_attn_layers = nn.ModuleList()
self.norm_layers_0 = nn.ModuleList() self.norm_layers_0 = nn.ModuleList()
self.encdec_attn_layers = nn.ModuleList() self.encdec_attn_layers = nn.ModuleList()
self.norm_layers_1 = nn.ModuleList() self.norm_layers_1 = nn.ModuleList()
self.ffn_layers = nn.ModuleList() self.ffn_layers = nn.ModuleList()
self.norm_layers_2 = nn.ModuleList() self.norm_layers_2 = nn.ModuleList()
for i in range(self.n_layers): for i in range(self.n_layers):
self.self_attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout, proximal_bias=proximal_bias, proximal_init=proximal_init)) self.self_attn_layers.append(
self.norm_layers_0.append(LayerNorm(hidden_channels)) MultiHeadAttention(
self.encdec_attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout)) hidden_channels,
self.norm_layers_1.append(LayerNorm(hidden_channels)) hidden_channels,
self.ffn_layers.append(FFN(hidden_channels, hidden_channels, filter_channels, kernel_size, p_dropout=p_dropout, causal=True)) n_heads,
self.norm_layers_2.append(LayerNorm(hidden_channels)) p_dropout=p_dropout,
proximal_bias=proximal_bias,
proximal_init=proximal_init,
)
)
self.norm_layers_0.append(LayerNorm(hidden_channels))
self.encdec_attn_layers.append(
MultiHeadAttention(
hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout
)
)
self.norm_layers_1.append(LayerNorm(hidden_channels))
self.ffn_layers.append(
FFN(
hidden_channels,
hidden_channels,
filter_channels,
kernel_size,
p_dropout=p_dropout,
causal=True,
)
)
self.norm_layers_2.append(LayerNorm(hidden_channels))
def forward(self, x, x_mask, h, h_mask): def forward(self, x, x_mask, h, h_mask):
""" """
x: decoder input x: decoder input
h: encoder output h: encoder output
""" """
self_attn_mask = commons.subsequent_mask(x_mask.size(2)).to(device=x.device, dtype=x.dtype) self_attn_mask = commons.subsequent_mask(x_mask.size(2)).to(
encdec_attn_mask = h_mask.unsqueeze(2) * x_mask.unsqueeze(-1) device=x.device, dtype=x.dtype
x = x * x_mask )
for i in range(self.n_layers): encdec_attn_mask = h_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
y = self.self_attn_layers[i](x, x, self_attn_mask) x = x * x_mask
y = self.drop(y) for i in range(self.n_layers):
x = self.norm_layers_0[i](x + y) y = self.self_attn_layers[i](x, x, self_attn_mask)
y = self.drop(y)
x = self.norm_layers_0[i](x + y)
y = self.encdec_attn_layers[i](x, h, encdec_attn_mask) y = self.encdec_attn_layers[i](x, h, encdec_attn_mask)
y = self.drop(y) y = self.drop(y)
x = self.norm_layers_1[i](x + y) x = self.norm_layers_1[i](x + y)
y = self.ffn_layers[i](x, x_mask) y = self.ffn_layers[i](x, x_mask)
y = self.drop(y) y = self.drop(y)
x = self.norm_layers_2[i](x + y) x = self.norm_layers_2[i](x + y)
x = x * x_mask x = x * x_mask
return x return x
class MultiHeadAttention(nn.Module): class MultiHeadAttention(nn.Module):
def __init__(self, channels, out_channels, n_heads, p_dropout=0., window_size=None, heads_share=True, block_length=None, proximal_bias=False, proximal_init=False): def __init__(
super().__init__() self,
assert channels % n_heads == 0 channels,
out_channels,
n_heads,
p_dropout=0.0,
window_size=None,
heads_share=True,
block_length=None,
proximal_bias=False,
proximal_init=False,
):
super().__init__()
assert channels % n_heads == 0
self.channels = channels self.channels = channels
self.out_channels = out_channels self.out_channels = out_channels
self.n_heads = n_heads self.n_heads = n_heads
self.p_dropout = p_dropout self.p_dropout = p_dropout
self.window_size = window_size self.window_size = window_size
self.heads_share = heads_share self.heads_share = heads_share
self.block_length = block_length self.block_length = block_length
self.proximal_bias = proximal_bias self.proximal_bias = proximal_bias
self.proximal_init = proximal_init self.proximal_init = proximal_init
self.attn = None self.attn = None
self.k_channels = channels // n_heads self.k_channels = channels // n_heads
self.conv_q = nn.Conv1d(channels, channels, 1) self.conv_q = nn.Conv1d(channels, channels, 1)
self.conv_k = nn.Conv1d(channels, channels, 1) self.conv_k = nn.Conv1d(channels, channels, 1)
self.conv_v = nn.Conv1d(channels, channels, 1) self.conv_v = nn.Conv1d(channels, channels, 1)
self.conv_o = nn.Conv1d(channels, out_channels, 1) self.conv_o = nn.Conv1d(channels, out_channels, 1)
self.drop = nn.Dropout(p_dropout) self.drop = nn.Dropout(p_dropout)
if window_size is not None: if window_size is not None:
n_heads_rel = 1 if heads_share else n_heads n_heads_rel = 1 if heads_share else n_heads
rel_stddev = self.k_channels**-0.5 rel_stddev = self.k_channels**-0.5
self.emb_rel_k = nn.Parameter(torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) * rel_stddev) self.emb_rel_k = nn.Parameter(
self.emb_rel_v = nn.Parameter(torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) * rel_stddev) torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels)
* rel_stddev
)
self.emb_rel_v = nn.Parameter(
torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels)
* rel_stddev
)
nn.init.xavier_uniform_(self.conv_q.weight) nn.init.xavier_uniform_(self.conv_q.weight)
nn.init.xavier_uniform_(self.conv_k.weight) nn.init.xavier_uniform_(self.conv_k.weight)
nn.init.xavier_uniform_(self.conv_v.weight) nn.init.xavier_uniform_(self.conv_v.weight)
if proximal_init: if proximal_init:
with torch.no_grad(): with torch.no_grad():
self.conv_k.weight.copy_(self.conv_q.weight) self.conv_k.weight.copy_(self.conv_q.weight)
self.conv_k.bias.copy_(self.conv_q.bias) self.conv_k.bias.copy_(self.conv_q.bias)
def forward(self, x, c, attn_mask=None):
q = self.conv_q(x)
k = self.conv_k(c)
v = self.conv_v(c)
x, self.attn = self.attention(q, k, v, mask=attn_mask)
x = self.conv_o(x) def forward(self, x, c, attn_mask=None):
return x q = self.conv_q(x)
k = self.conv_k(c)
v = self.conv_v(c)
def attention(self, query, key, value, mask=None): x, self.attn = self.attention(q, k, v, mask=attn_mask)
# reshape [b, d, t] -> [b, n_h, t, d_k]
b, d, t_s, t_t = (*key.size(), query.size(2))
query = query.view(b, self.n_heads, self.k_channels, t_t).transpose(2, 3)
key = key.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
value = value.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
scores = torch.matmul(query / math.sqrt(self.k_channels), key.transpose(-2, -1)) x = self.conv_o(x)
if self.window_size is not None: return x
assert t_s == t_t, "Relative attention is only available for self-attention."
key_relative_embeddings = self._get_relative_embeddings(self.emb_rel_k, t_s)
rel_logits = self._matmul_with_relative_keys(query /math.sqrt(self.k_channels), key_relative_embeddings)
scores_local = self._relative_position_to_absolute_position(rel_logits)
scores = scores + scores_local
if self.proximal_bias:
assert t_s == t_t, "Proximal bias is only available for self-attention."
scores = scores + self._attention_bias_proximal(t_s).to(device=scores.device, dtype=scores.dtype)
if mask is not None:
scores = scores.masked_fill(mask == 0, -1e4)
if self.block_length is not None:
assert t_s == t_t, "Local attention is only available for self-attention."
block_mask = torch.ones_like(scores).triu(-self.block_length).tril(self.block_length)
scores = scores.masked_fill(block_mask == 0, -1e4)
p_attn = F.softmax(scores, dim=-1) # [b, n_h, t_t, t_s]
p_attn = self.drop(p_attn)
output = torch.matmul(p_attn, value)
if self.window_size is not None:
relative_weights = self._absolute_position_to_relative_position(p_attn)
value_relative_embeddings = self._get_relative_embeddings(self.emb_rel_v, t_s)
output = output + self._matmul_with_relative_values(relative_weights, value_relative_embeddings)
output = output.transpose(2, 3).contiguous().view(b, d, t_t) # [b, n_h, t_t, d_k] -> [b, d, t_t]
return output, p_attn
def _matmul_with_relative_values(self, x, y): def attention(self, query, key, value, mask=None):
""" # reshape [b, d, t] -> [b, n_h, t, d_k]
x: [b, h, l, m] b, d, t_s, t_t = (*key.size(), query.size(2))
y: [h or 1, m, d] query = query.view(b, self.n_heads, self.k_channels, t_t).transpose(2, 3)
ret: [b, h, l, d] key = key.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
""" value = value.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
ret = torch.matmul(x, y.unsqueeze(0))
return ret
def _matmul_with_relative_keys(self, x, y): scores = torch.matmul(query / math.sqrt(self.k_channels), key.transpose(-2, -1))
""" if self.window_size is not None:
x: [b, h, l, d] assert (
y: [h or 1, m, d] t_s == t_t
ret: [b, h, l, m] ), "Relative attention is only available for self-attention."
""" key_relative_embeddings = self._get_relative_embeddings(self.emb_rel_k, t_s)
ret = torch.matmul(x, y.unsqueeze(0).transpose(-2, -1)) rel_logits = self._matmul_with_relative_keys(
return ret query / math.sqrt(self.k_channels), key_relative_embeddings
)
scores_local = self._relative_position_to_absolute_position(rel_logits)
scores = scores + scores_local
if self.proximal_bias:
assert t_s == t_t, "Proximal bias is only available for self-attention."
scores = scores + self._attention_bias_proximal(t_s).to(
device=scores.device, dtype=scores.dtype
)
if mask is not None:
scores = scores.masked_fill(mask == 0, -1e4)
if self.block_length is not None:
assert (
t_s == t_t
), "Local attention is only available for self-attention."
block_mask = (
torch.ones_like(scores)
.triu(-self.block_length)
.tril(self.block_length)
)
scores = scores.masked_fill(block_mask == 0, -1e4)
p_attn = F.softmax(scores, dim=-1) # [b, n_h, t_t, t_s]
p_attn = self.drop(p_attn)
output = torch.matmul(p_attn, value)
if self.window_size is not None:
relative_weights = self._absolute_position_to_relative_position(p_attn)
value_relative_embeddings = self._get_relative_embeddings(
self.emb_rel_v, t_s
)
output = output + self._matmul_with_relative_values(
relative_weights, value_relative_embeddings
)
output = (
output.transpose(2, 3).contiguous().view(b, d, t_t)
) # [b, n_h, t_t, d_k] -> [b, d, t_t]
return output, p_attn
def _get_relative_embeddings(self, relative_embeddings, length): def _matmul_with_relative_values(self, x, y):
max_relative_position = 2 * self.window_size + 1 """
# Pad first before slice to avoid using cond ops. x: [b, h, l, m]
pad_length = max(length - (self.window_size + 1), 0) y: [h or 1, m, d]
slice_start_position = max((self.window_size + 1) - length, 0) ret: [b, h, l, d]
slice_end_position = slice_start_position + 2 * length - 1 """
if pad_length > 0: ret = torch.matmul(x, y.unsqueeze(0))
padded_relative_embeddings = F.pad( return ret
relative_embeddings,
commons.convert_pad_shape([[0, 0], [pad_length, pad_length], [0, 0]]))
else:
padded_relative_embeddings = relative_embeddings
used_relative_embeddings = padded_relative_embeddings[:,slice_start_position:slice_end_position]
return used_relative_embeddings
def _relative_position_to_absolute_position(self, x): def _matmul_with_relative_keys(self, x, y):
""" """
x: [b, h, l, 2*l-1] x: [b, h, l, d]
ret: [b, h, l, l] y: [h or 1, m, d]
""" ret: [b, h, l, m]
batch, heads, length, _ = x.size() """
# Concat columns of pad to shift from relative to absolute indexing. ret = torch.matmul(x, y.unsqueeze(0).transpose(-2, -1))
x = F.pad(x, commons.convert_pad_shape([[0,0],[0,0],[0,0],[0,1]])) return ret
# Concat extra elements so to add up to shape (len+1, 2*len-1). def _get_relative_embeddings(self, relative_embeddings, length):
x_flat = x.view([batch, heads, length * 2 * length]) max_relative_position = 2 * self.window_size + 1
x_flat = F.pad(x_flat, commons.convert_pad_shape([[0,0],[0,0],[0,length-1]])) # Pad first before slice to avoid using cond ops.
pad_length = max(length - (self.window_size + 1), 0)
slice_start_position = max((self.window_size + 1) - length, 0)
slice_end_position = slice_start_position + 2 * length - 1
if pad_length > 0:
padded_relative_embeddings = F.pad(
relative_embeddings,
commons.convert_pad_shape([[0, 0], [pad_length, pad_length], [0, 0]]),
)
else:
padded_relative_embeddings = relative_embeddings
used_relative_embeddings = padded_relative_embeddings[
:, slice_start_position:slice_end_position
]
return used_relative_embeddings
# Reshape and slice out the padded elements. def _relative_position_to_absolute_position(self, x):
x_final = x_flat.view([batch, heads, length+1, 2*length-1])[:, :, :length, length-1:] """
return x_final x: [b, h, l, 2*l-1]
ret: [b, h, l, l]
"""
batch, heads, length, _ = x.size()
# Concat columns of pad to shift from relative to absolute indexing.
x = F.pad(x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, 1]]))
def _absolute_position_to_relative_position(self, x): # Concat extra elements so to add up to shape (len+1, 2*len-1).
""" x_flat = x.view([batch, heads, length * 2 * length])
x: [b, h, l, l] x_flat = F.pad(
ret: [b, h, l, 2*l-1] x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [0, length - 1]])
""" )
batch, heads, length, _ = x.size()
# padd along column
x = F.pad(x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, length-1]]))
x_flat = x.view([batch, heads, length**2 + length*(length -1)])
# add 0's in the beginning that will skew the elements after reshape
x_flat = F.pad(x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [length, 0]]))
x_final = x_flat.view([batch, heads, length, 2*length])[:,:,:,1:]
return x_final
def _attention_bias_proximal(self, length): # Reshape and slice out the padded elements.
"""Bias for self-attention to encourage attention to close positions. x_final = x_flat.view([batch, heads, length + 1, 2 * length - 1])[
Args: :, :, :length, length - 1 :
length: an integer scalar. ]
Returns: return x_final
a Tensor with shape [1, 1, length, length]
""" def _absolute_position_to_relative_position(self, x):
r = torch.arange(length, dtype=torch.float32) """
diff = torch.unsqueeze(r, 0) - torch.unsqueeze(r, 1) x: [b, h, l, l]
return torch.unsqueeze(torch.unsqueeze(-torch.log1p(torch.abs(diff)), 0), 0) ret: [b, h, l, 2*l-1]
"""
batch, heads, length, _ = x.size()
# padd along column
x = F.pad(
x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, length - 1]])
)
x_flat = x.view([batch, heads, length**2 + length * (length - 1)])
# add 0's in the beginning that will skew the elements after reshape
x_flat = F.pad(x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [length, 0]]))
x_final = x_flat.view([batch, heads, length, 2 * length])[:, :, :, 1:]
return x_final
def _attention_bias_proximal(self, length):
"""Bias for self-attention to encourage attention to close positions.
Args:
length: an integer scalar.
Returns:
a Tensor with shape [1, 1, length, length]
"""
r = torch.arange(length, dtype=torch.float32)
diff = torch.unsqueeze(r, 0) - torch.unsqueeze(r, 1)
return torch.unsqueeze(torch.unsqueeze(-torch.log1p(torch.abs(diff)), 0), 0)
class FFN(nn.Module): class FFN(nn.Module):
def __init__(self, in_channels, out_channels, filter_channels, kernel_size, p_dropout=0., activation=None, causal=False): def __init__(
super().__init__() self,
self.in_channels = in_channels in_channels,
self.out_channels = out_channels out_channels,
self.filter_channels = filter_channels filter_channels,
self.kernel_size = kernel_size kernel_size,
self.p_dropout = p_dropout p_dropout=0.0,
self.activation = activation activation=None,
self.causal = causal causal=False,
):
super().__init__()
self.in_channels = in_channels
self.out_channels = out_channels
self.filter_channels = filter_channels
self.kernel_size = kernel_size
self.p_dropout = p_dropout
self.activation = activation
self.causal = causal
if causal: if causal:
self.padding = self._causal_padding self.padding = self._causal_padding
else: else:
self.padding = self._same_padding self.padding = self._same_padding
self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size) self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size)
self.conv_2 = nn.Conv1d(filter_channels, out_channels, kernel_size) self.conv_2 = nn.Conv1d(filter_channels, out_channels, kernel_size)
self.drop = nn.Dropout(p_dropout) self.drop = nn.Dropout(p_dropout)
def forward(self, x, x_mask): def forward(self, x, x_mask):
x = self.conv_1(self.padding(x * x_mask)) x = self.conv_1(self.padding(x * x_mask))
if self.activation == "gelu": if self.activation == "gelu":
x = x * torch.sigmoid(1.702 * x) x = x * torch.sigmoid(1.702 * x)
else: else:
x = torch.relu(x) x = torch.relu(x)
x = self.drop(x) x = self.drop(x)
x = self.conv_2(self.padding(x * x_mask)) x = self.conv_2(self.padding(x * x_mask))
return x * x_mask return x * x_mask
def _causal_padding(self, x):
if self.kernel_size == 1:
return x
pad_l = self.kernel_size - 1
pad_r = 0
padding = [[0, 0], [0, 0], [pad_l, pad_r]]
x = F.pad(x, commons.convert_pad_shape(padding))
return x
def _same_padding(self, x): def _causal_padding(self, x):
if self.kernel_size == 1: if self.kernel_size == 1:
return x return x
pad_l = (self.kernel_size - 1) // 2 pad_l = self.kernel_size - 1
pad_r = self.kernel_size // 2 pad_r = 0
padding = [[0, 0], [0, 0], [pad_l, pad_r]] padding = [[0, 0], [0, 0], [pad_l, pad_r]]
x = F.pad(x, commons.convert_pad_shape(padding)) x = F.pad(x, commons.convert_pad_shape(padding))
return x return x
def _same_padding(self, x):
if self.kernel_size == 1:
return x
pad_l = (self.kernel_size - 1) // 2
pad_r = self.kernel_size // 2
padding = [[0, 0], [0, 0], [pad_l, pad_r]]
x = F.pad(x, commons.convert_pad_shape(padding))
return x

8
bert_gen.py
View File

@@ -7,6 +7,7 @@ from text import cleaned_text_to_sequence, get_bert
import argparse import argparse
import torch.multiprocessing as mp import torch.multiprocessing as mp
def process_line(line): def process_line(line):
rank = mp.current_process()._identity rank = mp.current_process()._identity
rank = rank[0] if len(rank) > 0 else 0 rank = rank[0] if len(rank) > 0 else 0
@@ -34,15 +35,15 @@ def process_line(line):
bert = torch.load(bert_path) bert = torch.load(bert_path)
assert bert.shape[-1] == len(phone) assert bert.shape[-1] == len(phone)
except Exception: except Exception:
bert = get_bert(text, word2ph, language_str,device) bert = get_bert(text, word2ph, language_str, device)
assert bert.shape[-1] == len(phone) assert bert.shape[-1] == len(phone)
torch.save(bert, bert_path) torch.save(bert, bert_path)
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('-c', '--config', type=str, default="configs/config.json") parser.add_argument("-c", "--config", type=str, default="configs/config.json")
parser.add_argument('--num_processes', type=int, default=2 ) parser.add_argument("--num_processes", type=int, default=2)
args = parser.parse_args() args = parser.parse_args()
config_path = args.config config_path = args.config
hps = utils.get_hparams_from_file(config_path) hps = utils.get_hparams_from_file(config_path)
@@ -53,7 +54,6 @@ if __name__ == "__main__":
with open(hps.data.validation_files, encoding="utf-8") as f: with open(hps.data.validation_files, encoding="utf-8") as f:
lines.extend(f.readlines()) lines.extend(f.readlines())
num_processes = args.num_processes num_processes = args.num_processes
with Pool(processes=num_processes) as pool: with Pool(processes=num_processes) as pool:
for _ in tqdm(pool.imap_unordered(process_line, lines), total=len(lines)): for _ in tqdm(pool.imap_unordered(process_line, lines), total=len(lines)):

198
commons.py
View File

@@ -6,156 +6,158 @@ from torch.nn import functional as F
def init_weights(m, mean=0.0, std=0.01): def init_weights(m, mean=0.0, std=0.01):
classname = m.__class__.__name__ classname = m.__class__.__name__
if classname.find("Conv") != -1: if classname.find("Conv") != -1:
m.weight.data.normal_(mean, std) m.weight.data.normal_(mean, std)
def get_padding(kernel_size, dilation=1): def get_padding(kernel_size, dilation=1):
return int((kernel_size*dilation - dilation)/2) return int((kernel_size * dilation - dilation) / 2)
def convert_pad_shape(pad_shape): def convert_pad_shape(pad_shape):
l = pad_shape[::-1] l = pad_shape[::-1]
pad_shape = [item for sublist in l for item in sublist] pad_shape = [item for sublist in l for item in sublist]
return pad_shape return pad_shape
def intersperse(lst, item): def intersperse(lst, item):
result = [item] * (len(lst) * 2 + 1) result = [item] * (len(lst) * 2 + 1)
result[1::2] = lst result[1::2] = lst
return result return result
def kl_divergence(m_p, logs_p, m_q, logs_q): def kl_divergence(m_p, logs_p, m_q, logs_q):
"""KL(P||Q)""" """KL(P||Q)"""
kl = (logs_q - logs_p) - 0.5 kl = (logs_q - logs_p) - 0.5
kl += 0.5 * (torch.exp(2. * logs_p) + ((m_p - m_q)**2)) * torch.exp(-2. * logs_q) kl += (
return kl 0.5 * (torch.exp(2.0 * logs_p) + ((m_p - m_q) ** 2)) * torch.exp(-2.0 * logs_q)
)
return kl
def rand_gumbel(shape): def rand_gumbel(shape):
"""Sample from the Gumbel distribution, protect from overflows.""" """Sample from the Gumbel distribution, protect from overflows."""
uniform_samples = torch.rand(shape) * 0.99998 + 0.00001 uniform_samples = torch.rand(shape) * 0.99998 + 0.00001
return -torch.log(-torch.log(uniform_samples)) return -torch.log(-torch.log(uniform_samples))
def rand_gumbel_like(x): def rand_gumbel_like(x):
g = rand_gumbel(x.size()).to(dtype=x.dtype, device=x.device) g = rand_gumbel(x.size()).to(dtype=x.dtype, device=x.device)
return g return g
def slice_segments(x, ids_str, segment_size=4): def slice_segments(x, ids_str, segment_size=4):
ret = torch.zeros_like(x[:, :, :segment_size]) ret = torch.zeros_like(x[:, :, :segment_size])
for i in range(x.size(0)): for i in range(x.size(0)):
idx_str = ids_str[i] idx_str = ids_str[i]
idx_end = idx_str + segment_size idx_end = idx_str + segment_size
ret[i] = x[i, :, idx_str:idx_end] ret[i] = x[i, :, idx_str:idx_end]
return ret return ret
def rand_slice_segments(x, x_lengths=None, segment_size=4): def rand_slice_segments(x, x_lengths=None, segment_size=4):
b, d, t = x.size() b, d, t = x.size()
if x_lengths is None: if x_lengths is None:
x_lengths = t x_lengths = t
ids_str_max = x_lengths - segment_size + 1 ids_str_max = x_lengths - segment_size + 1
ids_str = (torch.rand([b]).to(device=x.device) * ids_str_max).to(dtype=torch.long) ids_str = (torch.rand([b]).to(device=x.device) * ids_str_max).to(dtype=torch.long)
ret = slice_segments(x, ids_str, segment_size) ret = slice_segments(x, ids_str, segment_size)
return ret, ids_str return ret, ids_str
def get_timing_signal_1d( def get_timing_signal_1d(length, channels, min_timescale=1.0, max_timescale=1.0e4):
length, channels, min_timescale=1.0, max_timescale=1.0e4): position = torch.arange(length, dtype=torch.float)
position = torch.arange(length, dtype=torch.float) num_timescales = channels // 2
num_timescales = channels // 2 log_timescale_increment = math.log(float(max_timescale) / float(min_timescale)) / (
log_timescale_increment = ( num_timescales - 1
math.log(float(max_timescale) / float(min_timescale)) / )
(num_timescales - 1)) inv_timescales = min_timescale * torch.exp(
inv_timescales = min_timescale * torch.exp( torch.arange(num_timescales, dtype=torch.float) * -log_timescale_increment
torch.arange(num_timescales, dtype=torch.float) * -log_timescale_increment) )
scaled_time = position.unsqueeze(0) * inv_timescales.unsqueeze(1) scaled_time = position.unsqueeze(0) * inv_timescales.unsqueeze(1)
signal = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], 0) signal = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], 0)
signal = F.pad(signal, [0, 0, 0, channels % 2]) signal = F.pad(signal, [0, 0, 0, channels % 2])
signal = signal.view(1, channels, length) signal = signal.view(1, channels, length)
return signal return signal
def add_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4): def add_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4):
b, channels, length = x.size() b, channels, length = x.size()
signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale) signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
return x + signal.to(dtype=x.dtype, device=x.device) return x + signal.to(dtype=x.dtype, device=x.device)
def cat_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4, axis=1): def cat_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4, axis=1):
b, channels, length = x.size() b, channels, length = x.size()
signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale) signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
return torch.cat([x, signal.to(dtype=x.dtype, device=x.device)], axis) return torch.cat([x, signal.to(dtype=x.dtype, device=x.device)], axis)
def subsequent_mask(length): def subsequent_mask(length):
mask = torch.tril(torch.ones(length, length)).unsqueeze(0).unsqueeze(0) mask = torch.tril(torch.ones(length, length)).unsqueeze(0).unsqueeze(0)
return mask return mask
@torch.jit.script @torch.jit.script
def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels): def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
n_channels_int = n_channels[0] n_channels_int = n_channels[0]
in_act = input_a + input_b in_act = input_a + input_b
t_act = torch.tanh(in_act[:, :n_channels_int, :]) t_act = torch.tanh(in_act[:, :n_channels_int, :])
s_act = torch.sigmoid(in_act[:, n_channels_int:, :]) s_act = torch.sigmoid(in_act[:, n_channels_int:, :])
acts = t_act * s_act acts = t_act * s_act
return acts return acts
def convert_pad_shape(pad_shape): def convert_pad_shape(pad_shape):
l = pad_shape[::-1] l = pad_shape[::-1]
pad_shape = [item for sublist in l for item in sublist] pad_shape = [item for sublist in l for item in sublist]
return pad_shape return pad_shape
def shift_1d(x): def shift_1d(x):
x = F.pad(x, convert_pad_shape([[0, 0], [0, 0], [1, 0]]))[:, :, :-1] x = F.pad(x, convert_pad_shape([[0, 0], [0, 0], [1, 0]]))[:, :, :-1]
return x return x
def sequence_mask(length, max_length=None): def sequence_mask(length, max_length=None):
if max_length is None: if max_length is None:
max_length = length.max() max_length = length.max()
x = torch.arange(max_length, dtype=length.dtype, device=length.device) x = torch.arange(max_length, dtype=length.dtype, device=length.device)
return x.unsqueeze(0) < length.unsqueeze(1) return x.unsqueeze(0) < length.unsqueeze(1)
def generate_path(duration, mask): def generate_path(duration, mask):
""" """
duration: [b, 1, t_x] duration: [b, 1, t_x]
mask: [b, 1, t_y, t_x] mask: [b, 1, t_y, t_x]
""" """
device = duration.device device = duration.device
b, _, t_y, t_x = mask.shape b, _, t_y, t_x = mask.shape
cum_duration = torch.cumsum(duration, -1) cum_duration = torch.cumsum(duration, -1)
cum_duration_flat = cum_duration.view(b * t_x) cum_duration_flat = cum_duration.view(b * t_x)
path = sequence_mask(cum_duration_flat, t_y).to(mask.dtype) path = sequence_mask(cum_duration_flat, t_y).to(mask.dtype)
path = path.view(b, t_x, t_y) path = path.view(b, t_x, t_y)
path = path - F.pad(path, convert_pad_shape([[0, 0], [1, 0], [0, 0]]))[:, :-1] path = path - F.pad(path, convert_pad_shape([[0, 0], [1, 0], [0, 0]]))[:, :-1]
path = path.unsqueeze(1).transpose(2,3) * mask path = path.unsqueeze(1).transpose(2, 3) * mask
return path return path
def clip_grad_value_(parameters, clip_value, norm_type=2): def clip_grad_value_(parameters, clip_value, norm_type=2):
if isinstance(parameters, torch.Tensor): if isinstance(parameters, torch.Tensor):
parameters = [parameters] parameters = [parameters]
parameters = list(filter(lambda p: p.grad is not None, parameters)) parameters = list(filter(lambda p: p.grad is not None, parameters))
norm_type = float(norm_type) norm_type = float(norm_type)
if clip_value is not None:
clip_value = float(clip_value)
total_norm = 0
for p in parameters:
param_norm = p.grad.data.norm(norm_type)
total_norm += param_norm.item() ** norm_type
if clip_value is not None: if clip_value is not None:
p.grad.data.clamp_(min=-clip_value, max=clip_value) clip_value = float(clip_value)
total_norm = total_norm ** (1. / norm_type)
return total_norm total_norm = 0
for p in parameters:
param_norm = p.grad.data.norm(norm_type)
total_norm += param_norm.item() ** norm_type
if clip_value is not None:
p.grad.data.clamp_(min=-clip_value, max=clip_value)
total_norm = total_norm ** (1.0 / norm_type)
return total_norm

153
data_utils.py
View File

@@ -16,9 +16,9 @@ from text import cleaned_text_to_sequence, get_bert
class TextAudioSpeakerLoader(torch.utils.data.Dataset): class TextAudioSpeakerLoader(torch.utils.data.Dataset):
""" """
1) loads audio, speaker_id, text pairs 1) loads audio, speaker_id, text pairs
2) normalizes text and converts them to sequences of integers 2) normalizes text and converts them to sequences of integers
3) computes spectrograms from audio files. 3) computes spectrograms from audio files.
""" """
def __init__(self, audiopaths_sid_text, hparams): def __init__(self, audiopaths_sid_text, hparams):
@@ -32,7 +32,9 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
self.spk_map = hparams.spk2id self.spk_map = hparams.spk2id
self.hparams = hparams self.hparams = hparams
self.use_mel_spec_posterior = getattr(hparams, "use_mel_posterior_encoder", False) self.use_mel_spec_posterior = getattr(
hparams, "use_mel_posterior_encoder", False
)
if self.use_mel_spec_posterior: if self.use_mel_spec_posterior:
self.n_mel_channels = getattr(hparams, "n_mel_channels", 80) self.n_mel_channels = getattr(hparams, "n_mel_channels", 80)
@@ -58,17 +60,26 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
lengths = [] lengths = []
skipped = 0 skipped = 0
logger.info("Init dataset...") logger.info("Init dataset...")
for _id, spk, language, text, phones, tone, word2ph in tqdm(self.audiopaths_sid_text): for _id, spk, language, text, phones, tone, word2ph in tqdm(
audiopath = f'{_id}' self.audiopaths_sid_text
):
audiopath = f"{_id}"
if self.min_text_len <= len(phones) and len(phones) <= self.max_text_len: if self.min_text_len <= len(phones) and len(phones) <= self.max_text_len:
phones = phones.split(" ") phones = phones.split(" ")
tone = [int(i) for i in tone.split(" ")] tone = [int(i) for i in tone.split(" ")]
word2ph = [int(i) for i in word2ph.split(" ")] word2ph = [int(i) for i in word2ph.split(" ")]
audiopaths_sid_text_new.append([audiopath, spk, language, text, phones, tone, word2ph]) audiopaths_sid_text_new.append(
[audiopath, spk, language, text, phones, tone, word2ph]
)
lengths.append(os.path.getsize(audiopath) // (2 * self.hop_length)) lengths.append(os.path.getsize(audiopath) // (2 * self.hop_length))
else: else:
skipped += 1 skipped += 1
logger.info("skipped: " + str(skipped) + ", total: " + str(len(self.audiopaths_sid_text))) logger.info(
"skipped: "
+ str(skipped)
+ ", total: "
+ str(len(self.audiopaths_sid_text))
)
self.audiopaths_sid_text = audiopaths_sid_text_new self.audiopaths_sid_text = audiopaths_sid_text_new
self.lengths = lengths self.lengths = lengths
@@ -76,7 +87,9 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
# separate filename, speaker_id and text # separate filename, speaker_id and text
audiopath, sid, language, text, phones, tone, word2ph = audiopath_sid_text audiopath, sid, language, text, phones, tone, word2ph = audiopath_sid_text
bert, ja_bert, phones, tone, language = self.get_text(text, word2ph, phones, tone, language, audiopath) bert, ja_bert, phones, tone, language = self.get_text(
text, word2ph, phones, tone, language, audiopath
)
spec, wav = self.get_audio(audiopath) spec, wav = self.get_audio(audiopath)
sid = torch.LongTensor([int(self.spk_map[sid])]) sid = torch.LongTensor([int(self.spk_map[sid])])
@@ -85,8 +98,11 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
def get_audio(self, filename): def get_audio(self, filename):
audio, sampling_rate = load_wav_to_torch(filename) audio, sampling_rate = load_wav_to_torch(filename)
if sampling_rate != self.sampling_rate: if sampling_rate != self.sampling_rate:
raise ValueError("{} {} SR doesn't match target {} SR".format( raise ValueError(
sampling_rate, self.sampling_rate)) "{} {} SR doesn't match target {} SR".format(
sampling_rate, self.sampling_rate
)
)
audio_norm = audio / self.max_wav_value audio_norm = audio / self.max_wav_value
audio_norm = audio_norm.unsqueeze(0) audio_norm = audio_norm.unsqueeze(0)
spec_filename = filename.replace(".wav", ".spec.pt") spec_filename = filename.replace(".wav", ".spec.pt")
@@ -96,13 +112,26 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
spec = torch.load(spec_filename) spec = torch.load(spec_filename)
except: except:
if self.use_mel_spec_posterior: if self.use_mel_spec_posterior:
spec = mel_spectrogram_torch(audio_norm, self.filter_length, spec = mel_spectrogram_torch(
self.n_mel_channels, self.sampling_rate, self.hop_length, audio_norm,
self.win_length, self.hparams.mel_fmin, self.hparams.mel_fmax, center=False) self.filter_length,
self.n_mel_channels,
self.sampling_rate,
self.hop_length,
self.win_length,
self.hparams.mel_fmin,
self.hparams.mel_fmax,
center=False,
)
else: else:
spec = spectrogram_torch(audio_norm, self.filter_length, spec = spectrogram_torch(
self.sampling_rate, self.hop_length, self.win_length, audio_norm,
center=False) self.filter_length,
self.sampling_rate,
self.hop_length,
self.win_length,
center=False,
)
spec = torch.squeeze(spec, 0) spec = torch.squeeze(spec, 0)
torch.save(spec, spec_filename) torch.save(spec, spec_filename)
return spec, audio_norm return spec, audio_norm
@@ -125,17 +154,29 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
torch.save(bert, bert_path) torch.save(bert, bert_path)
assert bert.shape[-1] == len(phone), phone assert bert.shape[-1] == len(phone), phone
if language_str=='ZH': if language_str == "ZH":
bert = bert bert = bert
ja_bert = torch.zeros(768, len(phone)) ja_bert = torch.zeros(768, len(phone))
elif language_str=="JA": elif language_str == "JA":
ja_bert = bert ja_bert = bert
bert = torch.zeros(1024, len(phone)) bert = torch.zeros(1024, len(phone))
else: else:
bert = torch.zeros(1024, len(phone)) bert = torch.zeros(1024, len(phone))
ja_bert = torch.zeros(768, len(phone)) ja_bert = torch.zeros(768, len(phone))
assert bert.shape[-1] == len(phone), ( assert bert.shape[-1] == len(phone), (
bert.shape, len(phone), sum(word2ph), p1, p2, t1, t2, pold, pold2, word2ph, text, w2pho) bert.shape,
len(phone),
sum(word2ph),
p1,
p2,
t1,
t2,
pold,
pold2,
word2ph,
text,
w2pho,
)
phone = torch.LongTensor(phone) phone = torch.LongTensor(phone)
tone = torch.LongTensor(tone) tone = torch.LongTensor(tone)
language = torch.LongTensor(language) language = torch.LongTensor(language)
@@ -152,9 +193,8 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
return len(self.audiopaths_sid_text) return len(self.audiopaths_sid_text)
class TextAudioSpeakerCollate(): class TextAudioSpeakerCollate:
""" Zero-pads model inputs and targets """Zero-pads model inputs and targets"""
"""
def __init__(self, return_ids=False): def __init__(self, return_ids=False):
self.return_ids = return_ids self.return_ids = return_ids
@@ -167,8 +207,8 @@ class TextAudioSpeakerCollate():
""" """
# Right zero-pad all one-hot text sequences to max input length # Right zero-pad all one-hot text sequences to max input length
_, ids_sorted_decreasing = torch.sort( _, ids_sorted_decreasing = torch.sort(
torch.LongTensor([x[1].size(1) for x in batch]), torch.LongTensor([x[1].size(1) for x in batch]), dim=0, descending=True
dim=0, descending=True) )
max_text_len = max([len(x[0]) for x in batch]) max_text_len = max([len(x[0]) for x in batch])
max_spec_len = max([x[1].size(1) for x in batch]) max_spec_len = max([x[1].size(1) for x in batch])
@@ -198,32 +238,44 @@ class TextAudioSpeakerCollate():
row = batch[ids_sorted_decreasing[i]] row = batch[ids_sorted_decreasing[i]]
text = row[0] text = row[0]
text_padded[i, :text.size(0)] = text text_padded[i, : text.size(0)] = text
text_lengths[i] = text.size(0) text_lengths[i] = text.size(0)
spec = row[1] spec = row[1]
spec_padded[i, :, :spec.size(1)] = spec spec_padded[i, :, : spec.size(1)] = spec
spec_lengths[i] = spec.size(1) spec_lengths[i] = spec.size(1)
wav = row[2] wav = row[2]
wav_padded[i, :, :wav.size(1)] = wav wav_padded[i, :, : wav.size(1)] = wav
wav_lengths[i] = wav.size(1) wav_lengths[i] = wav.size(1)
sid[i] = row[3] sid[i] = row[3]
tone = row[4] tone = row[4]
tone_padded[i, :tone.size(0)] = tone tone_padded[i, : tone.size(0)] = tone
language = row[5] language = row[5]
language_padded[i, :language.size(0)] = language language_padded[i, : language.size(0)] = language
bert = row[6] bert = row[6]
bert_padded[i, :, :bert.size(1)] = bert bert_padded[i, :, : bert.size(1)] = bert
ja_bert = row[7] ja_bert = row[7]
ja_bert_padded[i, :, :ja_bert.size(1)] = ja_bert ja_bert_padded[i, :, : ja_bert.size(1)] = ja_bert
return text_padded, text_lengths, spec_padded, spec_lengths, wav_padded, wav_lengths, sid, tone_padded, language_padded, bert_padded, ja_bert_padded return (
text_padded,
text_lengths,
spec_padded,
spec_lengths,
wav_padded,
wav_lengths,
sid,
tone_padded,
language_padded,
bert_padded,
ja_bert_padded,
)
class DistributedBucketSampler(torch.utils.data.distributed.DistributedSampler): class DistributedBucketSampler(torch.utils.data.distributed.DistributedSampler):
@@ -236,7 +288,15 @@ class DistributedBucketSampler(torch.utils.data.distributed.DistributedSampler):
Ex) boundaries = [b1, b2, b3] -> any x s.t. length(x) <= b1 or length(x) > b3 are discarded. Ex) boundaries = [b1, b2, b3] -> any x s.t. length(x) <= b1 or length(x) > b3 are discarded.
""" """
def __init__(self, dataset, batch_size, boundaries, num_replicas=None, rank=None, shuffle=True): def __init__(
self,
dataset,
batch_size,
boundaries,
num_replicas=None,
rank=None,
shuffle=True,
):
super().__init__(dataset, num_replicas=num_replicas, rank=rank, shuffle=shuffle) super().__init__(dataset, num_replicas=num_replicas, rank=rank, shuffle=shuffle)
self.lengths = dataset.lengths self.lengths = dataset.lengths
self.batch_size = batch_size self.batch_size = batch_size
@@ -254,7 +314,7 @@ class DistributedBucketSampler(torch.utils.data.distributed.DistributedSampler):
if idx_bucket != -1: if idx_bucket != -1:
buckets[idx_bucket].append(i) buckets[idx_bucket].append(i)
try: try:
for i in range(len(buckets) - 1, 0, -1): for i in range(len(buckets) - 1, 0, -1):
if len(buckets[i]) == 0: if len(buckets[i]) == 0:
buckets.pop(i) buckets.pop(i)
@@ -262,7 +322,7 @@ class DistributedBucketSampler(torch.utils.data.distributed.DistributedSampler):
assert all(len(bucket) > 0 for bucket in buckets) assert all(len(bucket) > 0 for bucket in buckets)
# When one bucket is not traversed # When one bucket is not traversed
except Exception as e: except Exception as e:
print('Bucket warning ', e) print("Bucket warning ", e)
for i in range(len(buckets) - 1, -1, -1): for i in range(len(buckets) - 1, -1, -1):
if len(buckets[i]) == 0: if len(buckets[i]) == 0:
buckets.pop(i) buckets.pop(i)
@@ -272,7 +332,9 @@ class DistributedBucketSampler(torch.utils.data.distributed.DistributedSampler):
for i in range(len(buckets)): for i in range(len(buckets)):
len_bucket = len(buckets[i]) len_bucket = len(buckets[i])
total_batch_size = self.num_replicas * self.batch_size total_batch_size = self.num_replicas * self.batch_size
rem = (total_batch_size - (len_bucket % total_batch_size)) % total_batch_size rem = (
total_batch_size - (len_bucket % total_batch_size)
) % total_batch_size
num_samples_per_bucket.append(len_bucket + rem) num_samples_per_bucket.append(len_bucket + rem)
return buckets, num_samples_per_bucket return buckets, num_samples_per_bucket
@@ -293,21 +355,30 @@ class DistributedBucketSampler(torch.utils.data.distributed.DistributedSampler):
for i in range(len(self.buckets)): for i in range(len(self.buckets)):
bucket = self.buckets[i] bucket = self.buckets[i]
len_bucket = len(bucket) len_bucket = len(bucket)
if (len_bucket == 0): if len_bucket == 0:
continue continue
ids_bucket = indices[i] ids_bucket = indices[i]
num_samples_bucket = self.num_samples_per_bucket[i] num_samples_bucket = self.num_samples_per_bucket[i]
# add extra samples to make it evenly divisible # add extra samples to make it evenly divisible
rem = num_samples_bucket - len_bucket rem = num_samples_bucket - len_bucket
ids_bucket = ids_bucket + ids_bucket * (rem // len_bucket) + ids_bucket[:(rem % len_bucket)] ids_bucket = (
ids_bucket
+ ids_bucket * (rem // len_bucket)
+ ids_bucket[: (rem % len_bucket)]
)
# subsample # subsample
ids_bucket = ids_bucket[self.rank::self.num_replicas] ids_bucket = ids_bucket[self.rank :: self.num_replicas]
# batching # batching
for j in range(len(ids_bucket) // self.batch_size): for j in range(len(ids_bucket) // self.batch_size):
batch = [bucket[idx] for idx in ids_bucket[j * self.batch_size:(j + 1) * self.batch_size]] batch = [
bucket[idx]
for idx in ids_bucket[
j * self.batch_size : (j + 1) * self.batch_size
]
]
batches.append(batch) batches.append(batch)
if self.shuffle: if self.shuffle:

84
losses.py
View File

@@ -1,61 +1,61 @@
import torch import torch
from torch.nn import functional as F from torch.nn import functional as F
import commons import commons
def feature_loss(fmap_r, fmap_g): def feature_loss(fmap_r, fmap_g):
loss = 0 loss = 0
for dr, dg in zip(fmap_r, fmap_g): for dr, dg in zip(fmap_r, fmap_g):
for rl, gl in zip(dr, dg): for rl, gl in zip(dr, dg):
rl = rl.float().detach() rl = rl.float().detach()
gl = gl.float() gl = gl.float()
loss += torch.mean(torch.abs(rl - gl)) loss += torch.mean(torch.abs(rl - gl))
return loss * 2 return loss * 2
def discriminator_loss(disc_real_outputs, disc_generated_outputs): def discriminator_loss(disc_real_outputs, disc_generated_outputs):
loss = 0 loss = 0
r_losses = [] r_losses = []
g_losses = [] g_losses = []
for dr, dg in zip(disc_real_outputs, disc_generated_outputs): for dr, dg in zip(disc_real_outputs, disc_generated_outputs):
dr = dr.float() dr = dr.float()
dg = dg.float() dg = dg.float()
r_loss = torch.mean((1-dr)**2) r_loss = torch.mean((1 - dr) ** 2)
g_loss = torch.mean(dg**2) g_loss = torch.mean(dg**2)
loss += (r_loss + g_loss) loss += r_loss + g_loss
r_losses.append(r_loss.item()) r_losses.append(r_loss.item())
g_losses.append(g_loss.item()) g_losses.append(g_loss.item())
return loss, r_losses, g_losses return loss, r_losses, g_losses
def generator_loss(disc_outputs): def generator_loss(disc_outputs):
loss = 0 loss = 0
gen_losses = [] gen_losses = []
for dg in disc_outputs: for dg in disc_outputs:
dg = dg.float() dg = dg.float()
l = torch.mean((1-dg)**2) l = torch.mean((1 - dg) ** 2)
gen_losses.append(l) gen_losses.append(l)
loss += l loss += l
return loss, gen_losses return loss, gen_losses
def kl_loss(z_p, logs_q, m_p, logs_p, z_mask): def kl_loss(z_p, logs_q, m_p, logs_p, z_mask):
""" """
z_p, logs_q: [b, h, t_t] z_p, logs_q: [b, h, t_t]
m_p, logs_p: [b, h, t_t] m_p, logs_p: [b, h, t_t]
""" """
z_p = z_p.float() z_p = z_p.float()
logs_q = logs_q.float() logs_q = logs_q.float()
m_p = m_p.float() m_p = m_p.float()
logs_p = logs_p.float() logs_p = logs_p.float()
z_mask = z_mask.float() z_mask = z_mask.float()
kl = logs_p - logs_q - 0.5 kl = logs_p - logs_q - 0.5
kl += 0.5 * ((z_p - m_p)**2) * torch.exp(-2. * logs_p) kl += 0.5 * ((z_p - m_p) ** 2) * torch.exp(-2.0 * logs_p)
kl = torch.sum(kl * z_mask) kl = torch.sum(kl * z_mask)
l = kl / torch.sum(z_mask) l = kl / torch.sum(z_mask)
return l return l

90
mel_processing.py
View File

@@ -49,22 +49,38 @@ hann_window = {}
def spectrogram_torch(y, n_fft, sampling_rate, hop_size, win_size, center=False): def spectrogram_torch(y, n_fft, sampling_rate, hop_size, win_size, center=False):
if torch.min(y) < -1.: if torch.min(y) < -1.0:
print('min value is ', torch.min(y)) print("min value is ", torch.min(y))
if torch.max(y) > 1.: if torch.max(y) > 1.0:
print('max value is ', torch.max(y)) print("max value is ", torch.max(y))
global hann_window global hann_window
dtype_device = str(y.dtype) + '_' + str(y.device) dtype_device = str(y.dtype) + "_" + str(y.device)
wnsize_dtype_device = str(win_size) + '_' + dtype_device wnsize_dtype_device = str(win_size) + "_" + dtype_device
if wnsize_dtype_device not in hann_window: if wnsize_dtype_device not in hann_window:
hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(dtype=y.dtype, device=y.device) hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(
dtype=y.dtype, device=y.device
)
y = torch.nn.functional.pad(y.unsqueeze(1), (int((n_fft-hop_size)/2), int((n_fft-hop_size)/2)), mode='reflect') y = torch.nn.functional.pad(
y.unsqueeze(1),
(int((n_fft - hop_size) / 2), int((n_fft - hop_size) / 2)),
mode="reflect",
)
y = y.squeeze(1) y = y.squeeze(1)
spec = torch.stft(y, n_fft, hop_length=hop_size, win_length=win_size, window=hann_window[wnsize_dtype_device], spec = torch.stft(
center=center, pad_mode='reflect', normalized=False, onesided=True, return_complex=False) y,
n_fft,
hop_length=hop_size,
win_length=win_size,
window=hann_window[wnsize_dtype_device],
center=center,
pad_mode="reflect",
normalized=False,
onesided=True,
return_complex=False,
)
spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6) spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6)
return spec return spec
@@ -72,37 +88,59 @@ def spectrogram_torch(y, n_fft, sampling_rate, hop_size, win_size, center=False)
def spec_to_mel_torch(spec, n_fft, num_mels, sampling_rate, fmin, fmax): def spec_to_mel_torch(spec, n_fft, num_mels, sampling_rate, fmin, fmax):
global mel_basis global mel_basis
dtype_device = str(spec.dtype) + '_' + str(spec.device) dtype_device = str(spec.dtype) + "_" + str(spec.device)
fmax_dtype_device = str(fmax) + '_' + dtype_device fmax_dtype_device = str(fmax) + "_" + dtype_device
if fmax_dtype_device not in mel_basis: if fmax_dtype_device not in mel_basis:
mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax) mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax)
mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(dtype=spec.dtype, device=spec.device) mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(
dtype=spec.dtype, device=spec.device
)
spec = torch.matmul(mel_basis[fmax_dtype_device], spec) spec = torch.matmul(mel_basis[fmax_dtype_device], spec)
spec = spectral_normalize_torch(spec) spec = spectral_normalize_torch(spec)
return spec return spec
def mel_spectrogram_torch(y, n_fft, num_mels, sampling_rate, hop_size, win_size, fmin, fmax, center=False): def mel_spectrogram_torch(
if torch.min(y) < -1.: y, n_fft, num_mels, sampling_rate, hop_size, win_size, fmin, fmax, center=False
print('min value is ', torch.min(y)) ):
if torch.max(y) > 1.: if torch.min(y) < -1.0:
print('max value is ', torch.max(y)) print("min value is ", torch.min(y))
if torch.max(y) > 1.0:
print("max value is ", torch.max(y))
global mel_basis, hann_window global mel_basis, hann_window
dtype_device = str(y.dtype) + '_' + str(y.device) dtype_device = str(y.dtype) + "_" + str(y.device)
fmax_dtype_device = str(fmax) + '_' + dtype_device fmax_dtype_device = str(fmax) + "_" + dtype_device
wnsize_dtype_device = str(win_size) + '_' + dtype_device wnsize_dtype_device = str(win_size) + "_" + dtype_device
if fmax_dtype_device not in mel_basis: if fmax_dtype_device not in mel_basis:
mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax) mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax)
mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(dtype=y.dtype, device=y.device) mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(
dtype=y.dtype, device=y.device
)
if wnsize_dtype_device not in hann_window: if wnsize_dtype_device not in hann_window:
hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(dtype=y.dtype, device=y.device) hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(
dtype=y.dtype, device=y.device
)
y = torch.nn.functional.pad(y.unsqueeze(1), (int((n_fft-hop_size)/2), int((n_fft-hop_size)/2)), mode='reflect') y = torch.nn.functional.pad(
y.unsqueeze(1),
(int((n_fft - hop_size) / 2), int((n_fft - hop_size) / 2)),
mode="reflect",
)
y = y.squeeze(1) y = y.squeeze(1)
spec = torch.stft(y, n_fft, hop_length=hop_size, win_length=win_size, window=hann_window[wnsize_dtype_device], spec = torch.stft(
center=center, pad_mode='reflect', normalized=False, onesided=True, return_complex=False) y,
n_fft,
hop_length=hop_size,
win_length=win_size,
window=hann_window[wnsize_dtype_device],
center=center,
pad_mode="reflect",
normalized=False,
onesided=True,
return_complex=False,
)
spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6) spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6)

3
models.py
View File

@@ -104,7 +104,6 @@ class TransformerCouplingBlock(nn.Module):
gin_channels=0, gin_channels=0,
share_parameter=False, share_parameter=False,
): ):
super().__init__() super().__init__()
self.channels = channels self.channels = channels
self.hidden_channels = hidden_channels self.hidden_channels = hidden_channels
@@ -685,7 +684,6 @@ class ReferenceEncoder(nn.Module):
""" """
def __init__(self, spec_channels, gin_channels=0): def __init__(self, spec_channels, gin_channels=0):
super().__init__() super().__init__()
self.spec_channels = spec_channels self.spec_channels = spec_channels
ref_enc_filters = [32, 32, 64, 64, 128, 128] ref_enc_filters = [32, 32, 64, 64, 128, 128]
@@ -770,7 +768,6 @@ class SynthesizerTrn(nn.Module):
use_transformer_flow=True, use_transformer_flow=True,
**kwargs **kwargs
): ):
super().__init__() super().__init__()
self.n_vocab = n_vocab self.n_vocab = n_vocab
self.spec_channels = spec_channels self.spec_channels = spec_channels

806
modules.py
View File

@@ -16,193 +16,284 @@ from attentions import Encoder
LRELU_SLOPE = 0.1 LRELU_SLOPE = 0.1
class LayerNorm(nn.Module): class LayerNorm(nn.Module):
def __init__(self, channels, eps=1e-5): def __init__(self, channels, eps=1e-5):
super().__init__() super().__init__()
self.channels = channels self.channels = channels
self.eps = eps self.eps = eps
self.gamma = nn.Parameter(torch.ones(channels))
self.beta = nn.Parameter(torch.zeros(channels))
def forward(self, x):
x = x.transpose(1, -1)
x = F.layer_norm(x, (self.channels,), self.gamma, self.beta, self.eps)
return x.transpose(1, -1)
self.gamma = nn.Parameter(torch.ones(channels))
self.beta = nn.Parameter(torch.zeros(channels))
def forward(self, x):
x = x.transpose(1, -1)
x = F.layer_norm(x, (self.channels,), self.gamma, self.beta, self.eps)
return x.transpose(1, -1)
class ConvReluNorm(nn.Module): class ConvReluNorm(nn.Module):
def __init__(self, in_channels, hidden_channels, out_channels, kernel_size, n_layers, p_dropout): def __init__(
super().__init__() self,
self.in_channels = in_channels in_channels,
self.hidden_channels = hidden_channels hidden_channels,
self.out_channels = out_channels out_channels,
self.kernel_size = kernel_size kernel_size,
self.n_layers = n_layers n_layers,
self.p_dropout = p_dropout p_dropout,
assert n_layers > 1, "Number of layers should be larger than 0." ):
super().__init__()
self.in_channels = in_channels
self.hidden_channels = hidden_channels
self.out_channels = out_channels
self.kernel_size = kernel_size
self.n_layers = n_layers
self.p_dropout = p_dropout
assert n_layers > 1, "Number of layers should be larger than 0."
self.conv_layers = nn.ModuleList() self.conv_layers = nn.ModuleList()
self.norm_layers = nn.ModuleList() self.norm_layers = nn.ModuleList()
self.conv_layers.append(nn.Conv1d(in_channels, hidden_channels, kernel_size, padding=kernel_size//2)) self.conv_layers.append(
self.norm_layers.append(LayerNorm(hidden_channels)) nn.Conv1d(
self.relu_drop = nn.Sequential( in_channels, hidden_channels, kernel_size, padding=kernel_size // 2
nn.ReLU(), )
nn.Dropout(p_dropout)) )
for _ in range(n_layers-1): self.norm_layers.append(LayerNorm(hidden_channels))
self.conv_layers.append(nn.Conv1d(hidden_channels, hidden_channels, kernel_size, padding=kernel_size//2)) self.relu_drop = nn.Sequential(nn.ReLU(), nn.Dropout(p_dropout))
self.norm_layers.append(LayerNorm(hidden_channels)) for _ in range(n_layers - 1):
self.proj = nn.Conv1d(hidden_channels, out_channels, 1) self.conv_layers.append(
self.proj.weight.data.zero_() nn.Conv1d(
self.proj.bias.data.zero_() hidden_channels,
hidden_channels,
kernel_size,
padding=kernel_size // 2,
)
)
self.norm_layers.append(LayerNorm(hidden_channels))
self.proj = nn.Conv1d(hidden_channels, out_channels, 1)
self.proj.weight.data.zero_()
self.proj.bias.data.zero_()
def forward(self, x, x_mask): def forward(self, x, x_mask):
x_org = x x_org = x
for i in range(self.n_layers): for i in range(self.n_layers):
x = self.conv_layers[i](x * x_mask) x = self.conv_layers[i](x * x_mask)
x = self.norm_layers[i](x) x = self.norm_layers[i](x)
x = self.relu_drop(x) x = self.relu_drop(x)
x = x_org + self.proj(x) x = x_org + self.proj(x)
return x * x_mask return x * x_mask
class DDSConv(nn.Module): class DDSConv(nn.Module):
""" """
Dialted and Depth-Separable Convolution Dialted and Depth-Separable Convolution
""" """
def __init__(self, channels, kernel_size, n_layers, p_dropout=0.):
super().__init__()
self.channels = channels
self.kernel_size = kernel_size
self.n_layers = n_layers
self.p_dropout = p_dropout
self.drop = nn.Dropout(p_dropout) def __init__(self, channels, kernel_size, n_layers, p_dropout=0.0):
self.convs_sep = nn.ModuleList() super().__init__()
self.convs_1x1 = nn.ModuleList() self.channels = channels
self.norms_1 = nn.ModuleList() self.kernel_size = kernel_size
self.norms_2 = nn.ModuleList() self.n_layers = n_layers
for i in range(n_layers): self.p_dropout = p_dropout
dilation = kernel_size ** i
padding = (kernel_size * dilation - dilation) // 2
self.convs_sep.append(nn.Conv1d(channels, channels, kernel_size,
groups=channels, dilation=dilation, padding=padding
))
self.convs_1x1.append(nn.Conv1d(channels, channels, 1))
self.norms_1.append(LayerNorm(channels))
self.norms_2.append(LayerNorm(channels))
def forward(self, x, x_mask, g=None): self.drop = nn.Dropout(p_dropout)
if g is not None: self.convs_sep = nn.ModuleList()
x = x + g self.convs_1x1 = nn.ModuleList()
for i in range(self.n_layers): self.norms_1 = nn.ModuleList()
y = self.convs_sep[i](x * x_mask) self.norms_2 = nn.ModuleList()
y = self.norms_1[i](y) for i in range(n_layers):
y = F.gelu(y) dilation = kernel_size**i
y = self.convs_1x1[i](y) padding = (kernel_size * dilation - dilation) // 2
y = self.norms_2[i](y) self.convs_sep.append(
y = F.gelu(y) nn.Conv1d(
y = self.drop(y) channels,
x = x + y channels,
return x * x_mask kernel_size,
groups=channels,
dilation=dilation,
padding=padding,
)
)
self.convs_1x1.append(nn.Conv1d(channels, channels, 1))
self.norms_1.append(LayerNorm(channels))
self.norms_2.append(LayerNorm(channels))
def forward(self, x, x_mask, g=None):
if g is not None:
x = x + g
for i in range(self.n_layers):
y = self.convs_sep[i](x * x_mask)
y = self.norms_1[i](y)
y = F.gelu(y)
y = self.convs_1x1[i](y)
y = self.norms_2[i](y)
y = F.gelu(y)
y = self.drop(y)
x = x + y
return x * x_mask
class WN(torch.nn.Module): class WN(torch.nn.Module):
def __init__(self, hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=0, p_dropout=0): def __init__(
super(WN, self).__init__() self,
assert(kernel_size % 2 == 1) hidden_channels,
self.hidden_channels =hidden_channels kernel_size,
self.kernel_size = kernel_size, dilation_rate,
self.dilation_rate = dilation_rate n_layers,
self.n_layers = n_layers gin_channels=0,
self.gin_channels = gin_channels p_dropout=0,
self.p_dropout = p_dropout ):
super(WN, self).__init__()
assert kernel_size % 2 == 1
self.hidden_channels = hidden_channels
self.kernel_size = (kernel_size,)
self.dilation_rate = dilation_rate
self.n_layers = n_layers
self.gin_channels = gin_channels
self.p_dropout = p_dropout
self.in_layers = torch.nn.ModuleList() self.in_layers = torch.nn.ModuleList()
self.res_skip_layers = torch.nn.ModuleList() self.res_skip_layers = torch.nn.ModuleList()
self.drop = nn.Dropout(p_dropout) self.drop = nn.Dropout(p_dropout)
if gin_channels != 0: if gin_channels != 0:
cond_layer = torch.nn.Conv1d(gin_channels, 2*hidden_channels*n_layers, 1) cond_layer = torch.nn.Conv1d(
self.cond_layer = torch.nn.utils.weight_norm(cond_layer, name='weight') gin_channels, 2 * hidden_channels * n_layers, 1
)
self.cond_layer = torch.nn.utils.weight_norm(cond_layer, name="weight")
for i in range(n_layers): for i in range(n_layers):
dilation = dilation_rate ** i dilation = dilation_rate**i
padding = int((kernel_size * dilation - dilation) / 2) padding = int((kernel_size * dilation - dilation) / 2)
in_layer = torch.nn.Conv1d(hidden_channels, 2*hidden_channels, kernel_size, in_layer = torch.nn.Conv1d(
dilation=dilation, padding=padding) hidden_channels,
in_layer = torch.nn.utils.weight_norm(in_layer, name='weight') 2 * hidden_channels,
self.in_layers.append(in_layer) kernel_size,
dilation=dilation,
padding=padding,
)
in_layer = torch.nn.utils.weight_norm(in_layer, name="weight")
self.in_layers.append(in_layer)
# last one is not necessary # last one is not necessary
if i < n_layers - 1: if i < n_layers - 1:
res_skip_channels = 2 * hidden_channels res_skip_channels = 2 * hidden_channels
else: else:
res_skip_channels = hidden_channels res_skip_channels = hidden_channels
res_skip_layer = torch.nn.Conv1d(hidden_channels, res_skip_channels, 1) res_skip_layer = torch.nn.Conv1d(hidden_channels, res_skip_channels, 1)
res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name='weight') res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name="weight")
self.res_skip_layers.append(res_skip_layer) self.res_skip_layers.append(res_skip_layer)
def forward(self, x, x_mask, g=None, **kwargs): def forward(self, x, x_mask, g=None, **kwargs):
output = torch.zeros_like(x) output = torch.zeros_like(x)
n_channels_tensor = torch.IntTensor([self.hidden_channels]) n_channels_tensor = torch.IntTensor([self.hidden_channels])
if g is not None: if g is not None:
g = self.cond_layer(g) g = self.cond_layer(g)
for i in range(self.n_layers): for i in range(self.n_layers):
x_in = self.in_layers[i](x) x_in = self.in_layers[i](x)
if g is not None: if g is not None:
cond_offset = i * 2 * self.hidden_channels cond_offset = i * 2 * self.hidden_channels
g_l = g[:,cond_offset:cond_offset+2*self.hidden_channels,:] g_l = g[:, cond_offset : cond_offset + 2 * self.hidden_channels, :]
else: else:
g_l = torch.zeros_like(x_in) g_l = torch.zeros_like(x_in)
acts = commons.fused_add_tanh_sigmoid_multiply( acts = commons.fused_add_tanh_sigmoid_multiply(x_in, g_l, n_channels_tensor)
x_in, acts = self.drop(acts)
g_l,
n_channels_tensor)
acts = self.drop(acts)
res_skip_acts = self.res_skip_layers[i](acts) res_skip_acts = self.res_skip_layers[i](acts)
if i < self.n_layers - 1: if i < self.n_layers - 1:
res_acts = res_skip_acts[:,:self.hidden_channels,:] res_acts = res_skip_acts[:, : self.hidden_channels, :]
x = (x + res_acts) * x_mask x = (x + res_acts) * x_mask
output = output + res_skip_acts[:,self.hidden_channels:,:] output = output + res_skip_acts[:, self.hidden_channels :, :]
else: else:
output = output + res_skip_acts output = output + res_skip_acts
return output * x_mask return output * x_mask
def remove_weight_norm(self): def remove_weight_norm(self):
if self.gin_channels != 0: if self.gin_channels != 0:
torch.nn.utils.remove_weight_norm(self.cond_layer) torch.nn.utils.remove_weight_norm(self.cond_layer)
for l in self.in_layers: for l in self.in_layers:
torch.nn.utils.remove_weight_norm(l) torch.nn.utils.remove_weight_norm(l)
for l in self.res_skip_layers: for l in self.res_skip_layers:
torch.nn.utils.remove_weight_norm(l) torch.nn.utils.remove_weight_norm(l)
class ResBlock1(torch.nn.Module): class ResBlock1(torch.nn.Module):
def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5)): def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5)):
super(ResBlock1, self).__init__() super(ResBlock1, self).__init__()
self.convs1 = nn.ModuleList([ self.convs1 = nn.ModuleList(
weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0], [
padding=get_padding(kernel_size, dilation[0]))), weight_norm(
weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1], Conv1d(
padding=get_padding(kernel_size, dilation[1]))), channels,
weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[2], channels,
padding=get_padding(kernel_size, dilation[2]))) kernel_size,
]) 1,
dilation=dilation[0],
padding=get_padding(kernel_size, dilation[0]),
)
),
weight_norm(
Conv1d(
channels,
channels,
kernel_size,
1,
dilation=dilation[1],
padding=get_padding(kernel_size, dilation[1]),
)
),
weight_norm(
Conv1d(
channels,
channels,
kernel_size,
1,
dilation=dilation[2],
padding=get_padding(kernel_size, dilation[2]),
)
),
]
)
self.convs1.apply(init_weights) self.convs1.apply(init_weights)
self.convs2 = nn.ModuleList([ self.convs2 = nn.ModuleList(
weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1, [
padding=get_padding(kernel_size, 1))), weight_norm(
weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1, Conv1d(
padding=get_padding(kernel_size, 1))), channels,
weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1, channels,
padding=get_padding(kernel_size, 1))) kernel_size,
]) 1,
dilation=1,
padding=get_padding(kernel_size, 1),
)
),
weight_norm(
Conv1d(
channels,
channels,
kernel_size,
1,
dilation=1,
padding=get_padding(kernel_size, 1),
)
),
weight_norm(
Conv1d(
channels,
channels,
kernel_size,
1,
dilation=1,
padding=get_padding(kernel_size, 1),
)
),
]
)
self.convs2.apply(init_weights) self.convs2.apply(init_weights)
def forward(self, x, x_mask=None): def forward(self, x, x_mask=None):
@@ -230,12 +321,30 @@ class ResBlock1(torch.nn.Module):
class ResBlock2(torch.nn.Module): class ResBlock2(torch.nn.Module):
def __init__(self, channels, kernel_size=3, dilation=(1, 3)): def __init__(self, channels, kernel_size=3, dilation=(1, 3)):
super(ResBlock2, self).__init__() super(ResBlock2, self).__init__()
self.convs = nn.ModuleList([ self.convs = nn.ModuleList(
weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0], [
padding=get_padding(kernel_size, dilation[0]))), weight_norm(
weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1], Conv1d(
padding=get_padding(kernel_size, dilation[1]))) channels,
]) channels,
kernel_size,
1,
dilation=dilation[0],
padding=get_padding(kernel_size, dilation[0]),
)
),
weight_norm(
Conv1d(
channels,
channels,
kernel_size,
1,
dilation=dilation[1],
padding=get_padding(kernel_size, dilation[1]),
)
),
]
)
self.convs.apply(init_weights) self.convs.apply(init_weights)
def forward(self, x, x_mask=None): def forward(self, x, x_mask=None):
@@ -255,198 +364,237 @@ class ResBlock2(torch.nn.Module):
class Log(nn.Module): class Log(nn.Module):
def forward(self, x, x_mask, reverse=False, **kwargs): def forward(self, x, x_mask, reverse=False, **kwargs):
if not reverse: if not reverse:
y = torch.log(torch.clamp_min(x, 1e-5)) * x_mask y = torch.log(torch.clamp_min(x, 1e-5)) * x_mask
logdet = torch.sum(-y, [1, 2]) logdet = torch.sum(-y, [1, 2])
return y, logdet return y, logdet
else: else:
x = torch.exp(x) * x_mask x = torch.exp(x) * x_mask
return x return x
class Flip(nn.Module): class Flip(nn.Module):
def forward(self, x, *args, reverse=False, **kwargs): def forward(self, x, *args, reverse=False, **kwargs):
x = torch.flip(x, [1]) x = torch.flip(x, [1])
if not reverse: if not reverse:
logdet = torch.zeros(x.size(0)).to(dtype=x.dtype, device=x.device) logdet = torch.zeros(x.size(0)).to(dtype=x.dtype, device=x.device)
return x, logdet return x, logdet
else: else:
return x return x
class ElementwiseAffine(nn.Module): class ElementwiseAffine(nn.Module):
def __init__(self, channels): def __init__(self, channels):
super().__init__() super().__init__()
self.channels = channels self.channels = channels
self.m = nn.Parameter(torch.zeros(channels,1)) self.m = nn.Parameter(torch.zeros(channels, 1))
self.logs = nn.Parameter(torch.zeros(channels,1)) self.logs = nn.Parameter(torch.zeros(channels, 1))
def forward(self, x, x_mask, reverse=False, **kwargs): def forward(self, x, x_mask, reverse=False, **kwargs):
if not reverse: if not reverse:
y = self.m + torch.exp(self.logs) * x y = self.m + torch.exp(self.logs) * x
y = y * x_mask y = y * x_mask
logdet = torch.sum(self.logs * x_mask, [1,2]) logdet = torch.sum(self.logs * x_mask, [1, 2])
return y, logdet return y, logdet
else: else:
x = (x - self.m) * torch.exp(-self.logs) * x_mask x = (x - self.m) * torch.exp(-self.logs) * x_mask
return x return x
class ResidualCouplingLayer(nn.Module): class ResidualCouplingLayer(nn.Module):
def __init__(self, def __init__(
channels, self,
hidden_channels, channels,
kernel_size, hidden_channels,
dilation_rate, kernel_size,
n_layers, dilation_rate,
p_dropout=0, n_layers,
gin_channels=0, p_dropout=0,
mean_only=False): gin_channels=0,
assert channels % 2 == 0, "channels should be divisible by 2" mean_only=False,
super().__init__() ):
self.channels = channels assert channels % 2 == 0, "channels should be divisible by 2"
self.hidden_channels = hidden_channels super().__init__()
self.kernel_size = kernel_size self.channels = channels
self.dilation_rate = dilation_rate self.hidden_channels = hidden_channels
self.n_layers = n_layers self.kernel_size = kernel_size
self.half_channels = channels // 2 self.dilation_rate = dilation_rate
self.mean_only = mean_only self.n_layers = n_layers
self.half_channels = channels // 2
self.mean_only = mean_only
self.pre = nn.Conv1d(self.half_channels, hidden_channels, 1) self.pre = nn.Conv1d(self.half_channels, hidden_channels, 1)
self.enc = WN(hidden_channels, kernel_size, dilation_rate, n_layers, p_dropout=p_dropout, gin_channels=gin_channels) self.enc = WN(
self.post = nn.Conv1d(hidden_channels, self.half_channels * (2 - mean_only), 1) hidden_channels,
self.post.weight.data.zero_() kernel_size,
self.post.bias.data.zero_() dilation_rate,
n_layers,
p_dropout=p_dropout,
gin_channels=gin_channels,
)
self.post = nn.Conv1d(hidden_channels, self.half_channels * (2 - mean_only), 1)
self.post.weight.data.zero_()
self.post.bias.data.zero_()
def forward(self, x, x_mask, g=None, reverse=False): def forward(self, x, x_mask, g=None, reverse=False):
x0, x1 = torch.split(x, [self.half_channels]*2, 1) x0, x1 = torch.split(x, [self.half_channels] * 2, 1)
h = self.pre(x0) * x_mask h = self.pre(x0) * x_mask
h = self.enc(h, x_mask, g=g) h = self.enc(h, x_mask, g=g)
stats = self.post(h) * x_mask stats = self.post(h) * x_mask
if not self.mean_only: if not self.mean_only:
m, logs = torch.split(stats, [self.half_channels]*2, 1) m, logs = torch.split(stats, [self.half_channels] * 2, 1)
else: else:
m = stats m = stats
logs = torch.zeros_like(m) logs = torch.zeros_like(m)
if not reverse: if not reverse:
x1 = m + x1 * torch.exp(logs) * x_mask x1 = m + x1 * torch.exp(logs) * x_mask
x = torch.cat([x0, x1], 1) x = torch.cat([x0, x1], 1)
logdet = torch.sum(logs, [1,2]) logdet = torch.sum(logs, [1, 2])
return x, logdet return x, logdet
else: else:
x1 = (x1 - m) * torch.exp(-logs) * x_mask x1 = (x1 - m) * torch.exp(-logs) * x_mask
x = torch.cat([x0, x1], 1) x = torch.cat([x0, x1], 1)
return x return x
class ConvFlow(nn.Module): class ConvFlow(nn.Module):
def __init__(self, in_channels, filter_channels, kernel_size, n_layers, num_bins=10, tail_bound=5.0): def __init__(
super().__init__() self,
self.in_channels = in_channels in_channels,
self.filter_channels = filter_channels filter_channels,
self.kernel_size = kernel_size kernel_size,
self.n_layers = n_layers n_layers,
self.num_bins = num_bins num_bins=10,
self.tail_bound = tail_bound tail_bound=5.0,
self.half_channels = in_channels // 2 ):
super().__init__()
self.in_channels = in_channels
self.filter_channels = filter_channels
self.kernel_size = kernel_size
self.n_layers = n_layers
self.num_bins = num_bins
self.tail_bound = tail_bound
self.half_channels = in_channels // 2
self.pre = nn.Conv1d(self.half_channels, filter_channels, 1) self.pre = nn.Conv1d(self.half_channels, filter_channels, 1)
self.convs = DDSConv(filter_channels, kernel_size, n_layers, p_dropout=0.) self.convs = DDSConv(filter_channels, kernel_size, n_layers, p_dropout=0.0)
self.proj = nn.Conv1d(filter_channels, self.half_channels * (num_bins * 3 - 1), 1) self.proj = nn.Conv1d(
self.proj.weight.data.zero_() filter_channels, self.half_channels * (num_bins * 3 - 1), 1
self.proj.bias.data.zero_() )
self.proj.weight.data.zero_()
self.proj.bias.data.zero_()
def forward(self, x, x_mask, g=None, reverse=False): def forward(self, x, x_mask, g=None, reverse=False):
x0, x1 = torch.split(x, [self.half_channels]*2, 1) x0, x1 = torch.split(x, [self.half_channels] * 2, 1)
h = self.pre(x0) h = self.pre(x0)
h = self.convs(h, x_mask, g=g) h = self.convs(h, x_mask, g=g)
h = self.proj(h) * x_mask h = self.proj(h) * x_mask
b, c, t = x0.shape b, c, t = x0.shape
h = h.reshape(b, c, -1, t).permute(0, 1, 3, 2) # [b, cx?, t] -> [b, c, t, ?] h = h.reshape(b, c, -1, t).permute(0, 1, 3, 2) # [b, cx?, t] -> [b, c, t, ?]
unnormalized_widths = h[..., :self.num_bins] / math.sqrt(self.filter_channels) unnormalized_widths = h[..., : self.num_bins] / math.sqrt(self.filter_channels)
unnormalized_heights = h[..., self.num_bins:2*self.num_bins] / math.sqrt(self.filter_channels) unnormalized_heights = h[..., self.num_bins : 2 * self.num_bins] / math.sqrt(
unnormalized_derivatives = h[..., 2 * self.num_bins:] self.filter_channels
)
unnormalized_derivatives = h[..., 2 * self.num_bins :]
x1, logabsdet = piecewise_rational_quadratic_transform(
x1,
unnormalized_widths,
unnormalized_heights,
unnormalized_derivatives,
inverse=reverse,
tails="linear",
tail_bound=self.tail_bound,
)
x = torch.cat([x0, x1], 1) * x_mask
logdet = torch.sum(logabsdet * x_mask, [1, 2])
if not reverse:
return x, logdet
else:
return x
x1, logabsdet = piecewise_rational_quadratic_transform(x1,
unnormalized_widths,
unnormalized_heights,
unnormalized_derivatives,
inverse=reverse,
tails='linear',
tail_bound=self.tail_bound
)
x = torch.cat([x0, x1], 1) * x_mask
logdet = torch.sum(logabsdet * x_mask, [1,2])
if not reverse:
return x, logdet
else:
return x
class TransformerCouplingLayer(nn.Module): class TransformerCouplingLayer(nn.Module):
def __init__(self, def __init__(
channels, self,
hidden_channels, channels,
kernel_size, hidden_channels,
n_layers, kernel_size,
n_heads, n_layers,
p_dropout=0, n_heads,
filter_channels=0, p_dropout=0,
mean_only=False, filter_channels=0,
wn_sharing_parameter=None, mean_only=False,
gin_channels = 0 wn_sharing_parameter=None,
): gin_channels=0,
assert channels % 2 == 0, "channels should be divisible by 2" ):
super().__init__() assert channels % 2 == 0, "channels should be divisible by 2"
self.channels = channels super().__init__()
self.hidden_channels = hidden_channels self.channels = channels
self.kernel_size = kernel_size self.hidden_channels = hidden_channels
self.n_layers = n_layers self.kernel_size = kernel_size
self.half_channels = channels // 2 self.n_layers = n_layers
self.mean_only = mean_only self.half_channels = channels // 2
self.mean_only = mean_only
self.pre = nn.Conv1d(self.half_channels, hidden_channels, 1) self.pre = nn.Conv1d(self.half_channels, hidden_channels, 1)
self.enc = Encoder(hidden_channels, filter_channels, n_heads, n_layers, kernel_size, p_dropout, isflow = True, gin_channels = gin_channels) if wn_sharing_parameter is None else wn_sharing_parameter self.enc = (
self.post = nn.Conv1d(hidden_channels, self.half_channels * (2 - mean_only), 1) Encoder(
self.post.weight.data.zero_() hidden_channels,
self.post.bias.data.zero_() filter_channels,
n_heads,
n_layers,
kernel_size,
p_dropout,
isflow=True,
gin_channels=gin_channels,
)
if wn_sharing_parameter is None
else wn_sharing_parameter
)
self.post = nn.Conv1d(hidden_channels, self.half_channels * (2 - mean_only), 1)
self.post.weight.data.zero_()
self.post.bias.data.zero_()
def forward(self, x, x_mask, g=None, reverse=False): def forward(self, x, x_mask, g=None, reverse=False):
x0, x1 = torch.split(x, [self.half_channels]*2, 1) x0, x1 = torch.split(x, [self.half_channels] * 2, 1)
h = self.pre(x0) * x_mask h = self.pre(x0) * x_mask
h = self.enc(h, x_mask, g=g) h = self.enc(h, x_mask, g=g)
stats = self.post(h) * x_mask stats = self.post(h) * x_mask
if not self.mean_only: if not self.mean_only:
m, logs = torch.split(stats, [self.half_channels]*2, 1) m, logs = torch.split(stats, [self.half_channels] * 2, 1)
else: else:
m = stats m = stats
logs = torch.zeros_like(m) logs = torch.zeros_like(m)
if not reverse: if not reverse:
x1 = m + x1 * torch.exp(logs) * x_mask x1 = m + x1 * torch.exp(logs) * x_mask
x = torch.cat([x0, x1], 1) x = torch.cat([x0, x1], 1)
logdet = torch.sum(logs, [1,2]) logdet = torch.sum(logs, [1, 2])
return x, logdet return x, logdet
else: else:
x1 = (x1 - m) * torch.exp(-logs) * x_mask x1 = (x1 - m) * torch.exp(-logs) * x_mask
x = torch.cat([x0, x1], 1) x = torch.cat([x0, x1], 1)
return x return x
x1, logabsdet = piecewise_rational_quadratic_transform(x1, x1, logabsdet = piecewise_rational_quadratic_transform(
unnormalized_widths, x1,
unnormalized_heights, unnormalized_widths,
unnormalized_derivatives, unnormalized_heights,
inverse=reverse, unnormalized_derivatives,
tails='linear', inverse=reverse,
tail_bound=self.tail_bound tails="linear",
) tail_bound=self.tail_bound,
)
x = torch.cat([x0, x1], 1) * x_mask x = torch.cat([x0, x1], 1) * x_mask
logdet = torch.sum(logabsdet * x_mask, [1,2]) logdet = torch.sum(logabsdet * x_mask, [1, 2])
if not reverse: if not reverse:
return x, logdet return x, logdet
else: else:
return x return x

19
monotonic_align/__init__.py
View File

@@ -3,13 +3,14 @@ from torch import from_numpy
from .core import maximum_path_jit from .core import maximum_path_jit
def maximum_path(neg_cent, mask):
device = neg_cent.device
dtype = neg_cent.dtype
neg_cent = neg_cent.data.cpu().numpy().astype(float32)
path = zeros(neg_cent.shape, dtype=int32)
t_t_max = mask.sum(1)[:, 0].data.cpu().numpy().astype(int32) def maximum_path(neg_cent, mask):
t_s_max = mask.sum(2)[:, 0].data.cpu().numpy().astype(int32) device = neg_cent.device
maximum_path_jit(path, neg_cent, t_t_max, t_s_max) dtype = neg_cent.dtype
return from_numpy(path).to(device=device, dtype=dtype) neg_cent = neg_cent.data.cpu().numpy().astype(float32)
path = zeros(neg_cent.shape, dtype=int32)
t_t_max = mask.sum(1)[:, 0].data.cpu().numpy().astype(int32)
t_s_max = mask.sum(2)[:, 0].data.cpu().numpy().astype(int32)
maximum_path_jit(path, neg_cent, t_t_max, t_s_max)
return from_numpy(path).to(device=device, dtype=dtype)

67
monotonic_align/core.py
View File

@@ -1,35 +1,46 @@
import numba import numba
@numba.jit(numba.void(numba.int32[:,:,::1], numba.float32[:,:,::1], numba.int32[::1], numba.int32[::1]), nopython=True, nogil=True) @numba.jit(
numba.void(
numba.int32[:, :, ::1],
numba.float32[:, :, ::1],
numba.int32[::1],
numba.int32[::1],
),
nopython=True,
nogil=True,
)
def maximum_path_jit(paths, values, t_ys, t_xs): def maximum_path_jit(paths, values, t_ys, t_xs):
b = paths.shape[0] b = paths.shape[0]
max_neg_val=-1e9 max_neg_val = -1e9
for i in range(int(b)): for i in range(int(b)):
path = paths[i] path = paths[i]
value = values[i] value = values[i]
t_y = t_ys[i] t_y = t_ys[i]
t_x = t_xs[i] t_x = t_xs[i]
v_prev = v_cur = 0.0 v_prev = v_cur = 0.0
index = t_x - 1 index = t_x - 1
for y in range(t_y): for y in range(t_y):
for x in range(max(0, t_x + y - t_y), min(t_x, y + 1)): for x in range(max(0, t_x + y - t_y), min(t_x, y + 1)):
if x == y: if x == y:
v_cur = max_neg_val v_cur = max_neg_val
else: else:
v_cur = value[y-1, x] v_cur = value[y - 1, x]
if x == 0: if x == 0:
if y == 0: if y == 0:
v_prev = 0. v_prev = 0.0
else: else:
v_prev = max_neg_val v_prev = max_neg_val
else: else:
v_prev = value[y-1, x-1] v_prev = value[y - 1, x - 1]
value[y, x] += max(v_prev, v_cur) value[y, x] += max(v_prev, v_cur)
for y in range(t_y - 1, -1, -1): for y in range(t_y - 1, -1, -1):
path[y, index] = 1 path[y, index] = 1
if index != 0 and (index == y or value[y-1, index] < value[y-1, index-1]): if index != 0 and (
index = index - 1 index == y or value[y - 1, index] < value[y - 1, index - 1]
):
index = index - 1

1
preprocess_text.py
View File

@@ -35,7 +35,6 @@ def main(
max_val_total: int, max_val_total: int,
clean: bool, clean: bool,
): ):
if cleaned_path is None: if cleaned_path is None:
cleaned_path = transcription_path + ".cleaned" cleaned_path = transcription_path + ".cleaned"

30
resample.py
View File

@@ -13,30 +13,38 @@ def process(item):
spkdir, wav_name, args = item spkdir, wav_name, args = item
speaker = spkdir.replace("\\", "/").split("/")[-1] speaker = spkdir.replace("\\", "/").split("/")[-1]
wav_path = os.path.join(args.in_dir, speaker, wav_name) wav_path = os.path.join(args.in_dir, speaker, wav_name)
if os.path.exists(wav_path) and '.wav' in wav_path: if os.path.exists(wav_path) and ".wav" in wav_path:
os.makedirs(os.path.join(args.out_dir, speaker), exist_ok=True) os.makedirs(os.path.join(args.out_dir, speaker), exist_ok=True)
wav, sr = librosa.load(wav_path, sr=args.sr) wav, sr = librosa.load(wav_path, sr=args.sr)
soundfile.write( soundfile.write(os.path.join(args.out_dir, speaker, wav_name), wav, sr)
os.path.join(args.out_dir, speaker, wav_name),
wav,
sr
)
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument("--sr", type=int, default=44100, help="sampling rate") parser.add_argument("--sr", type=int, default=44100, help="sampling rate")
parser.add_argument("--in_dir", type=str, default="./raw", help="path to source dir") parser.add_argument(
parser.add_argument("--out_dir", type=str, default="./dataset", help="path to target dir") "--in_dir", type=str, default="./raw", help="path to source dir"
)
parser.add_argument(
"--out_dir", type=str, default="./dataset", help="path to target dir"
)
args = parser.parse_args() args = parser.parse_args()
# processs = 8 # processs = 8
processs = cpu_count()-2 if cpu_count() >4 else 1 processs = cpu_count() - 2 if cpu_count() > 4 else 1
pool = Pool(processes=processs) pool = Pool(processes=processs)
for speaker in os.listdir(args.in_dir): for speaker in os.listdir(args.in_dir):
spk_dir = os.path.join(args.in_dir, speaker) spk_dir = os.path.join(args.in_dir, speaker)
if os.path.isdir(spk_dir): if os.path.isdir(spk_dir):
print(spk_dir) print(spk_dir)
for _ in tqdm(pool.imap_unordered(process, [(spk_dir, i, args) for i in os.listdir(spk_dir) if i.endswith("wav")])): for _ in tqdm(
pool.imap_unordered(
process,
[
(spk_dir, i, args)
for i in os.listdir(spk_dir)
if i.endswith("wav")
],
)
):
pass pass

93
server.py
View File

@@ -13,7 +13,8 @@ from scipy.io import wavfile
# Flask Init # Flask Init
app = Flask(__name__) app = Flask(__name__)
app.config['JSON_AS_ASCII'] = False app.config["JSON_AS_ASCII"] = False
def get_text(text, language_str, hps): def get_text(text, language_str, hps):
norm_text, phone, tone, word2ph = clean_text(text, language_str) norm_text, phone, tone, word2ph = clean_text(text, language_str)
@@ -30,84 +31,121 @@ def get_text(text, language_str, hps):
del word2ph del word2ph
assert bert.shape[-1] == len(phone), phone assert bert.shape[-1] == len(phone), phone
if language_str=='ZH': if language_str == "ZH":
bert = bert bert = bert
ja_bert = torch.zeros(768, len(phone)) ja_bert = torch.zeros(768, len(phone))
elif language_str=="JA": elif language_str == "JA":
ja_bert = bert ja_bert = bert
bert = torch.zeros(1024, len(phone)) bert = torch.zeros(1024, len(phone))
else: else:
bert = torch.zeros(1024, len(phone)) bert = torch.zeros(1024, len(phone))
ja_bert = torch.zeros(768, len(phone)) ja_bert = torch.zeros(768, len(phone))
assert bert.shape[-1] == len(phone), ( assert bert.shape[-1] == len(phone), (
bert.shape, len(phone), sum(word2ph), p1, p2, t1, t2, pold, pold2, word2ph, text, w2pho) bert.shape,
len(phone),
sum(word2ph),
p1,
p2,
t1,
t2,
pold,
pold2,
word2ph,
text,
w2pho,
)
phone = torch.LongTensor(phone) phone = torch.LongTensor(phone)
tone = torch.LongTensor(tone) tone = torch.LongTensor(tone)
language = torch.LongTensor(language) language = torch.LongTensor(language)
return bert, ja_bert, phone, tone, language return bert, ja_bert, phone, tone, language
def infer(text, sdp_ratio, noise_scale, noise_scale_w, length_scale, sid, language): def infer(text, sdp_ratio, noise_scale, noise_scale_w, length_scale, sid, language):
bert, ja_bert, phones, tones, lang_ids = get_text(text, language, hps) bert, ja_bert, phones, tones, lang_ids = get_text(text, language, hps)
with torch.no_grad(): with torch.no_grad():
x_tst=phones.to(dev).unsqueeze(0) x_tst = phones.to(dev).unsqueeze(0)
tones=tones.to(dev).unsqueeze(0) tones = tones.to(dev).unsqueeze(0)
lang_ids=lang_ids.to(dev).unsqueeze(0) lang_ids = lang_ids.to(dev).unsqueeze(0)
bert = bert.to(dev).unsqueeze(0) bert = bert.to(dev).unsqueeze(0)
ja_bert = ja_bert.to(device).unsqueeze(0) ja_bert = ja_bert.to(device).unsqueeze(0)
x_tst_lengths = torch.LongTensor([phones.size(0)]).to(dev) x_tst_lengths = torch.LongTensor([phones.size(0)]).to(dev)
speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(dev) speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(dev)
audio = net_g.infer(x_tst, x_tst_lengths, speakers, tones, lang_ids, bert, ja_bert, sdp_ratio=sdp_ratio audio = (
, noise_scale=noise_scale, noise_scale_w=noise_scale_w, length_scale=length_scale)[0][0,0].data.cpu().float().numpy() net_g.infer(
x_tst,
x_tst_lengths,
speakers,
tones,
lang_ids,
bert,
ja_bert,
sdp_ratio=sdp_ratio,
noise_scale=noise_scale,
noise_scale_w=noise_scale_w,
length_scale=length_scale,
)[0][0, 0]
.data.cpu()
.float()
.numpy()
)
return audio return audio
def replace_punctuation(text, i=2): def replace_punctuation(text, i=2):
punctuation = "，。？！" punctuation = "，。？！"
for char in punctuation: for char in punctuation:
text = text.replace(char, char * i) text = text.replace(char, char * i)
return text return text
def wav2(i, o, format): def wav2(i, o, format):
inp = avopen(i, 'rb') inp = avopen(i, "rb")
out = avopen(o, 'wb', format=format) out = avopen(o, "wb", format=format)
if format == "ogg": format = "libvorbis" if format == "ogg":
format = "libvorbis"
ostream = out.add_stream(format) ostream = out.add_stream(format)
for frame in inp.decode(audio=0): for frame in inp.decode(audio=0):
for p in ostream.encode(frame): out.mux(p) for p in ostream.encode(frame):
out.mux(p)
for p in ostream.encode(None): out.mux(p) for p in ostream.encode(None):
out.mux(p)
out.close() out.close()
inp.close() inp.close()
# Load Generator # Load Generator
hps = utils.get_hparams_from_file("./configs/config.json") hps = utils.get_hparams_from_file("./configs/config.json")
dev='cuda' dev = "cuda"
net_g = SynthesizerTrn( net_g = SynthesizerTrn(
len(symbols), len(symbols),
hps.data.filter_length // 2 + 1, hps.data.filter_length // 2 + 1,
hps.train.segment_size // hps.data.hop_length, hps.train.segment_size // hps.data.hop_length,
n_speakers=hps.data.n_speakers, n_speakers=hps.data.n_speakers,
**hps.model).to(dev) **hps.model
).to(dev)
_ = net_g.eval() _ = net_g.eval()
_ = utils.load_checkpoint("logs/G_649000.pth", net_g, None,skip_optimizer=True) _ = utils.load_checkpoint("logs/G_649000.pth", net_g, None, skip_optimizer=True)
@app.route("/") @app.route("/")
def main(): def main():
try: try:
speaker = request.args.get('speaker') speaker = request.args.get("speaker")
text = request.args.get('text').replace("/n","") text = request.args.get("text").replace("/n", "")
sdp_ratio = float(request.args.get("sdp_ratio", 0.2)) sdp_ratio = float(request.args.get("sdp_ratio", 0.2))
noise = float(request.args.get("noise", 0.5)) noise = float(request.args.get("noise", 0.5))
noisew = float(request.args.get("noisew", 0.6)) noisew = float(request.args.get("noisew", 0.6))
length = float(request.args.get("length", 1.2)) length = float(request.args.get("length", 1.2))
language = request.args.get('language') language = request.args.get("language")
if length >= 2: if length >= 2:
return "Too big length" return "Too big length"
if len(text) >=250: if len(text) >= 250:
return "Too long text" return "Too long text"
fmt = request.args.get("format", "wav") fmt = request.args.get("format", "wav")
if None in (speaker, text): if None in (speaker, text):
@@ -120,7 +158,15 @@ def main():
return "Invalid Parameter" return "Invalid Parameter"
with torch.no_grad(): with torch.no_grad():
audio = infer(text, sdp_ratio=sdp_ratio, noise_scale=noise, noise_scale_w=noisew, length_scale=length, sid=speaker,language = language) audio = infer(
text,
sdp_ratio=sdp_ratio,
noise_scale=noise,
noise_scale_w=noisew,
length_scale=length,
sid=speaker,
language=language,
)
with BytesIO() as wav: with BytesIO() as wav:
wavfile.write(wav, hps.data.sampling_rate, audio) wavfile.write(wav, hps.data.sampling_rate, audio)
@@ -131,6 +177,5 @@ def main():
with BytesIO() as ofp: with BytesIO() as ofp:
wav2(wav, ofp, fmt) wav2(wav, ofp, fmt)
return Response( return Response(
ofp.getvalue(), ofp.getvalue(), mimetype="audio/mpeg" if fmt == "mp3" else "audio/ogg"
mimetype="audio/mpeg" if fmt == "mp3" else "audio/ogg"
) )

35
text/__init__.py
View File

@@ -3,28 +3,27 @@ from text.symbols import *
_symbol_to_id = {s: i for i, s in enumerate(symbols)} _symbol_to_id = {s: i for i, s in enumerate(symbols)}
def cleaned_text_to_sequence(cleaned_text, tones, language): def cleaned_text_to_sequence(cleaned_text, tones, language):
'''Converts a string of text to a sequence of IDs corresponding to the symbols in the text. """Converts a string of text to a sequence of IDs corresponding to the symbols in the text.
Args: Args:
text: string to convert to a sequence text: string to convert to a sequence
Returns: Returns:
List of integers corresponding to the symbols in the text List of integers corresponding to the symbols in the text
''' """
phones = [_symbol_to_id[symbol] for symbol in cleaned_text] phones = [_symbol_to_id[symbol] for symbol in cleaned_text]
tone_start = language_tone_start_map[language] tone_start = language_tone_start_map[language]
tones = [i + tone_start for i in tones] tones = [i + tone_start for i in tones]
lang_id = language_id_map[language] lang_id = language_id_map[language]
lang_ids = [lang_id for i in phones] lang_ids = [lang_id for i in phones]
return phones, tones, lang_ids return phones, tones, lang_ids
def get_bert(norm_text, word2ph, language, device): def get_bert(norm_text, word2ph, language, device):
from .chinese_bert import get_bert_feature as zh_bert from .chinese_bert import get_bert_feature as zh_bert
from .english_bert_mock import get_bert_feature as en_bert from .english_bert_mock import get_bert_feature as en_bert
from .japanese_bert import get_bert_feature as jp_bert from .japanese_bert import get_bert_feature as jp_bert
lang_bert_func_map = {
'ZH': zh_bert, lang_bert_func_map = {"ZH": zh_bert, "EN": en_bert, "JP": jp_bert}
'EN': en_bert, bert = lang_bert_func_map[language](norm_text, word2ph, device)
'JP': jp_bert return bert
}
bert = lang_bert_func_map[language](norm_text, word2ph, device)
return bert

143
text/chinese.py
View File

@@ -9,65 +9,70 @@ from text.symbols import punctuation
from text.tone_sandhi import ToneSandhi from text.tone_sandhi import ToneSandhi
current_file_path = os.path.dirname(__file__) current_file_path = os.path.dirname(__file__)
pinyin_to_symbol_map = {line.split("\t")[0]: line.strip().split("\t")[1] for line in pinyin_to_symbol_map = {
open(os.path.join(current_file_path, 'opencpop-strict.txt')).readlines()} line.split("\t")[0]: line.strip().split("\t")[1]
for line in open(os.path.join(current_file_path, "opencpop-strict.txt")).readlines()
}
import jieba.posseg as psg import jieba.posseg as psg
rep_map = { rep_map = {
'：': ',', "：": ",",
'；': ',', "；": ",",
'，': ',', "，": ",",
'。': '.', "。": ".",
'！': '!', "！": "!",
'？': '?', "？": "?",
'\n': '.', "\n": ".",
"·": ",", "·": ",",
'、': ",", "、": ",",
'...': '…', "...": "…",
'$': '.', "$": ".",
'“': "'", "“": "'",
'”': "'", "”": "'",
'‘': "'", "‘": "'",
'’': "'", "’": "'",
'（': "'", "（": "'",
'）': "'", "）": "'",
'(': "'", "(": "'",
')': "'", ")": "'",
'《': "'", "《": "'",
'》': "'", "》": "'",
'【': "'", "【": "'",
'】': "'", "】": "'",
'[': "'", "[": "'",
']': "'", "]": "'",
'—': "-", "—": "-",
'～': "-", "～": "-",
'~': "-", "~": "-",
'「': "'", "「": "'",
'」': "'", "」": "'",
} }
tone_modifier = ToneSandhi() tone_modifier = ToneSandhi()
def replace_punctuation(text): def replace_punctuation(text):
text = text.replace("嗯", "恩").replace("呣","母") text = text.replace("嗯", "恩").replace("呣", "母")
pattern = re.compile('|'.join(re.escape(p) for p in rep_map.keys())) pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys()))
replaced_text = pattern.sub(lambda x: rep_map[x.group()], text) replaced_text = pattern.sub(lambda x: rep_map[x.group()], text)
replaced_text = re.sub(r'[^\u4e00-\u9fa5'+"".join(punctuation)+r']+', '', replaced_text) replaced_text = re.sub(
r"[^\u4e00-\u9fa5" + "".join(punctuation) + r"]+", "", replaced_text
)
return replaced_text return replaced_text
def g2p(text): def g2p(text):
pattern = r'(?<=[{0}])\s*'.format(''.join(punctuation)) pattern = r"(?<=[{0}])\s*".format("".join(punctuation))
sentences = [i for i in re.split(pattern, text) if i.strip()!=''] sentences = [i for i in re.split(pattern, text) if i.strip() != ""]
phones, tones, word2ph = _g2p(sentences) phones, tones, word2ph = _g2p(sentences)
assert sum(word2ph) == len(phones) assert sum(word2ph) == len(phones)
assert len(word2ph) == len(text) #Sometimes it will crash,you can add a try-catch. assert len(word2ph) == len(text) # Sometimes it will crash,you can add a try-catch.
phones = ['_'] + phones + ["_"] phones = ["_"] + phones + ["_"]
tones = [0] + tones + [0] tones = [0] + tones + [0]
word2ph = [1] + word2ph + [1] word2ph = [1] + word2ph + [1]
return phones, tones, word2ph return phones, tones, word2ph
@@ -76,10 +81,10 @@ def g2p(text):
def _get_initials_finals(word): def _get_initials_finals(word):
initials = [] initials = []
finals = [] finals = []
orig_initials = lazy_pinyin( orig_initials = lazy_pinyin(word, neutral_tone_with_five=True, style=Style.INITIALS)
word, neutral_tone_with_five=True, style=Style.INITIALS)
orig_finals = lazy_pinyin( orig_finals = lazy_pinyin(
word, neutral_tone_with_five=True, style=Style.FINALS_TONE3) word, neutral_tone_with_five=True, style=Style.FINALS_TONE3
)
for c, v in zip(orig_initials, orig_finals): for c, v in zip(orig_initials, orig_finals):
initials.append(c) initials.append(c)
finals.append(v) finals.append(v)
@@ -93,17 +98,16 @@ def _g2p(segments):
for seg in segments: for seg in segments:
pinyins = [] pinyins = []
# Replace all English words in the sentence # Replace all English words in the sentence
seg = re.sub('[a-zA-Z]+', '', seg) seg = re.sub("[a-zA-Z]+", "", seg)
seg_cut = psg.lcut(seg) seg_cut = psg.lcut(seg)
initials = [] initials = []
finals = [] finals = []
seg_cut = tone_modifier.pre_merge_for_modify(seg_cut) seg_cut = tone_modifier.pre_merge_for_modify(seg_cut)
for word, pos in seg_cut: for word, pos in seg_cut:
if pos == 'eng': if pos == "eng":
continue continue
sub_initials, sub_finals = _get_initials_finals(word) sub_initials, sub_finals = _get_initials_finals(word)
sub_finals = tone_modifier.modified_tone(word, pos, sub_finals = tone_modifier.modified_tone(word, pos, sub_finals)
sub_finals)
initials.append(sub_initials) initials.append(sub_initials)
finals.append(sub_finals) finals.append(sub_finals)
@@ -112,52 +116,52 @@ def _g2p(segments):
finals = sum(finals, []) finals = sum(finals, [])
# #
for c, v in zip(initials, finals): for c, v in zip(initials, finals):
raw_pinyin = c+v raw_pinyin = c + v
# NOTE: post process for pypinyin outputs # NOTE: post process for pypinyin outputs
# we discriminate i, ii and iii # we discriminate i, ii and iii
if c == v: if c == v:
assert c in punctuation assert c in punctuation
phone = [c] phone = [c]
tone = '0' tone = "0"
word2ph.append(1) word2ph.append(1)
else: else:
v_without_tone = v[:-1] v_without_tone = v[:-1]
tone = v[-1] tone = v[-1]
pinyin = c+v_without_tone pinyin = c + v_without_tone
assert tone in '12345' assert tone in "12345"
if c: if c:
# 多音节 # 多音节
v_rep_map = { v_rep_map = {
"uei": 'ui', "uei": "ui",
'iou': 'iu', "iou": "iu",
'uen': 'un', "uen": "un",
} }
if v_without_tone in v_rep_map.keys(): if v_without_tone in v_rep_map.keys():
pinyin = c+v_rep_map[v_without_tone] pinyin = c + v_rep_map[v_without_tone]
else: else:
# 单音节 # 单音节
pinyin_rep_map = { pinyin_rep_map = {
'ing': 'ying', "ing": "ying",
'i': 'yi', "i": "yi",
'in': 'yin', "in": "yin",
'u': 'wu', "u": "wu",
} }
if pinyin in pinyin_rep_map.keys(): if pinyin in pinyin_rep_map.keys():
pinyin = pinyin_rep_map[pinyin] pinyin = pinyin_rep_map[pinyin]
else: else:
single_rep_map = { single_rep_map = {
'v': 'yu', "v": "yu",
'e': 'e', "e": "e",
'i': 'y', "i": "y",
'u': 'w', "u": "w",
} }
if pinyin[0] in single_rep_map.keys(): if pinyin[0] in single_rep_map.keys():
pinyin = single_rep_map[pinyin[0]]+pinyin[1:] pinyin = single_rep_map[pinyin[0]] + pinyin[1:]
assert pinyin in pinyin_to_symbol_map.keys(), (pinyin, seg, raw_pinyin) assert pinyin in pinyin_to_symbol_map.keys(), (pinyin, seg, raw_pinyin)
phone = pinyin_to_symbol_map[pinyin].split(' ') phone = pinyin_to_symbol_map[pinyin].split(" ")
word2ph.append(len(phone)) word2ph.append(len(phone))
phones_list += phone phones_list += phone
@@ -165,20 +169,23 @@ def _g2p(segments):
return phones_list, tones_list, word2ph return phones_list, tones_list, word2ph
def text_normalize(text): def text_normalize(text):
numbers = re.findall(r'\d+(?:\.?\d+)?', text) numbers = re.findall(r"\d+(?:\.?\d+)?", text)
for number in numbers: for number in numbers:
text = text.replace(number, cn2an.an2cn(number), 1) text = text.replace(number, cn2an.an2cn(number), 1)
text = replace_punctuation(text) text = replace_punctuation(text)
return text return text
def get_bert_feature(text, word2ph): def get_bert_feature(text, word2ph):
from text import chinese_bert from text import chinese_bert
return chinese_bert.get_bert_feature(text, word2ph) return chinese_bert.get_bert_feature(text, word2ph)
if __name__ == '__main__':
if __name__ == "__main__":
from text.chinese_bert import get_bert_feature from text.chinese_bert import get_bert_feature
text = "啊！但是《原神》是由,米哈\游自主， [研发]的一款全.新开放世界.冒险游戏" text = "啊！但是《原神》是由,米哈\游自主， [研发]的一款全.新开放世界.冒险游戏"
text = text_normalize(text) text = text_normalize(text)
print(text) print(text)

63
text/chinese_bert.py
View File

@@ -4,20 +4,27 @@ from transformers import AutoTokenizer, AutoModelForMaskedLM
tokenizer = AutoTokenizer.from_pretrained("./bert/chinese-roberta-wwm-ext-large") tokenizer = AutoTokenizer.from_pretrained("./bert/chinese-roberta-wwm-ext-large")
def get_bert_feature(text, word2ph, device=None): def get_bert_feature(text, word2ph, device=None):
if sys.platform == "darwin" and torch.backends.mps.is_available() and device == "cpu": if (
sys.platform == "darwin"
and torch.backends.mps.is_available()
and device == "cpu"
):
device = "mps" device = "mps"
if not device: if not device:
device = "cuda" device = "cuda"
model = AutoModelForMaskedLM.from_pretrained("./bert/chinese-roberta-wwm-ext-large").to(device) model = AutoModelForMaskedLM.from_pretrained(
"./bert/chinese-roberta-wwm-ext-large"
).to(device)
with torch.no_grad(): with torch.no_grad():
inputs = tokenizer(text, return_tensors='pt') inputs = tokenizer(text, return_tensors="pt")
for i in inputs: for i in inputs:
inputs[i] = inputs[i].to(device) inputs[i] = inputs[i].to(device)
res = model(**inputs, output_hidden_states=True) res = model(**inputs, output_hidden_states=True)
res = torch.cat(res['hidden_states'][-3:-2], -1)[0].cpu() res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
assert len(word2ph) == len(text)+2 assert len(word2ph) == len(text) + 2
word2phone = word2ph word2phone = word2ph
phone_level_feature = [] phone_level_feature = []
for i in range(len(word2phone)): for i in range(len(word2phone)):
@@ -26,14 +33,53 @@ def get_bert_feature(text, word2ph, device=None):
phone_level_feature = torch.cat(phone_level_feature, dim=0) phone_level_feature = torch.cat(phone_level_feature, dim=0)
return phone_level_feature.T return phone_level_feature.T
if __name__ == '__main__':
if __name__ == "__main__":
import torch import torch
word_level_feature = torch.rand(38, 1024) # 12个词,每个词1024维特征 word_level_feature = torch.rand(38, 1024) # 12个词,每个词1024维特征
word2phone = [1, 2, 1, 2, 2, 1, 2, 2, 1, 2, 2, 1, 2, 2, 2, 2, 2, 1, 1, 2, 2, 1, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 1] word2phone = [
1,
2,
1,
2,
2,
1,
2,
2,
1,
2,
2,
1,
2,
2,
2,
2,
2,
1,
1,
2,
2,
1,
2,
2,
2,
2,
1,
2,
2,
2,
2,
2,
1,
2,
2,
2,
2,
1,
]
# 计算总帧数 # 计算总帧数
total_frames = sum(word2phone) total_frames = sum(word2phone)
@@ -49,4 +95,3 @@ if __name__ == '__main__':
phone_level_feature = torch.cat(phone_level_feature, dim=0) phone_level_feature = torch.cat(phone_level_feature, dim=0)
print(phone_level_feature.shape) # torch.Size([36, 1024]) print(phone_level_feature.shape) # torch.Size([36, 1024])

10
text/cleaner.py
View File

@@ -1,10 +1,7 @@
from text import chinese, japanese, cleaned_text_to_sequence from text import chinese, japanese, cleaned_text_to_sequence
language_module_map = { language_module_map = {"ZH": chinese, "JP": japanese}
'ZH': chinese,
'JP': japanese
}
def clean_text(text, language): def clean_text(text, language):
@@ -13,6 +10,7 @@ def clean_text(text, language):
phones, tones, word2ph = language_module.g2p(norm_text) phones, tones, word2ph = language_module.g2p(norm_text)
return norm_text, phones, tones, word2ph return norm_text, phones, tones, word2ph
def clean_text_bert(text, language): def clean_text_bert(text, language):
language_module = language_module_map[language] language_module = language_module_map[language]
norm_text = language_module.text_normalize(text) norm_text = language_module.text_normalize(text)
@@ -20,9 +18,11 @@ def clean_text_bert(text, language):
bert = language_module.get_bert_feature(norm_text, word2ph) bert = language_module.get_bert_feature(norm_text, word2ph)
return phones, tones, bert return phones, tones, bert
def text_to_sequence(text, language): def text_to_sequence(text, language):
norm_text, phones, tones, word2ph = clean_text(text, language) norm_text, phones, tones, word2ph = clean_text(text, language)
return cleaned_text_to_sequence(phones, tones, language) return cleaned_text_to_sequence(phones, tones, language)
if __name__ == '__main__':
if __name__ == "__main__":
pass pass

121
text/english.py
View File

@@ -7,35 +7,108 @@ from string import punctuation
from text import symbols from text import symbols
current_file_path = os.path.dirname(__file__) current_file_path = os.path.dirname(__file__)
CMU_DICT_PATH = os.path.join(current_file_path, 'cmudict.rep') CMU_DICT_PATH = os.path.join(current_file_path, "cmudict.rep")
CACHE_PATH = os.path.join(current_file_path, 'cmudict_cache.pickle') CACHE_PATH = os.path.join(current_file_path, "cmudict_cache.pickle")
_g2p = G2p() _g2p = G2p()
arpa = {'AH0', 'S', 'AH1', 'EY2', 'AE2', 'EH0', 'OW2', 'UH0', 'NG', 'B', 'G', 'AY0', 'M', 'AA0', 'F', 'AO0', 'ER2', 'UH1', 'IY1', 'AH2', 'DH', 'IY0', 'EY1', 'IH0', 'K', 'N', 'W', 'IY2', 'T', 'AA1', 'ER1', 'EH2', 'OY0', 'UH2', 'UW1', 'Z', 'AW2', 'AW1', 'V', 'UW2', 'AA2', 'ER', 'AW0', 'UW0', 'R', 'OW1', 'EH1', 'ZH', 'AE0', 'IH2', 'IH', 'Y', 'JH', 'P', 'AY1', 'EY0', 'OY2', 'TH', 'HH', 'D', 'ER0', 'CH', 'AO1', 'AE1', 'AO2', 'OY1', 'AY2', 'IH1', 'OW0', 'L', 'SH'} arpa = {
"AH0",
"S",
"AH1",
"EY2",
"AE2",
"EH0",
"OW2",
"UH0",
"NG",
"B",
"G",
"AY0",
"M",
"AA0",
"F",
"AO0",
"ER2",
"UH1",
"IY1",
"AH2",
"DH",
"IY0",
"EY1",
"IH0",
"K",
"N",
"W",
"IY2",
"T",
"AA1",
"ER1",
"EH2",
"OY0",
"UH2",
"UW1",
"Z",
"AW2",
"AW1",
"V",
"UW2",
"AA2",
"ER",
"AW0",
"UW0",
"R",
"OW1",
"EH1",
"ZH",
"AE0",
"IH2",
"IH",
"Y",
"JH",
"P",
"AY1",
"EY0",
"OY2",
"TH",
"HH",
"D",
"ER0",
"CH",
"AO1",
"AE1",
"AO2",
"OY1",
"AY2",
"IH1",
"OW0",
"L",
"SH",
}
def post_replace_ph(ph): def post_replace_ph(ph):
rep_map = { rep_map = {
'：': ',', "：": ",",
'；': ',', "；": ",",
'，': ',', "，": ",",
'。': '.', "。": ".",
'！': '!', "！": "!",
'？': '?', "？": "?",
'\n': '.', "\n": ".",
"·": ",", "·": ",",
'、': ",", "、": ",",
'...': '…', "...": "…",
'v': "V" "v": "V",
} }
if ph in rep_map.keys(): if ph in rep_map.keys():
ph = rep_map[ph] ph = rep_map[ph]
if ph in symbols: if ph in symbols:
return ph return ph
if ph not in symbols: if ph not in symbols:
ph = 'UNK' ph = "UNK"
return ph return ph
def read_dict(): def read_dict():
g2p_dict = {} g2p_dict = {}
start_line = 49 start_line = 49
@@ -45,13 +118,13 @@ def read_dict():
while line: while line:
if line_index >= start_line: if line_index >= start_line:
line = line.strip() line = line.strip()
word_split = line.split(' ') word_split = line.split(" ")
word = word_split[0] word = word_split[0]
syllable_split = word_split[1].split(' - ') syllable_split = word_split[1].split(" - ")
g2p_dict[word] = [] g2p_dict[word] = []
for syllable in syllable_split: for syllable in syllable_split:
phone_split = syllable.split(' ') phone_split = syllable.split(" ")
g2p_dict[word].append(phone_split) g2p_dict[word].append(phone_split)
line_index = line_index + 1 line_index = line_index + 1
@@ -61,13 +134,13 @@ def read_dict():
def cache_dict(g2p_dict, file_path): def cache_dict(g2p_dict, file_path):
with open(file_path, 'wb') as pickle_file: with open(file_path, "wb") as pickle_file:
pickle.dump(g2p_dict, pickle_file) pickle.dump(g2p_dict, pickle_file)
def get_dict(): def get_dict():
if os.path.exists(CACHE_PATH): if os.path.exists(CACHE_PATH):
with open(CACHE_PATH, 'rb') as pickle_file: with open(CACHE_PATH, "rb") as pickle_file:
g2p_dict = pickle.load(pickle_file) g2p_dict = pickle.load(pickle_file)
else: else:
g2p_dict = read_dict() g2p_dict = read_dict()
@@ -75,15 +148,18 @@ def get_dict():
return g2p_dict return g2p_dict
eng_dict = get_dict() eng_dict = get_dict()
def refine_ph(phn): def refine_ph(phn):
tone = 0 tone = 0
if re.search(r'\d$', phn): if re.search(r"\d$", phn):
tone = int(phn[-1]) + 1 tone = int(phn[-1]) + 1
phn = phn[:-1] phn = phn[:-1]
return phn.lower(), tone return phn.lower(), tone
def refine_syllables(syllables): def refine_syllables(syllables):
tones = [] tones = []
phonemes = [] phonemes = []
@@ -100,8 +176,8 @@ def text_normalize(text):
# todo: eng text normalize # todo: eng text normalize
return text return text
def g2p(text):
def g2p(text):
phones = [] phones = []
tones = [] tones = []
words = re.split(r"([,;.\-\?\!\s+])", text) words = re.split(r"([,;.\-\?\!\s+])", text)
@@ -126,6 +202,7 @@ def g2p(text):
phones = [post_replace_ph(i) for i in phones] phones = [post_replace_ph(i) for i in phones]
return phones, tones, word2ph return phones, tones, word2ph
if __name__ == "__main__": if __name__ == "__main__":
# print(get_dict()) # print(get_dict())
# print(eng_word_to_phoneme("hello")) # print(eng_word_to_phoneme("hello"))
@@ -135,4 +212,4 @@ if __name__ == "__main__":
# for group in syllables: # for group in syllables:
# for ph in group: # for ph in group:
# all_phones.add(ph) # all_phones.add(ph)
# print(all_phones) # print(all_phones)

54
text/japanese.py
View File

@@ -331,12 +331,12 @@ def kata2phoneme(text: str) -> str:
x = _RULEMAP2.get(text[:2]) x = _RULEMAP2.get(text[:2])
if x is not None: if x is not None:
text = text[2:] text = text[2:]
res += x.split(' ')[1:] res += x.split(" ")[1:]
continue continue
x = _RULEMAP1.get(text[0]) x = _RULEMAP1.get(text[0])
if x is not None: if x is not None:
text = text[1:] text = text[1:]
res += x.split(' ')[1:] res += x.split(" ")[1:]
continue continue
res.append(text[0]) res.append(text[0])
text = text[1:] text = text[1:]
@@ -358,6 +358,7 @@ _SYMBOL_TOKENS = set(list("・、。？！"))
_NO_YOMI_TOKENS = set(list("「」『』―（）［］[]")) _NO_YOMI_TOKENS = set(list("「」『』―（）［］[]"))
_TAGGER = MeCab.Tagger() _TAGGER = MeCab.Tagger()
def text2kata(text: str) -> str: def text2kata(text: str) -> str:
parsed = _TAGGER.parse(text) parsed = _TAGGER.parse(text)
res = [] res = []
@@ -472,6 +473,7 @@ def japanese_text_to_phonemes(text: str) -> str:
res = kata2phoneme(res) res = kata2phoneme(res)
return res return res
def is_japanese_character(char): def is_japanese_character(char):
# 定义日语文字系统的 Unicode 范围 # 定义日语文字系统的 Unicode 范围
japanese_ranges = [ japanese_ranges = [
@@ -493,27 +495,37 @@ def is_japanese_character(char):
return False return False
rep_map = { rep_map = {
'：': ',', "：": ",",
'；': ',', "；": ",",
'，': ',', "，": ",",
'。': '.', "。": ".",
'！': '!', "！": "!",
'？': '?', "？": "?",
'\n': '.', "\n": ".",
"·": ",", "·": ",",
'、': ",", "、": ",",
'...': '…' "...": "…",
} }
def replace_punctuation(text): def replace_punctuation(text):
pattern = re.compile('|'.join(re.escape(p) for p in rep_map.keys())) pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys()))
replaced_text = pattern.sub(lambda x: rep_map[x.group()], text) replaced_text = pattern.sub(lambda x: rep_map[x.group()], text)
replaced_text = re.sub(r'[^\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\u3400-\u4DBF'+"".join(punctuation)+r']+', '', replaced_text) replaced_text = re.sub(
r"[^\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\u3400-\u4DBF"
+ "".join(punctuation)
+ r"]+",
"",
replaced_text,
)
return replaced_text return replaced_text
def text_normalize(text): def text_normalize(text):
res = unicodedata.normalize("NFKC", text) res = unicodedata.normalize("NFKC", text)
res = japanese_convert_numbers_to_words(res) res = japanese_convert_numbers_to_words(res)
@@ -521,6 +533,7 @@ def text_normalize(text):
res = replace_punctuation(res) res = replace_punctuation(res)
return res return res
def distribute_phone(n_phone, n_word): def distribute_phone(n_phone, n_word):
phones_per_word = [0] * n_word phones_per_word = [0] * n_word
for task in range(n_phone): for task in range(n_phone):
@@ -529,16 +542,19 @@ def distribute_phone(n_phone, n_word):
phones_per_word[min_index] += 1 phones_per_word[min_index] += 1
return phones_per_word return phones_per_word
tokenizer = AutoTokenizer.from_pretrained("./bert/bert-base-japanese-v3") tokenizer = AutoTokenizer.from_pretrained("./bert/bert-base-japanese-v3")
def g2p(norm_text): def g2p(norm_text):
tokenized = tokenizer.tokenize(norm_text) tokenized = tokenizer.tokenize(norm_text)
phs = [] phs = []
ph_groups = [] ph_groups = []
for t in tokenized: for t in tokenized:
if not t.startswith('#'): if not t.startswith("#"):
ph_groups.append([t]) ph_groups.append([t])
else: else:
ph_groups[-1].append(t.replace("#", '')) ph_groups[-1].append(t.replace("#", ""))
word2ph = [] word2ph = []
for group in ph_groups: for group in ph_groups:
phonemes = kata2phoneme(text2kata("".join(group))) phonemes = kata2phoneme(text2kata("".join(group)))
@@ -552,12 +568,13 @@ def g2p(norm_text):
word2ph += aaa word2ph += aaa
phs += phonemes phs += phonemes
phones = ['_'] + phs + ["_"] phones = ["_"] + phs + ["_"]
tones = [0 for i in phones] tones = [0 for i in phones]
word2ph = [1] + word2ph + [1] word2ph = [1] + word2ph + [1]
return phones, tones, word2ph return phones, tones, word2ph
if __name__ == '__main__':
if __name__ == "__main__":
tokenizer = AutoTokenizer.from_pretrained("./bert/bert-base-japanese-v3") tokenizer = AutoTokenizer.from_pretrained("./bert/bert-base-japanese-v3")
text = "hello,こんにちは、世界！……" text = "hello,こんにちは、世界！……"
from text.japanese_bert import get_bert_feature from text.japanese_bert import get_bert_feature
@@ -568,6 +585,3 @@ if __name__ == '__main__':
bert = get_bert_feature(text, word2ph) bert = get_bert_feature(text, word2ph)
print(phones, tones, word2ph, bert.shape) print(phones, tones, word2ph, bert.shape)

17
text/japanese_bert.py
View File

@@ -4,19 +4,26 @@ import sys
tokenizer = AutoTokenizer.from_pretrained("./bert/bert-base-japanese-v3") tokenizer = AutoTokenizer.from_pretrained("./bert/bert-base-japanese-v3")
def get_bert_feature(text, word2ph, device=None): def get_bert_feature(text, word2ph, device=None):
if sys.platform == "darwin" and torch.backends.mps.is_available() and device == "cpu": if (
sys.platform == "darwin"
and torch.backends.mps.is_available()
and device == "cpu"
):
device = "mps" device = "mps"
if not device: if not device:
device = "cuda" device = "cuda"
model = AutoModelForMaskedLM.from_pretrained("./bert/bert-base-japanese-v3").to(device) model = AutoModelForMaskedLM.from_pretrained("./bert/bert-base-japanese-v3").to(
device
)
with torch.no_grad(): with torch.no_grad():
inputs = tokenizer(text, return_tensors='pt') inputs = tokenizer(text, return_tensors="pt")
for i in inputs: for i in inputs:
inputs[i] = inputs[i].to(device) inputs[i] = inputs[i].to(device)
res = model(**inputs, output_hidden_states=True) res = model(**inputs, output_hidden_states=True)
res = torch.cat(res['hidden_states'][-3:-2], -1)[0].cpu() res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
assert inputs['input_ids'].shape[-1] == len(word2ph) assert inputs["input_ids"].shape[-1] == len(word2ph)
word2phone = word2ph word2phone = word2ph
phone_level_feature = [] phone_level_feature = []
for i in range(len(word2phone)): for i in range(len(word2phone)):

183
text/symbols.py
View File

@@ -1,26 +1,166 @@
punctuation = ['!', '?', '…', ",", ".", "'", '-'] punctuation = ["!", "?", "…", ",", ".", "'", "-"]
pu_symbols = punctuation + ["SP", "UNK"] pu_symbols = punctuation + ["SP", "UNK"]
pad = '_' pad = "_"
# chinese # chinese
zh_symbols = ['E', 'En', 'a', 'ai', 'an', 'ang', 'ao', 'b', 'c', 'ch', 'd', 'e', 'ei', 'en', 'eng', 'er', 'f', 'g', 'h', zh_symbols = [
'i', 'i0', 'ia', 'ian', 'iang', 'iao', 'ie', 'in', 'ing', 'iong', 'ir', 'iu', 'j', 'k', 'l', 'm', 'n', 'o', "E",
'ong', "En",
'ou', 'p', 'q', 'r', 's', 'sh', 't', 'u', 'ua', 'uai', 'uan', 'uang', 'ui', 'un', 'uo', 'v', 'van', 've', 'vn', "a",
'w', 'x', 'y', 'z', 'zh', "ai",
"AA", "EE", "OO"] "an",
"ang",
"ao",
"b",
"c",
"ch",
"d",
"e",
"ei",
"en",
"eng",
"er",
"f",
"g",
"h",
"i",
"i0",
"ia",
"ian",
"iang",
"iao",
"ie",
"in",
"ing",
"iong",
"ir",
"iu",
"j",
"k",
"l",
"m",
"n",
"o",
"ong",
"ou",
"p",
"q",
"r",
"s",
"sh",
"t",
"u",
"ua",
"uai",
"uan",
"uang",
"ui",
"un",
"uo",
"v",
"van",
"ve",
"vn",
"w",
"x",
"y",
"z",
"zh",
"AA",
"EE",
"OO",
]
num_zh_tones = 6 num_zh_tones = 6
# japanese # japanese
ja_symbols = ['N', 'a', 'a:', 'b', 'by', 'ch', 'd', 'dy', 'e', 'e:', 'f', 'g', 'gy', 'h', 'hy', 'i', 'i:', 'j', 'k', 'ky', ja_symbols = [
'm', 'my', 'n', 'ny', 'o', 'o:', 'p', 'py', 'q', 'r', 'ry', 's', 'sh', 't', 'ts', 'ty', 'u', 'u:', "N",
'w', 'y', 'z', 'zy'] "a",
"a:",
"b",
"by",
"ch",
"d",
"dy",
"e",
"e:",
"f",
"g",
"gy",
"h",
"hy",
"i",
"i:",
"j",
"k",
"ky",
"m",
"my",
"n",
"ny",
"o",
"o:",
"p",
"py",
"q",
"r",
"ry",
"s",
"sh",
"t",
"ts",
"ty",
"u",
"u:",
"w",
"y",
"z",
"zy",
]
num_ja_tones = 1 num_ja_tones = 1
# English # English
en_symbols = ['aa', 'ae', 'ah', 'ao', 'aw', 'ay', 'b', 'ch', 'd', 'dh', 'eh', 'er', 'ey', 'f', 'g', 'hh', 'ih', 'iy', en_symbols = [
'jh', 'k', 'l', 'm', 'n', 'ng', 'ow', 'oy', 'p', 'r', 's', "aa",
'sh', 't', 'th', 'uh', 'uw', 'V', 'w', 'y', 'z', 'zh'] "ae",
"ah",
"ao",
"aw",
"ay",
"b",
"ch",
"d",
"dh",
"eh",
"er",
"ey",
"f",
"g",
"hh",
"ih",
"iy",
"jh",
"k",
"l",
"m",
"n",
"ng",
"ow",
"oy",
"p",
"r",
"s",
"sh",
"t",
"th",
"uh",
"uw",
"V",
"w",
"y",
"z",
"zh",
]
num_en_tones = 4 num_en_tones = 4
# combine all symbols # combine all symbols
@@ -32,21 +172,16 @@ sil_phonemes_ids = [symbols.index(i) for i in pu_symbols]
num_tones = num_zh_tones + num_ja_tones + num_en_tones num_tones = num_zh_tones + num_ja_tones + num_en_tones
# language maps # language maps
language_id_map = { language_id_map = {"ZH": 0, "JP": 1, "EN": 2}
'ZH': 0,
"JP": 1,
"EN": 2
}
num_languages = len(language_id_map.keys()) num_languages = len(language_id_map.keys())
language_tone_start_map = { language_tone_start_map = {
'ZH': 0, "ZH": 0,
"JP": num_zh_tones, "JP": num_zh_tones,
"EN": num_zh_tones + num_ja_tones "EN": num_zh_tones + num_ja_tones,
} }
if __name__ == '__main__': if __name__ == "__main__":
a = set(zh_symbols) a = set(zh_symbols)
b = set(en_symbols) b = set(en_symbols)
print(sorted(a&b)) print(sorted(a & b))

610
text/tone_sandhi.py
View File

@@ -19,51 +19,442 @@ from pypinyin import lazy_pinyin
from pypinyin import Style from pypinyin import Style
class ToneSandhi(): class ToneSandhi:
def __init__(self): def __init__(self):
self.must_neural_tone_words = { self.must_neural_tone_words = {
'麻烦', '麻利', '鸳鸯', '高粱', '骨头', '骆驼', '马虎', '首饰', '馒头', '馄饨', '风筝', "麻烦",
'难为', '队伍', '阔气', '闺女', '门道', '锄头', '铺盖', '铃铛', '铁匠', '钥匙', '里脊', "麻利",
'里头', '部分', '那么', '道士', '造化', '迷糊', '连累', '这么', '这个', '运气', '过去', "鸳鸯",
'软和', '转悠', '踏实', '跳蚤', '跟头', '趔趄', '财主', '豆腐', '讲究', '记性', '记号', "高粱",
'认识', '规矩', '见识', '裁缝', '补丁', '衣裳', '衣服', '衙门', '街坊', '行李', '行当', "骨头",
'蛤蟆', '蘑菇', '薄荷', '葫芦', '葡萄', '萝卜', '荸荠', '苗条', '苗头', '苍蝇', '芝麻', "骆驼",
'舒服', '舒坦', '舌头', '自在', '膏药', '脾气', '脑袋', '脊梁', '能耐', '胳膊', '胭脂', "马虎",
'胡萝', '胡琴', '胡同', '聪明', '耽误', '耽搁', '耷拉', '耳朵', '老爷', '老实', '老婆', "首饰",
'老头', '老太', '翻腾', '罗嗦', '罐头', '编辑', '结实', '红火', '累赘', '糨糊', '糊涂', "馒头",
'精神', '粮食', '簸箕', '篱笆', '算计', '算盘', '答应', '笤帚', '笑语', '笑话', '窟窿', "馄饨",
'窝囊', '窗户', '稳当', '稀罕', '称呼', '秧歌', '秀气', '秀才', '福气', '祖宗', '砚台', "风筝",
'码头', '石榴', '石头', '石匠', '知识', '眼睛', '眯缝', '眨巴', '眉毛', '相声', '盘算', "难为",
'白净', '痢疾', '痛快', '疟疾', '疙瘩', '疏忽', '畜生', '生意', '甘蔗', '琵琶', '琢磨', "队伍",
'琉璃', '玻璃', '玫瑰', '玄乎', '狐狸', '状元', '特务', '牲口', '牙碜', '牌楼', '爽快', "阔气",
'爱人', '热闹', '烧饼', '烟筒', '烂糊', '点心', '炊帚', '灯笼', '火候', '漂亮', '滑溜', "闺女",
'溜达', '温和', '清楚', '消息', '浪头', '活泼', '比方', '正经', '欺负', '模糊', '槟榔', "门道",
'棺材', '棒槌', '棉花', '核桃', '栅栏', '柴火', '架势', '枕头', '枇杷', '机灵', '本事', "锄头",
'木头', '木匠', '朋友', '月饼', '月亮', '暖和', '明白', '时候', '新鲜', '故事', '收拾', "铺盖",
'收成', '提防', '挖苦', '挑剔', '指甲', '指头', '拾掇', '拳头', '拨弄', '招牌', '招呼', "铃铛",
'抬举', '护士', '折腾', '扫帚', '打量', '打算', '打点', '打扮', '打听', '打发', '扎实', "铁匠",
'扁担', '戒指', '懒得', '意识', '意思', '情形', '悟性', '怪物', '思量', '怎么', '念头', "钥匙",
'念叨', '快活', '忙活', '志气', '心思', '得罪', '张罗', '弟兄', '开通', '应酬', '庄稼', "里脊",
'干事', '帮手', '帐篷', '希罕', '师父', '师傅', '巴结', '巴掌', '差事', '工夫', '岁数', "里头",
'屁股', '尾巴', '少爷', '小气', '小伙', '将就', '对头', '对付', '寡妇', '家伙', '客气', "部分",
'实在', '官司', '学问', '学生', '字号', '嫁妆', '媳妇', '媒人', '婆家', '娘家', '委屈', "那么",
'姑娘', '姐夫', '妯娌', '妥当', '妖精', '奴才', '女婿', '头发', '太阳', '大爷', '大方', "道士",
'大意', '大夫', '多少', '多么', '外甥', '壮实', '地道', '地方', '在乎', '困难', '嘴巴', "造化",
'嘱咐', '嘟囔', '嘀咕', '喜欢', '喇嘛', '喇叭', '商量', '唾沫', '哑巴', '哈欠', '哆嗦', "迷糊",
'咳嗽', '和尚', '告诉', '告示', '含糊', '吓唬', '后头', '名字', '名堂', '合同', '吆喝', "连累",
'叫唤', '口袋', '厚道', '厉害', '千斤', '包袱', '包涵', '匀称', '勤快', '动静', '动弹', "这么",
'功夫', '力气', '前头', '刺猬', '刺激', '别扭', '利落', '利索', '利害', '分析', '出息', "这个",
'凑合', '凉快', '冷战', '冤枉', '冒失', '养活', '关系', '先生', '兄弟', '便宜', '使唤', "运气",
'佩服', '作坊', '体面', '位置', '似的', '伙计', '休息', '什么', '人家', '亲戚', '亲家', "过去",
'交情', '云彩', '事情', '买卖', '主意', '丫头', '丧气', '两口', '东西', '东家', '世故', "软和",
'不由', '不在', '下水', '下巴', '上头', '上司', '丈夫', '丈人', '一辈', '那个', '菩萨', "转悠",
'父亲', '母亲', '咕噜', '邋遢', '费用', '冤家', '甜头', '介绍', '荒唐', '大人', '泥鳅', "踏实",
'幸福', '熟悉', '计划', '扑腾', '蜡烛', '姥爷', '照顾', '喉咙', '吉他', '弄堂', '蚂蚱', "跳蚤",
'凤凰', '拖沓', '寒碜', '糟蹋', '倒腾', '报复', '逻辑', '盘缠', '喽啰', '牢骚', '咖喱', "跟头",
'扫把', '惦记' "趔趄",
"财主",
"豆腐",
"讲究",
"记性",
"记号",
"认识",
"规矩",
"见识",
"裁缝",
"补丁",
"衣裳",
"衣服",
"衙门",
"街坊",
"行李",
"行当",
"蛤蟆",
"蘑菇",
"薄荷",
"葫芦",
"葡萄",
"萝卜",
"荸荠",
"苗条",
"苗头",
"苍蝇",
"芝麻",
"舒服",
"舒坦",
"舌头",
"自在",
"膏药",
"脾气",
"脑袋",
"脊梁",
"能耐",
"胳膊",
"胭脂",
"胡萝",
"胡琴",
"胡同",
"聪明",
"耽误",
"耽搁",
"耷拉",
"耳朵",
"老爷",
"老实",
"老婆",
"老头",
"老太",
"翻腾",
"罗嗦",
"罐头",
"编辑",
"结实",
"红火",
"累赘",
"糨糊",
"糊涂",
"精神",
"粮食",
"簸箕",
"篱笆",
"算计",
"算盘",
"答应",
"笤帚",
"笑语",
"笑话",
"窟窿",
"窝囊",
"窗户",
"稳当",
"稀罕",
"称呼",
"秧歌",
"秀气",
"秀才",
"福气",
"祖宗",
"砚台",
"码头",
"石榴",
"石头",
"石匠",
"知识",
"眼睛",
"眯缝",
"眨巴",
"眉毛",
"相声",
"盘算",
"白净",
"痢疾",
"痛快",
"疟疾",
"疙瘩",
"疏忽",
"畜生",
"生意",
"甘蔗",
"琵琶",
"琢磨",
"琉璃",
"玻璃",
"玫瑰",
"玄乎",
"狐狸",
"状元",
"特务",
"牲口",
"牙碜",
"牌楼",
"爽快",
"爱人",
"热闹",
"烧饼",
"烟筒",
"烂糊",
"点心",
"炊帚",
"灯笼",
"火候",
"漂亮",
"滑溜",
"溜达",
"温和",
"清楚",
"消息",
"浪头",
"活泼",
"比方",
"正经",
"欺负",
"模糊",
"槟榔",
"棺材",
"棒槌",
"棉花",
"核桃",
"栅栏",
"柴火",
"架势",
"枕头",
"枇杷",
"机灵",
"本事",
"木头",
"木匠",
"朋友",
"月饼",
"月亮",
"暖和",
"明白",
"时候",
"新鲜",
"故事",
"收拾",
"收成",
"提防",
"挖苦",
"挑剔",
"指甲",
"指头",
"拾掇",
"拳头",
"拨弄",
"招牌",
"招呼",
"抬举",
"护士",
"折腾",
"扫帚",
"打量",
"打算",
"打点",
"打扮",
"打听",
"打发",
"扎实",
"扁担",
"戒指",
"懒得",
"意识",
"意思",
"情形",
"悟性",
"怪物",
"思量",
"怎么",
"念头",
"念叨",
"快活",
"忙活",
"志气",
"心思",
"得罪",
"张罗",
"弟兄",
"开通",
"应酬",
"庄稼",
"干事",
"帮手",
"帐篷",
"希罕",
"师父",
"师傅",
"巴结",
"巴掌",
"差事",
"工夫",
"岁数",
"屁股",
"尾巴",
"少爷",
"小气",
"小伙",
"将就",
"对头",
"对付",
"寡妇",
"家伙",
"客气",
"实在",
"官司",
"学问",
"学生",
"字号",
"嫁妆",
"媳妇",
"媒人",
"婆家",
"娘家",
"委屈",
"姑娘",
"姐夫",
"妯娌",
"妥当",
"妖精",
"奴才",
"女婿",
"头发",
"太阳",
"大爷",
"大方",
"大意",
"大夫",
"多少",
"多么",
"外甥",
"壮实",
"地道",
"地方",
"在乎",
"困难",
"嘴巴",
"嘱咐",
"嘟囔",
"嘀咕",
"喜欢",
"喇嘛",
"喇叭",
"商量",
"唾沫",
"哑巴",
"哈欠",
"哆嗦",
"咳嗽",
"和尚",
"告诉",
"告示",
"含糊",
"吓唬",
"后头",
"名字",
"名堂",
"合同",
"吆喝",
"叫唤",
"口袋",
"厚道",
"厉害",
"千斤",
"包袱",
"包涵",
"匀称",
"勤快",
"动静",
"动弹",
"功夫",
"力气",
"前头",
"刺猬",
"刺激",
"别扭",
"利落",
"利索",
"利害",
"分析",
"出息",
"凑合",
"凉快",
"冷战",
"冤枉",
"冒失",
"养活",
"关系",
"先生",
"兄弟",
"便宜",
"使唤",
"佩服",
"作坊",
"体面",
"位置",
"似的",
"伙计",
"休息",
"什么",
"人家",
"亲戚",
"亲家",
"交情",
"云彩",
"事情",
"买卖",
"主意",
"丫头",
"丧气",
"两口",
"东西",
"东家",
"世故",
"不由",
"不在",
"下水",
"下巴",
"上头",
"上司",
"丈夫",
"丈人",
"一辈",
"那个",
"菩萨",
"父亲",
"母亲",
"咕噜",
"邋遢",
"费用",
"冤家",
"甜头",
"介绍",
"荒唐",
"大人",
"泥鳅",
"幸福",
"熟悉",
"计划",
"扑腾",
"蜡烛",
"姥爷",
"照顾",
"喉咙",
"吉他",
"弄堂",
"蚂蚱",
"凤凰",
"拖沓",
"寒碜",
"糟蹋",
"倒腾",
"报复",
"逻辑",
"盘缠",
"喽啰",
"牢骚",
"咖喱",
"扫把",
"惦记",
} }
self.must_not_neural_tone_words = { self.must_not_neural_tone_words = {
"男子", "女子", "分子", "原子", "量子", "莲子", "石子", "瓜子", "电子", "人人", "虎虎" "男子",
"女子",
"分子",
"原子",
"量子",
"莲子",
"石子",
"瓜子",
"电子",
"人人",
"虎虎",
} }
self.punc = "：，；。？！“”‘’':,;.?!" self.punc = "：，；。？！“”‘’':,;.?!"
@@ -72,14 +463,15 @@ class ToneSandhi():
# word: "家里" # word: "家里"
# pos: "s" # pos: "s"
# finals: ['ia1', 'i3'] # finals: ['ia1', 'i3']
def _neural_sandhi(self, word: str, pos: str, def _neural_sandhi(self, word: str, pos: str, finals: List[str]) -> List[str]:
finals: List[str]) -> List[str]:
# reduplication words for n. and v. e.g. 奶奶, 试试, 旺旺 # reduplication words for n. and v. e.g. 奶奶, 试试, 旺旺
for j, item in enumerate(word): for j, item in enumerate(word):
if j - 1 >= 0 and item == word[j - 1] and pos[0] in { if (
"n", "v", "a" j - 1 >= 0
} and word not in self.must_not_neural_tone_words: and item == word[j - 1]
and pos[0] in {"n", "v", "a"}
and word not in self.must_not_neural_tone_words
):
finals[j] = finals[j][:-1] + "5" finals[j] = finals[j][:-1] + "5"
ge_idx = word.find("个") ge_idx = word.find("个")
if len(word) >= 1 and word[-1] in "吧呢啊呐噻嘛吖嗨呐哦哒额滴哩哟喽啰耶喔诶": if len(word) >= 1 and word[-1] in "吧呢啊呐噻嘛吖嗨呐哦哒额滴哩哟喽啰耶喔诶":
@@ -89,9 +481,12 @@ class ToneSandhi():
# e.g. 走了, 看着, 去过 # e.g. 走了, 看着, 去过
# elif len(word) == 1 and word in "了着过" and pos in {"ul", "uz", "ug"}: # elif len(word) == 1 and word in "了着过" and pos in {"ul", "uz", "ug"}:
# finals[-1] = finals[-1][:-1] + "5" # finals[-1] = finals[-1][:-1] + "5"
elif len(word) > 1 and word[-1] in "们子" and pos in { elif (
"r", "n" len(word) > 1
} and word not in self.must_not_neural_tone_words: and word[-1] in "们子"
and pos in {"r", "n"}
and word not in self.must_not_neural_tone_words
):
finals[-1] = finals[-1][:-1] + "5" finals[-1] = finals[-1][:-1] + "5"
# e.g. 桌上, 地下, 家里 # e.g. 桌上, 地下, 家里
elif len(word) > 1 and word[-1] in "上下里" and pos in {"s", "l", "f"}: elif len(word) > 1 and word[-1] in "上下里" and pos in {"s", "l", "f"}:
@@ -100,21 +495,26 @@ class ToneSandhi():
elif len(word) > 1 and word[-1] in "来去" and word[-2] in "上下进出回过起开": elif len(word) > 1 and word[-1] in "来去" and word[-2] in "上下进出回过起开":
finals[-1] = finals[-1][:-1] + "5" finals[-1] = finals[-1][:-1] + "5"
# 个做量词 # 个做量词
elif (ge_idx >= 1 and elif (
(word[ge_idx - 1].isnumeric() or ge_idx >= 1
word[ge_idx - 1] in "几有两半多各整每做是")) or word == '个': and (word[ge_idx - 1].isnumeric() or word[ge_idx - 1] in "几有两半多各整每做是")
) or word == "个":
finals[ge_idx] = finals[ge_idx][:-1] + "5" finals[ge_idx] = finals[ge_idx][:-1] + "5"
else: else:
if word in self.must_neural_tone_words or word[ if (
-2:] in self.must_neural_tone_words: word in self.must_neural_tone_words
or word[-2:] in self.must_neural_tone_words
):
finals[-1] = finals[-1][:-1] + "5" finals[-1] = finals[-1][:-1] + "5"
word_list = self._split_word(word) word_list = self._split_word(word)
finals_list = [finals[:len(word_list[0])], finals[len(word_list[0]):]] finals_list = [finals[: len(word_list[0])], finals[len(word_list[0]) :]]
for i, word in enumerate(word_list): for i, word in enumerate(word_list):
# conventional neural in Chinese # conventional neural in Chinese
if word in self.must_neural_tone_words or word[ if (
-2:] in self.must_neural_tone_words: word in self.must_neural_tone_words
or word[-2:] in self.must_neural_tone_words
):
finals_list[i][-1] = finals_list[i][-1][:-1] + "5" finals_list[i][-1] = finals_list[i][-1][:-1] + "5"
finals = sum(finals_list, []) finals = sum(finals_list, [])
return finals return finals
@@ -126,15 +526,15 @@ class ToneSandhi():
else: else:
for i, char in enumerate(word): for i, char in enumerate(word):
# "不" before tone4 should be bu2, e.g. 不怕 # "不" before tone4 should be bu2, e.g. 不怕
if char == "不" and i + 1 < len(word) and finals[i + if char == "不" and i + 1 < len(word) and finals[i + 1][-1] == "4":
1][-1] == "4":
finals[i] = finals[i][:-1] + "2" finals[i] = finals[i][:-1] + "2"
return finals return finals
def _yi_sandhi(self, word: str, finals: List[str]) -> List[str]: def _yi_sandhi(self, word: str, finals: List[str]) -> List[str]:
# "一" in number sequences, e.g. 一零零, 二一零 # "一" in number sequences, e.g. 一零零, 二一零
if word.find("一") != -1 and all( if word.find("一") != -1 and all(
[item.isnumeric() for item in word if item != "一"]): [item.isnumeric() for item in word if item != "一"]
):
return finals return finals
# "一" between reduplication words shold be yi5, e.g. 看一看 # "一" between reduplication words shold be yi5, e.g. 看一看
elif len(word) == 3 and word[1] == "一" and word[0] == word[-1]: elif len(word) == 3 and word[1] == "一" and word[0] == word[-1]:
@@ -161,10 +561,10 @@ class ToneSandhi():
first_subword = word_list[0] first_subword = word_list[0]
first_begin_idx = word.find(first_subword) first_begin_idx = word.find(first_subword)
if first_begin_idx == 0: if first_begin_idx == 0:
second_subword = word[len(first_subword):] second_subword = word[len(first_subword) :]
new_word_list = [first_subword, second_subword] new_word_list = [first_subword, second_subword]
else: else:
second_subword = word[:-len(first_subword)] second_subword = word[: -len(first_subword)]
new_word_list = [second_subword, first_subword] new_word_list = [second_subword, first_subword]
return new_word_list return new_word_list
@@ -182,18 +582,19 @@ class ToneSandhi():
elif len(word_list[0]) == 1: elif len(word_list[0]) == 1:
finals[1] = finals[1][:-1] + "2" finals[1] = finals[1][:-1] + "2"
else: else:
finals_list = [ finals_list = [finals[: len(word_list[0])], finals[len(word_list[0]) :]]
finals[:len(word_list[0])], finals[len(word_list[0]):]
]
if len(finals_list) == 2: if len(finals_list) == 2:
for i, sub in enumerate(finals_list): for i, sub in enumerate(finals_list):
# e.g. 所有/人 # e.g. 所有/人
if self._all_tone_three(sub) and len(sub) == 2: if self._all_tone_three(sub) and len(sub) == 2:
finals_list[i][0] = finals_list[i][0][:-1] + "2" finals_list[i][0] = finals_list[i][0][:-1] + "2"
# e.g. 好/喜欢 # e.g. 好/喜欢
elif i == 1 and not self._all_tone_three(sub) and finals_list[i][0][-1] == "3" and \ elif (
finals_list[0][-1][-1] == "3": i == 1
and not self._all_tone_three(sub)
and finals_list[i][0][-1] == "3"
and finals_list[0][-1][-1] == "3"
):
finals_list[0][-1] = finals_list[0][-1][:-1] + "2" finals_list[0][-1] = finals_list[0][-1][:-1] + "2"
finals = sum(finals_list, []) finals = sum(finals_list, [])
# split idiom into two words who's length is 2 # split idiom into two words who's length is 2
@@ -222,7 +623,7 @@ class ToneSandhi():
new_seg.append((word, pos)) new_seg.append((word, pos))
last_word = word[:] last_word = word[:]
if last_word == "不": if last_word == "不":
new_seg.append((last_word, 'd')) new_seg.append((last_word, "d"))
last_word = "" last_word = ""
return new_seg return new_seg
@@ -236,12 +637,21 @@ class ToneSandhi():
new_seg = [] new_seg = []
# function 1 # function 1
for i, (word, pos) in enumerate(seg): for i, (word, pos) in enumerate(seg):
if i - 1 >= 0 and word == "一" and i + 1 < len(seg) and seg[i - 1][ if (
0] == seg[i + 1][0] and seg[i - 1][1] == "v": i - 1 >= 0
and word == "一"
and i + 1 < len(seg)
and seg[i - 1][0] == seg[i + 1][0]
and seg[i - 1][1] == "v"
):
new_seg[i - 1][0] = new_seg[i - 1][0] + "一" + new_seg[i - 1][0] new_seg[i - 1][0] = new_seg[i - 1][0] + "一" + new_seg[i - 1][0]
else: else:
if i - 2 >= 0 and seg[i - 1][0] == "一" and seg[i - 2][ if (
0] == word and pos == "v": i - 2 >= 0
and seg[i - 1][0] == "一"
and seg[i - 2][0] == word
and pos == "v"
):
continue continue
else: else:
new_seg.append([word, pos]) new_seg.append([word, pos])
@@ -257,22 +667,27 @@ class ToneSandhi():
# the first and the second words are all_tone_three # the first and the second words are all_tone_three
def _merge_continuous_three_tones( def _merge_continuous_three_tones(
self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]: self, seg: List[Tuple[str, str]]
) -> List[Tuple[str, str]]:
new_seg = [] new_seg = []
sub_finals_list = [ sub_finals_list = [
lazy_pinyin( lazy_pinyin(word, neutral_tone_with_five=True, style=Style.FINALS_TONE3)
word, neutral_tone_with_five=True, style=Style.FINALS_TONE3)
for (word, pos) in seg for (word, pos) in seg
] ]
assert len(sub_finals_list) == len(seg) assert len(sub_finals_list) == len(seg)
merge_last = [False] * len(seg) merge_last = [False] * len(seg)
for i, (word, pos) in enumerate(seg): for i, (word, pos) in enumerate(seg):
if i - 1 >= 0 and self._all_tone_three( if (
sub_finals_list[i - 1]) and self._all_tone_three( i - 1 >= 0
sub_finals_list[i]) and not merge_last[i - 1]: and self._all_tone_three(sub_finals_list[i - 1])
and self._all_tone_three(sub_finals_list[i])
and not merge_last[i - 1]
):
# if the last word is reduplication, not merge, because reduplication need to be _neural_sandhi # if the last word is reduplication, not merge, because reduplication need to be _neural_sandhi
if not self._is_reduplication(seg[i - 1][0]) and len( if (
seg[i - 1][0]) + len(seg[i][0]) <= 3: not self._is_reduplication(seg[i - 1][0])
and len(seg[i - 1][0]) + len(seg[i][0]) <= 3
):
new_seg[-1][0] = new_seg[-1][0] + seg[i][0] new_seg[-1][0] = new_seg[-1][0] + seg[i][0]
merge_last[i] = True merge_last[i] = True
else: else:
@@ -287,21 +702,27 @@ class ToneSandhi():
# the last char of first word and the first char of second word is tone_three # the last char of first word and the first char of second word is tone_three
def _merge_continuous_three_tones_2( def _merge_continuous_three_tones_2(
self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]: self, seg: List[Tuple[str, str]]
) -> List[Tuple[str, str]]:
new_seg = [] new_seg = []
sub_finals_list = [ sub_finals_list = [
lazy_pinyin( lazy_pinyin(word, neutral_tone_with_five=True, style=Style.FINALS_TONE3)
word, neutral_tone_with_five=True, style=Style.FINALS_TONE3)
for (word, pos) in seg for (word, pos) in seg
] ]
assert len(sub_finals_list) == len(seg) assert len(sub_finals_list) == len(seg)
merge_last = [False] * len(seg) merge_last = [False] * len(seg)
for i, (word, pos) in enumerate(seg): for i, (word, pos) in enumerate(seg):
if i - 1 >= 0 and sub_finals_list[i - 1][-1][-1] == "3" and sub_finals_list[i][0][-1] == "3" and not \ if (
merge_last[i - 1]: i - 1 >= 0
and sub_finals_list[i - 1][-1][-1] == "3"
and sub_finals_list[i][0][-1] == "3"
and not merge_last[i - 1]
):
# if the last word is reduplication, not merge, because reduplication need to be _neural_sandhi # if the last word is reduplication, not merge, because reduplication need to be _neural_sandhi
if not self._is_reduplication(seg[i - 1][0]) and len( if (
seg[i - 1][0]) + len(seg[i][0]) <= 3: not self._is_reduplication(seg[i - 1][0])
and len(seg[i - 1][0]) + len(seg[i][0]) <= 3
):
new_seg[-1][0] = new_seg[-1][0] + seg[i][0] new_seg[-1][0] = new_seg[-1][0] + seg[i][0]
merge_last[i] = True merge_last[i] = True
else: else:
@@ -313,14 +734,13 @@ class ToneSandhi():
def _merge_er(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]: def _merge_er(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
new_seg = [] new_seg = []
for i, (word, pos) in enumerate(seg): for i, (word, pos) in enumerate(seg):
if i - 1 >= 0 and word == "儿" and seg[i-1][0] != "#": if i - 1 >= 0 and word == "儿" and seg[i - 1][0] != "#":
new_seg[-1][0] = new_seg[-1][0] + seg[i][0] new_seg[-1][0] = new_seg[-1][0] + seg[i][0]
else: else:
new_seg.append([word, pos]) new_seg.append([word, pos])
return new_seg return new_seg
def _merge_reduplication( def _merge_reduplication(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
new_seg = [] new_seg = []
for i, (word, pos) in enumerate(seg): for i, (word, pos) in enumerate(seg):
if new_seg and word == new_seg[-1][0]: if new_seg and word == new_seg[-1][0]:
@@ -329,8 +749,7 @@ class ToneSandhi():
new_seg.append([word, pos]) new_seg.append([word, pos])
return new_seg return new_seg
def pre_merge_for_modify( def pre_merge_for_modify(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
seg = self._merge_bu(seg) seg = self._merge_bu(seg)
try: try:
seg = self._merge_yi(seg) seg = self._merge_yi(seg)
@@ -342,8 +761,7 @@ class ToneSandhi():
seg = self._merge_er(seg) seg = self._merge_er(seg)
return seg return seg
def modified_tone(self, word: str, pos: str, def modified_tone(self, word: str, pos: str, finals: List[str]) -> List[str]:
finals: List[str]) -> List[str]:
finals = self._bu_sandhi(word, finals) finals = self._bu_sandhi(word, finals)
finals = self._yi_sandhi(word, finals) finals = self._yi_sandhi(word, finals)
finals = self._neural_sandhi(word, pos, finals) finals = self._neural_sandhi(word, pos, finals)

425
train_ms.py
View File

@@ -14,37 +14,38 @@ from torch.nn.parallel import DistributedDataParallel as DDP
from torch.cuda.amp import autocast, GradScaler from torch.cuda.amp import autocast, GradScaler
from tqdm import tqdm from tqdm import tqdm
import logging import logging
logging.getLogger('numba').setLevel(logging.WARNING)
logging.getLogger("numba").setLevel(logging.WARNING)
import commons import commons
import utils import utils
from data_utils import ( from data_utils import (
TextAudioSpeakerLoader, TextAudioSpeakerLoader,
TextAudioSpeakerCollate, TextAudioSpeakerCollate,
DistributedBucketSampler DistributedBucketSampler,
) )
from models import ( from models import (
SynthesizerTrn, SynthesizerTrn,
MultiPeriodDiscriminator, MultiPeriodDiscriminator,
DurationDiscriminator, DurationDiscriminator,
) )
from losses import ( from losses import generator_loss, discriminator_loss, feature_loss, kl_loss
generator_loss,
discriminator_loss,
feature_loss,
kl_loss
)
from mel_processing import mel_spectrogram_torch, spec_to_mel_torch from mel_processing import mel_spectrogram_torch, spec_to_mel_torch
from text.symbols import symbols from text.symbols import symbols
torch.backends.cudnn.benchmark = True torch.backends.cudnn.benchmark = True
torch.backends.cuda.sdp_kernel("flash") torch.backends.cuda.sdp_kernel("flash")
torch.backends.cuda.enable_flash_sdp(True) torch.backends.cuda.enable_flash_sdp(True)
torch.backends.cuda.enable_mem_efficient_sdp(True) # Not avaliable if torch version is lower than 2.0 torch.backends.cuda.enable_mem_efficient_sdp(
True
) # Not avaliable if torch version is lower than 2.0
torch.backends.cuda.enable_math_sdp(True) torch.backends.cuda.enable_math_sdp(True)
global_step = 0 global_step = 0
def run(): def run():
dist.init_process_group(backend="nccl", init_method="env://") # Use torchrun instead of mp.spawn dist.init_process_group(
backend="nccl", init_method="env://"
) # Use torchrun instead of mp.spawn
rank = dist.get_rank() rank = dist.get_rank()
n_gpus = dist.get_world_size() n_gpus = dist.get_world_size()
hps = utils.get_hparams() hps = utils.get_hparams()
@@ -64,17 +65,34 @@ def run():
[32, 300, 400, 500, 600, 700, 800, 900, 1000], [32, 300, 400, 500, 600, 700, 800, 900, 1000],
num_replicas=n_gpus, num_replicas=n_gpus,
rank=rank, rank=rank,
shuffle=True) shuffle=True,
)
collate_fn = TextAudioSpeakerCollate() collate_fn = TextAudioSpeakerCollate()
train_loader = DataLoader(train_dataset, num_workers=16, shuffle=False, pin_memory=True, train_loader = DataLoader(
collate_fn=collate_fn, batch_sampler=train_sampler, train_dataset,
persistent_workers=True,prefetch_factor=4) #128G Memory suitable loader. num_workers=16,
shuffle=False,
pin_memory=True,
collate_fn=collate_fn,
batch_sampler=train_sampler,
persistent_workers=True,
prefetch_factor=4,
) # 128G Memory suitable loader.
if rank == 0: if rank == 0:
eval_dataset = TextAudioSpeakerLoader(hps.data.validation_files, hps.data) eval_dataset = TextAudioSpeakerLoader(hps.data.validation_files, hps.data)
eval_loader = DataLoader(eval_dataset, num_workers=0, shuffle=False, eval_loader = DataLoader(
batch_size=1, pin_memory=True, eval_dataset,
drop_last=False, collate_fn=collate_fn) num_workers=0,
if "use_noise_scaled_mas" in hps.model.keys() and hps.model.use_noise_scaled_mas == True: shuffle=False,
batch_size=1,
pin_memory=True,
drop_last=False,
collate_fn=collate_fn,
)
if (
"use_noise_scaled_mas" in hps.model.keys()
and hps.model.use_noise_scaled_mas == True
):
print("Using noise scaled MAS for VITS2") print("Using noise scaled MAS for VITS2")
use_noise_scaled_mas = True use_noise_scaled_mas = True
mas_noise_scale_initial = 0.01 mas_noise_scale_initial = 0.01
@@ -84,19 +102,27 @@ def run():
use_noise_scaled_mas = False use_noise_scaled_mas = False
mas_noise_scale_initial = 0.0 mas_noise_scale_initial = 0.0
noise_scale_delta = 0.0 noise_scale_delta = 0.0
if "use_duration_discriminator" in hps.model.keys() and hps.model.use_duration_discriminator == True: if (
"use_duration_discriminator" in hps.model.keys()
and hps.model.use_duration_discriminator == True
):
print("Using duration discriminator for VITS2") print("Using duration discriminator for VITS2")
use_duration_discriminator = True use_duration_discriminator = True
net_dur_disc = DurationDiscriminator( net_dur_disc = DurationDiscriminator(
hps.model.hidden_channels, hps.model.hidden_channels,
hps.model.hidden_channels, hps.model.hidden_channels,
3, 3,
0.1, 0.1,
gin_channels=hps.model.gin_channels if hps.data.n_speakers != 0 else 0, gin_channels=hps.model.gin_channels if hps.data.n_speakers != 0 else 0,
).cuda(rank) ).cuda(rank)
if "use_spk_conditioned_encoder" in hps.model.keys() and hps.model.use_spk_conditioned_encoder == True: if (
"use_spk_conditioned_encoder" in hps.model.keys()
and hps.model.use_spk_conditioned_encoder == True
):
if hps.data.n_speakers == 0: if hps.data.n_speakers == 0:
raise ValueError("n_speakers must be > 0 when using spk conditioned encoder to train multi-speaker model") raise ValueError(
"n_speakers must be > 0 when using spk conditioned encoder to train multi-speaker model"
)
use_spk_conditioned_encoder = True use_spk_conditioned_encoder = True
else: else:
print("Using normal encoder for VITS1") print("Using normal encoder for VITS1")
@@ -107,27 +133,31 @@ def run():
hps.data.filter_length // 2 + 1, hps.data.filter_length // 2 + 1,
hps.train.segment_size // hps.data.hop_length, hps.train.segment_size // hps.data.hop_length,
n_speakers=hps.data.n_speakers, n_speakers=hps.data.n_speakers,
mas_noise_scale_initial = mas_noise_scale_initial, mas_noise_scale_initial=mas_noise_scale_initial,
noise_scale_delta = noise_scale_delta, noise_scale_delta=noise_scale_delta,
**hps.model).cuda(rank) **hps.model,
).cuda(rank)
net_d = MultiPeriodDiscriminator(hps.model.use_spectral_norm).cuda(rank) net_d = MultiPeriodDiscriminator(hps.model.use_spectral_norm).cuda(rank)
optim_g = torch.optim.AdamW( optim_g = torch.optim.AdamW(
filter(lambda p: p.requires_grad, net_g.parameters()), filter(lambda p: p.requires_grad, net_g.parameters()),
hps.train.learning_rate, hps.train.learning_rate,
betas=hps.train.betas, betas=hps.train.betas,
eps=hps.train.eps) eps=hps.train.eps,
)
optim_d = torch.optim.AdamW( optim_d = torch.optim.AdamW(
net_d.parameters(), net_d.parameters(),
hps.train.learning_rate, hps.train.learning_rate,
betas=hps.train.betas, betas=hps.train.betas,
eps=hps.train.eps) eps=hps.train.eps,
)
if net_dur_disc is not None: if net_dur_disc is not None:
optim_dur_disc = torch.optim.AdamW( optim_dur_disc = torch.optim.AdamW(
net_dur_disc.parameters(), net_dur_disc.parameters(),
hps.train.learning_rate, hps.train.learning_rate,
betas=hps.train.betas, betas=hps.train.betas,
eps=hps.train.eps) eps=hps.train.eps,
)
else: else:
optim_dur_disc = None optim_dur_disc = None
net_g = DDP(net_g, device_ids=[rank], find_unused_parameters=True) net_g = DDP(net_g, device_ids=[rank], find_unused_parameters=True)
@@ -136,40 +166,82 @@ def run():
net_dur_disc = DDP(net_dur_disc, device_ids=[rank], find_unused_parameters=True) net_dur_disc = DDP(net_dur_disc, device_ids=[rank], find_unused_parameters=True)
try: try:
if net_dur_disc is not None: if net_dur_disc is not None:
_, _, _, epoch_str = utils.load_checkpoint(utils.latest_checkpoint_path(hps.model_dir, "DUR_*.pth"), net_dur_disc, optim_dur_disc, skip_optimizer=True) _, _, _, epoch_str = utils.load_checkpoint(
_, optim_g, _, epoch_str = utils.load_checkpoint(utils.latest_checkpoint_path(hps.model_dir, "G_*.pth"), net_g, utils.latest_checkpoint_path(hps.model_dir, "DUR_*.pth"),
optim_g, skip_optimizer=True) net_dur_disc,
_, optim_d, _, epoch_str = utils.load_checkpoint(utils.latest_checkpoint_path(hps.model_dir, "D_*.pth"), net_d, optim_dur_disc,
optim_d, skip_optimizer=True) skip_optimizer=True,
)
_, optim_g, _, epoch_str = utils.load_checkpoint(
utils.latest_checkpoint_path(hps.model_dir, "G_*.pth"),
net_g,
optim_g,
skip_optimizer=True,
)
_, optim_d, _, epoch_str = utils.load_checkpoint(
utils.latest_checkpoint_path(hps.model_dir, "D_*.pth"),
net_d,
optim_d,
skip_optimizer=True,
)
epoch_str = max(epoch_str, 1) epoch_str = max(epoch_str, 1)
global_step = (epoch_str - 1) * len(train_loader) global_step = (epoch_str - 1) * len(train_loader)
except Exception as e: except Exception as e:
print(e) print(e)
epoch_str = 1 epoch_str = 1
global_step = 0 global_step = 0
scheduler_g = torch.optim.lr_scheduler.ExponentialLR(
scheduler_g = torch.optim.lr_scheduler.ExponentialLR(optim_g, gamma=hps.train.lr_decay, last_epoch=epoch_str - 2) optim_g, gamma=hps.train.lr_decay, last_epoch=epoch_str - 2
scheduler_d = torch.optim.lr_scheduler.ExponentialLR(optim_d, gamma=hps.train.lr_decay, last_epoch=epoch_str - 2) )
scheduler_d = torch.optim.lr_scheduler.ExponentialLR(
optim_d, gamma=hps.train.lr_decay, last_epoch=epoch_str - 2
)
if net_dur_disc is not None: if net_dur_disc is not None:
scheduler_dur_disc = torch.optim.lr_scheduler.ExponentialLR(optim_dur_disc, gamma=hps.train.lr_decay, last_epoch=epoch_str-2) scheduler_dur_disc = torch.optim.lr_scheduler.ExponentialLR(
optim_dur_disc, gamma=hps.train.lr_decay, last_epoch=epoch_str - 2
)
else: else:
scheduler_dur_disc = None scheduler_dur_disc = None
scaler = GradScaler(enabled=hps.train.fp16_run) scaler = GradScaler(enabled=hps.train.fp16_run)
for epoch in range(epoch_str, hps.train.epochs + 1): for epoch in range(epoch_str, hps.train.epochs + 1):
if rank == 0: if rank == 0:
train_and_evaluate(rank, epoch, hps, [net_g, net_d, net_dur_disc], [optim_g, optim_d, optim_dur_disc], [scheduler_g, scheduler_d, scheduler_dur_disc], scaler, [train_loader, eval_loader], logger, [writer, writer_eval]) train_and_evaluate(
rank,
epoch,
hps,
[net_g, net_d, net_dur_disc],
[optim_g, optim_d, optim_dur_disc],
[scheduler_g, scheduler_d, scheduler_dur_disc],
scaler,
[train_loader, eval_loader],
logger,
[writer, writer_eval],
)
else: else:
train_and_evaluate(rank, epoch, hps, [net_g, net_d, net_dur_disc], [optim_g, optim_d, optim_dur_disc], [scheduler_g, scheduler_d, scheduler_dur_disc], scaler, [train_loader, None], None, None) train_and_evaluate(
rank,
epoch,
hps,
[net_g, net_d, net_dur_disc],
[optim_g, optim_d, optim_dur_disc],
[scheduler_g, scheduler_d, scheduler_dur_disc],
scaler,
[train_loader, None],
None,
None,
)
scheduler_g.step() scheduler_g.step()
scheduler_d.step() scheduler_d.step()
if net_dur_disc is not None: if net_dur_disc is not None:
scheduler_dur_disc.step() scheduler_dur_disc.step()
def train_and_evaluate(rank, epoch, hps, nets, optims, schedulers, scaler, loaders, logger, writers): def train_and_evaluate(
rank, epoch, hps, nets, optims, schedulers, scaler, loaders, logger, writers
):
net_g, net_d, net_dur_disc = nets net_g, net_d, net_dur_disc = nets
optim_g, optim_d, optim_dur_disc = optims optim_g, optim_d, optim_dur_disc = optims
scheduler_g, scheduler_d, scheduler_dur_disc = schedulers scheduler_g, scheduler_d, scheduler_dur_disc = schedulers
@@ -184,13 +256,34 @@ def train_and_evaluate(rank, epoch, hps, nets, optims, schedulers, scaler, loade
net_d.train() net_d.train()
if net_dur_disc is not None: if net_dur_disc is not None:
net_dur_disc.train() net_dur_disc.train()
for batch_idx, (x, x_lengths, spec, spec_lengths, y, y_lengths, speakers, tone, language, bert, ja_bert) in tqdm(enumerate(train_loader)): for batch_idx, (
x,
x_lengths,
spec,
spec_lengths,
y,
y_lengths,
speakers,
tone,
language,
bert,
ja_bert,
) in tqdm(enumerate(train_loader)):
if net_g.module.use_noise_scaled_mas: if net_g.module.use_noise_scaled_mas:
current_mas_noise_scale = net_g.module.mas_noise_scale_initial - net_g.module.noise_scale_delta * global_step current_mas_noise_scale = (
net_g.module.mas_noise_scale_initial
- net_g.module.noise_scale_delta * global_step
)
net_g.module.current_mas_noise_scale = max(current_mas_noise_scale, 0.0) net_g.module.current_mas_noise_scale = max(current_mas_noise_scale, 0.0)
x, x_lengths = x.cuda(rank, non_blocking=True), x_lengths.cuda(rank, non_blocking=True) x, x_lengths = x.cuda(rank, non_blocking=True), x_lengths.cuda(
spec, spec_lengths = spec.cuda(rank, non_blocking=True), spec_lengths.cuda(rank, non_blocking=True) rank, non_blocking=True
y, y_lengths = y.cuda(rank, non_blocking=True), y_lengths.cuda(rank, non_blocking=True) )
spec, spec_lengths = spec.cuda(rank, non_blocking=True), spec_lengths.cuda(
rank, non_blocking=True
)
y, y_lengths = y.cuda(rank, non_blocking=True), y_lengths.cuda(
rank, non_blocking=True
)
speakers = speakers.cuda(rank, non_blocking=True) speakers = speakers.cuda(rank, non_blocking=True)
tone = tone.cuda(rank, non_blocking=True) tone = tone.cuda(rank, non_blocking=True)
language = language.cuda(rank, non_blocking=True) language = language.cuda(rank, non_blocking=True)
@@ -198,16 +291,37 @@ def train_and_evaluate(rank, epoch, hps, nets, optims, schedulers, scaler, loade
ja_bert = ja_bert.cuda(rank, non_blocking=True) ja_bert = ja_bert.cuda(rank, non_blocking=True)
with autocast(enabled=hps.train.fp16_run): with autocast(enabled=hps.train.fp16_run):
y_hat, l_length, attn, ids_slice, x_mask, z_mask, \ (
(z, z_p, m_p, logs_p, m_q, logs_q), (hidden_x, logw, logw_) = net_g(x, x_lengths, spec, spec_lengths, speakers, tone, language, bert, ja_bert) y_hat,
l_length,
attn,
ids_slice,
x_mask,
z_mask,
(z, z_p, m_p, logs_p, m_q, logs_q),
(hidden_x, logw, logw_),
) = net_g(
x,
x_lengths,
spec,
spec_lengths,
speakers,
tone,
language,
bert,
ja_bert,
)
mel = spec_to_mel_torch( mel = spec_to_mel_torch(
spec, spec,
hps.data.filter_length, hps.data.filter_length,
hps.data.n_mel_channels, hps.data.n_mel_channels,
hps.data.sampling_rate, hps.data.sampling_rate,
hps.data.mel_fmin, hps.data.mel_fmin,
hps.data.mel_fmax) hps.data.mel_fmax,
y_mel = commons.slice_segments(mel, ids_slice, hps.train.segment_size // hps.data.hop_length) )
y_mel = commons.slice_segments(
mel, ids_slice, hps.train.segment_size // hps.data.hop_length
)
y_hat_mel = mel_spectrogram_torch( y_hat_mel = mel_spectrogram_torch(
y_hat.squeeze(1), y_hat.squeeze(1),
hps.data.filter_length, hps.data.filter_length,
@@ -216,26 +330,38 @@ def train_and_evaluate(rank, epoch, hps, nets, optims, schedulers, scaler, loade
hps.data.hop_length, hps.data.hop_length,
hps.data.win_length, hps.data.win_length,
hps.data.mel_fmin, hps.data.mel_fmin,
hps.data.mel_fmax hps.data.mel_fmax,
) )
y = commons.slice_segments(y, ids_slice * hps.data.hop_length, hps.train.segment_size) # slice y = commons.slice_segments(
y, ids_slice * hps.data.hop_length, hps.train.segment_size
) # slice
# Discriminator # Discriminator
y_d_hat_r, y_d_hat_g, _, _ = net_d(y, y_hat.detach()) y_d_hat_r, y_d_hat_g, _, _ = net_d(y, y_hat.detach())
with autocast(enabled=False): with autocast(enabled=False):
loss_disc, losses_disc_r, losses_disc_g = discriminator_loss(y_d_hat_r, y_d_hat_g) loss_disc, losses_disc_r, losses_disc_g = discriminator_loss(
y_d_hat_r, y_d_hat_g
)
loss_disc_all = loss_disc loss_disc_all = loss_disc
if net_dur_disc is not None: if net_dur_disc is not None:
y_dur_hat_r, y_dur_hat_g = net_dur_disc(hidden_x.detach(), x_mask.detach(), logw.detach(), logw_.detach()) y_dur_hat_r, y_dur_hat_g = net_dur_disc(
hidden_x.detach(), x_mask.detach(), logw.detach(), logw_.detach()
)
with autocast(enabled=False): with autocast(enabled=False):
# TODO: I think need to mean using the mask, but for now, just mean all # TODO: I think need to mean using the mask, but for now, just mean all
loss_dur_disc, losses_dur_disc_r, losses_dur_disc_g = discriminator_loss(y_dur_hat_r, y_dur_hat_g) (
loss_dur_disc,
losses_dur_disc_r,
losses_dur_disc_g,
) = discriminator_loss(y_dur_hat_r, y_dur_hat_g)
loss_dur_disc_all = loss_dur_disc loss_dur_disc_all = loss_dur_disc
optim_dur_disc.zero_grad() optim_dur_disc.zero_grad()
scaler.scale(loss_dur_disc_all).backward() scaler.scale(loss_dur_disc_all).backward()
scaler.unscale_(optim_dur_disc) scaler.unscale_(optim_dur_disc)
grad_norm_dur_disc = commons.clip_grad_value_(net_dur_disc.parameters(), None) grad_norm_dur_disc = commons.clip_grad_value_(
net_dur_disc.parameters(), None
)
scaler.step(optim_dur_disc) scaler.step(optim_dur_disc)
optim_d.zero_grad() optim_d.zero_grad()
@@ -269,51 +395,97 @@ def train_and_evaluate(rank, epoch, hps, nets, optims, schedulers, scaler, loade
if rank == 0: if rank == 0:
if global_step % hps.train.log_interval == 0: if global_step % hps.train.log_interval == 0:
lr = optim_g.param_groups[0]['lr'] lr = optim_g.param_groups[0]["lr"]
losses = [loss_disc, loss_gen, loss_fm, loss_mel, loss_dur, loss_kl] losses = [loss_disc, loss_gen, loss_fm, loss_mel, loss_dur, loss_kl]
logger.info('Train Epoch: {} [{:.0f}%]'.format( logger.info(
epoch, "Train Epoch: {} [{:.0f}%]".format(
100. * batch_idx / len(train_loader))) epoch, 100.0 * batch_idx / len(train_loader)
)
)
logger.info([x.item() for x in losses] + [global_step, lr]) logger.info([x.item() for x in losses] + [global_step, lr])
scalar_dict = {"loss/g/total": loss_gen_all, "loss/d/total": loss_disc_all, "learning_rate": lr, scalar_dict = {
"grad_norm_d": grad_norm_d, "grad_norm_g": grad_norm_g} "loss/g/total": loss_gen_all,
"loss/d/total": loss_disc_all,
"learning_rate": lr,
"grad_norm_d": grad_norm_d,
"grad_norm_g": grad_norm_g,
}
scalar_dict.update( scalar_dict.update(
{"loss/g/fm": loss_fm, "loss/g/mel": loss_mel, "loss/g/dur": loss_dur, "loss/g/kl": loss_kl}) {
scalar_dict.update({"loss/g/{}".format(i): v for i, v in enumerate(losses_gen)}) "loss/g/fm": loss_fm,
scalar_dict.update({"loss/d_r/{}".format(i): v for i, v in enumerate(losses_disc_r)}) "loss/g/mel": loss_mel,
scalar_dict.update({"loss/d_g/{}".format(i): v for i, v in enumerate(losses_disc_g)}) "loss/g/dur": loss_dur,
"loss/g/kl": loss_kl,
}
)
scalar_dict.update(
{"loss/g/{}".format(i): v for i, v in enumerate(losses_gen)}
)
scalar_dict.update(
{"loss/d_r/{}".format(i): v for i, v in enumerate(losses_disc_r)}
)
scalar_dict.update(
{"loss/d_g/{}".format(i): v for i, v in enumerate(losses_disc_g)}
)
image_dict = { image_dict = {
"slice/mel_org": utils.plot_spectrogram_to_numpy(y_mel[0].data.cpu().numpy()), "slice/mel_org": utils.plot_spectrogram_to_numpy(
"slice/mel_gen": utils.plot_spectrogram_to_numpy(y_hat_mel[0].data.cpu().numpy()), y_mel[0].data.cpu().numpy()
"all/mel": utils.plot_spectrogram_to_numpy(mel[0].data.cpu().numpy()), ),
"all/attn": utils.plot_alignment_to_numpy(attn[0, 0].data.cpu().numpy()) "slice/mel_gen": utils.plot_spectrogram_to_numpy(
y_hat_mel[0].data.cpu().numpy()
),
"all/mel": utils.plot_spectrogram_to_numpy(
mel[0].data.cpu().numpy()
),
"all/attn": utils.plot_alignment_to_numpy(
attn[0, 0].data.cpu().numpy()
),
} }
utils.summarize( utils.summarize(
writer=writer, writer=writer,
global_step=global_step, global_step=global_step,
images=image_dict, images=image_dict,
scalars=scalar_dict) scalars=scalar_dict,
)
if global_step % hps.train.eval_interval == 0: if global_step % hps.train.eval_interval == 0:
evaluate(hps, net_g, eval_loader, writer_eval) evaluate(hps, net_g, eval_loader, writer_eval)
utils.save_checkpoint(net_g, optim_g, hps.train.learning_rate, epoch, utils.save_checkpoint(
os.path.join(hps.model_dir, "G_{}.pth".format(global_step))) net_g,
utils.save_checkpoint(net_d, optim_d, hps.train.learning_rate, epoch, optim_g,
os.path.join(hps.model_dir, "D_{}.pth".format(global_step))) hps.train.learning_rate,
epoch,
os.path.join(hps.model_dir, "G_{}.pth".format(global_step)),
)
utils.save_checkpoint(
net_d,
optim_d,
hps.train.learning_rate,
epoch,
os.path.join(hps.model_dir, "D_{}.pth".format(global_step)),
)
if net_dur_disc is not None: if net_dur_disc is not None:
utils.save_checkpoint(net_dur_disc, optim_dur_disc, hps.train.learning_rate, epoch, os.path.join(hps.model_dir, "DUR_{}.pth".format(global_step))) utils.save_checkpoint(
keep_ckpts = getattr(hps.train, 'keep_ckpts', 5) net_dur_disc,
optim_dur_disc,
hps.train.learning_rate,
epoch,
os.path.join(hps.model_dir, "DUR_{}.pth".format(global_step)),
)
keep_ckpts = getattr(hps.train, "keep_ckpts", 5)
if keep_ckpts > 0: if keep_ckpts > 0:
utils.clean_checkpoints(path_to_models=hps.model_dir, n_ckpts_to_keep=keep_ckpts, sort_by_time=True) utils.clean_checkpoints(
path_to_models=hps.model_dir,
n_ckpts_to_keep=keep_ckpts,
sort_by_time=True,
)
global_step += 1 global_step += 1
if rank == 0: if rank == 0:
logger.info('====> Epoch: {}'.format(epoch)) logger.info("====> Epoch: {}".format(epoch))
def evaluate(hps, generator, eval_loader, writer_eval): def evaluate(hps, generator, eval_loader, writer_eval):
@@ -322,7 +494,19 @@ def evaluate(hps, generator, eval_loader, writer_eval):
audio_dict = {} audio_dict = {}
print("Evaluating ...") print("Evaluating ...")
with torch.no_grad(): with torch.no_grad():
for batch_idx, (x, x_lengths, spec, spec_lengths, y, y_lengths, speakers, tone, language, bert, ja_bert) in enumerate(eval_loader): for batch_idx, (
x,
x_lengths,
spec,
spec_lengths,
y,
y_lengths,
speakers,
tone,
language,
bert,
ja_bert,
) in enumerate(eval_loader):
x, x_lengths = x.cuda(), x_lengths.cuda() x, x_lengths = x.cuda(), x_lengths.cuda()
spec, spec_lengths = spec.cuda(), spec_lengths.cuda() spec, spec_lengths = spec.cuda(), spec_lengths.cuda()
y, y_lengths = y.cuda(), y_lengths.cuda() y, y_lengths = y.cuda(), y_lengths.cuda()
@@ -332,7 +516,18 @@ def evaluate(hps, generator, eval_loader, writer_eval):
tone = tone.cuda() tone = tone.cuda()
language = language.cuda() language = language.cuda()
for use_sdp in [True, False]: for use_sdp in [True, False]:
y_hat, attn, mask, *_ = generator.module.infer(x, x_lengths, speakers, tone, language, bert, ja_bert, y=spec, max_len=1000, sdp_ratio=0.0 if not use_sdp else 1.0) y_hat, attn, mask, *_ = generator.module.infer(
x,
x_lengths,
speakers,
tone,
language,
bert,
ja_bert,
y=spec,
max_len=1000,
sdp_ratio=0.0 if not use_sdp else 1.0,
)
y_hat_lengths = mask.sum([1, 2]).long() * hps.data.hop_length y_hat_lengths = mask.sum([1, 2]).long() * hps.data.hop_length
mel = spec_to_mel_torch( mel = spec_to_mel_torch(
@@ -341,7 +536,8 @@ def evaluate(hps, generator, eval_loader, writer_eval):
hps.data.n_mel_channels, hps.data.n_mel_channels,
hps.data.sampling_rate, hps.data.sampling_rate,
hps.data.mel_fmin, hps.data.mel_fmin,
hps.data.mel_fmax) hps.data.mel_fmax,
)
y_hat_mel = mel_spectrogram_torch( y_hat_mel = mel_spectrogram_torch(
y_hat.squeeze(1).float(), y_hat.squeeze(1).float(),
hps.data.filter_length, hps.data.filter_length,
@@ -350,25 +546,40 @@ def evaluate(hps, generator, eval_loader, writer_eval):
hps.data.hop_length, hps.data.hop_length,
hps.data.win_length, hps.data.win_length,
hps.data.mel_fmin, hps.data.mel_fmin,
hps.data.mel_fmax hps.data.mel_fmax,
) )
image_dict.update({ image_dict.update(
f"gen/mel_{batch_idx}": utils.plot_spectrogram_to_numpy(y_hat_mel[0].cpu().numpy()) {
}) f"gen/mel_{batch_idx}": utils.plot_spectrogram_to_numpy(
audio_dict.update({ y_hat_mel[0].cpu().numpy()
f"gen/audio_{batch_idx}_{use_sdp}": y_hat[0, :, :y_hat_lengths[0]] )
}) }
image_dict.update({f"gt/mel_{batch_idx}": utils.plot_spectrogram_to_numpy(mel[0].cpu().numpy())}) )
audio_dict.update({f"gt/audio_{batch_idx}": y[0, :, :y_lengths[0]]}) audio_dict.update(
{
f"gen/audio_{batch_idx}_{use_sdp}": y_hat[
0, :, : y_hat_lengths[0]
]
}
)
image_dict.update(
{
f"gt/mel_{batch_idx}": utils.plot_spectrogram_to_numpy(
mel[0].cpu().numpy()
)
}
)
audio_dict.update({f"gt/audio_{batch_idx}": y[0, :, : y_lengths[0]]})
utils.summarize( utils.summarize(
writer=writer_eval, writer=writer_eval,
global_step=global_step, global_step=global_step,
images=image_dict, images=image_dict,
audios=audio_dict, audios=audio_dict,
audio_sampling_rate=hps.data.sampling_rate audio_sampling_rate=hps.data.sampling_rate,
) )
generator.train() generator.train()
if __name__ == "__main__": if __name__ == "__main__":
run() run()

172
transforms.py
View File

@@ -9,66 +9,63 @@ DEFAULT_MIN_BIN_HEIGHT = 1e-3
DEFAULT_MIN_DERIVATIVE = 1e-3 DEFAULT_MIN_DERIVATIVE = 1e-3
def piecewise_rational_quadratic_transform(inputs, def piecewise_rational_quadratic_transform(
unnormalized_widths, inputs,
unnormalized_heights, unnormalized_widths,
unnormalized_derivatives, unnormalized_heights,
inverse=False, unnormalized_derivatives,
tails=None, inverse=False,
tail_bound=1., tails=None,
min_bin_width=DEFAULT_MIN_BIN_WIDTH, tail_bound=1.0,
min_bin_height=DEFAULT_MIN_BIN_HEIGHT, min_bin_width=DEFAULT_MIN_BIN_WIDTH,
min_derivative=DEFAULT_MIN_DERIVATIVE): min_bin_height=DEFAULT_MIN_BIN_HEIGHT,
min_derivative=DEFAULT_MIN_DERIVATIVE,
):
if tails is None: if tails is None:
spline_fn = rational_quadratic_spline spline_fn = rational_quadratic_spline
spline_kwargs = {} spline_kwargs = {}
else: else:
spline_fn = unconstrained_rational_quadratic_spline spline_fn = unconstrained_rational_quadratic_spline
spline_kwargs = { spline_kwargs = {"tails": tails, "tail_bound": tail_bound}
'tails': tails,
'tail_bound': tail_bound
}
outputs, logabsdet = spline_fn( outputs, logabsdet = spline_fn(
inputs=inputs, inputs=inputs,
unnormalized_widths=unnormalized_widths, unnormalized_widths=unnormalized_widths,
unnormalized_heights=unnormalized_heights, unnormalized_heights=unnormalized_heights,
unnormalized_derivatives=unnormalized_derivatives, unnormalized_derivatives=unnormalized_derivatives,
inverse=inverse, inverse=inverse,
min_bin_width=min_bin_width, min_bin_width=min_bin_width,
min_bin_height=min_bin_height, min_bin_height=min_bin_height,
min_derivative=min_derivative, min_derivative=min_derivative,
**spline_kwargs **spline_kwargs
) )
return outputs, logabsdet return outputs, logabsdet
def searchsorted(bin_locations, inputs, eps=1e-6): def searchsorted(bin_locations, inputs, eps=1e-6):
bin_locations[..., -1] += eps bin_locations[..., -1] += eps
return torch.sum( return torch.sum(inputs[..., None] >= bin_locations, dim=-1) - 1
inputs[..., None] >= bin_locations,
dim=-1
) - 1
def unconstrained_rational_quadratic_spline(inputs, def unconstrained_rational_quadratic_spline(
unnormalized_widths, inputs,
unnormalized_heights, unnormalized_widths,
unnormalized_derivatives, unnormalized_heights,
inverse=False, unnormalized_derivatives,
tails='linear', inverse=False,
tail_bound=1., tails="linear",
min_bin_width=DEFAULT_MIN_BIN_WIDTH, tail_bound=1.0,
min_bin_height=DEFAULT_MIN_BIN_HEIGHT, min_bin_width=DEFAULT_MIN_BIN_WIDTH,
min_derivative=DEFAULT_MIN_DERIVATIVE): min_bin_height=DEFAULT_MIN_BIN_HEIGHT,
min_derivative=DEFAULT_MIN_DERIVATIVE,
):
inside_interval_mask = (inputs >= -tail_bound) & (inputs <= tail_bound) inside_interval_mask = (inputs >= -tail_bound) & (inputs <= tail_bound)
outside_interval_mask = ~inside_interval_mask outside_interval_mask = ~inside_interval_mask
outputs = torch.zeros_like(inputs) outputs = torch.zeros_like(inputs)
logabsdet = torch.zeros_like(inputs) logabsdet = torch.zeros_like(inputs)
if tails == 'linear': if tails == "linear":
unnormalized_derivatives = F.pad(unnormalized_derivatives, pad=(1, 1)) unnormalized_derivatives = F.pad(unnormalized_derivatives, pad=(1, 1))
constant = np.log(np.exp(1 - min_derivative) - 1) constant = np.log(np.exp(1 - min_derivative) - 1)
unnormalized_derivatives[..., 0] = constant unnormalized_derivatives[..., 0] = constant
@@ -77,45 +74,57 @@ def unconstrained_rational_quadratic_spline(inputs,
outputs[outside_interval_mask] = inputs[outside_interval_mask] outputs[outside_interval_mask] = inputs[outside_interval_mask]
logabsdet[outside_interval_mask] = 0 logabsdet[outside_interval_mask] = 0
else: else:
raise RuntimeError('{} tails are not implemented.'.format(tails)) raise RuntimeError("{} tails are not implemented.".format(tails))
outputs[inside_interval_mask], logabsdet[inside_interval_mask] = rational_quadratic_spline( (
outputs[inside_interval_mask],
logabsdet[inside_interval_mask],
) = rational_quadratic_spline(
inputs=inputs[inside_interval_mask], inputs=inputs[inside_interval_mask],
unnormalized_widths=unnormalized_widths[inside_interval_mask, :], unnormalized_widths=unnormalized_widths[inside_interval_mask, :],
unnormalized_heights=unnormalized_heights[inside_interval_mask, :], unnormalized_heights=unnormalized_heights[inside_interval_mask, :],
unnormalized_derivatives=unnormalized_derivatives[inside_interval_mask, :], unnormalized_derivatives=unnormalized_derivatives[inside_interval_mask, :],
inverse=inverse, inverse=inverse,
left=-tail_bound, right=tail_bound, bottom=-tail_bound, top=tail_bound, left=-tail_bound,
right=tail_bound,
bottom=-tail_bound,
top=tail_bound,
min_bin_width=min_bin_width, min_bin_width=min_bin_width,
min_bin_height=min_bin_height, min_bin_height=min_bin_height,
min_derivative=min_derivative min_derivative=min_derivative,
) )
return outputs, logabsdet return outputs, logabsdet
def rational_quadratic_spline(inputs,
unnormalized_widths, def rational_quadratic_spline(
unnormalized_heights, inputs,
unnormalized_derivatives, unnormalized_widths,
inverse=False, unnormalized_heights,
left=0., right=1., bottom=0., top=1., unnormalized_derivatives,
min_bin_width=DEFAULT_MIN_BIN_WIDTH, inverse=False,
min_bin_height=DEFAULT_MIN_BIN_HEIGHT, left=0.0,
min_derivative=DEFAULT_MIN_DERIVATIVE): right=1.0,
bottom=0.0,
top=1.0,
min_bin_width=DEFAULT_MIN_BIN_WIDTH,
min_bin_height=DEFAULT_MIN_BIN_HEIGHT,
min_derivative=DEFAULT_MIN_DERIVATIVE,
):
if torch.min(inputs) < left or torch.max(inputs) > right: if torch.min(inputs) < left or torch.max(inputs) > right:
raise ValueError('Input to a transform is not within its domain') raise ValueError("Input to a transform is not within its domain")
num_bins = unnormalized_widths.shape[-1] num_bins = unnormalized_widths.shape[-1]
if min_bin_width * num_bins > 1.0: if min_bin_width * num_bins > 1.0:
raise ValueError('Minimal bin width too large for the number of bins') raise ValueError("Minimal bin width too large for the number of bins")
if min_bin_height * num_bins > 1.0: if min_bin_height * num_bins > 1.0:
raise ValueError('Minimal bin height too large for the number of bins') raise ValueError("Minimal bin height too large for the number of bins")
widths = F.softmax(unnormalized_widths, dim=-1) widths = F.softmax(unnormalized_widths, dim=-1)
widths = min_bin_width + (1 - min_bin_width * num_bins) * widths widths = min_bin_width + (1 - min_bin_width * num_bins) * widths
cumwidths = torch.cumsum(widths, dim=-1) cumwidths = torch.cumsum(widths, dim=-1)
cumwidths = F.pad(cumwidths, pad=(1, 0), mode='constant', value=0.0) cumwidths = F.pad(cumwidths, pad=(1, 0), mode="constant", value=0.0)
cumwidths = (right - left) * cumwidths + left cumwidths = (right - left) * cumwidths + left
cumwidths[..., 0] = left cumwidths[..., 0] = left
cumwidths[..., -1] = right cumwidths[..., -1] = right
@@ -126,7 +135,7 @@ def rational_quadratic_spline(inputs,
heights = F.softmax(unnormalized_heights, dim=-1) heights = F.softmax(unnormalized_heights, dim=-1)
heights = min_bin_height + (1 - min_bin_height * num_bins) * heights heights = min_bin_height + (1 - min_bin_height * num_bins) * heights
cumheights = torch.cumsum(heights, dim=-1) cumheights = torch.cumsum(heights, dim=-1)
cumheights = F.pad(cumheights, pad=(1, 0), mode='constant', value=0.0) cumheights = F.pad(cumheights, pad=(1, 0), mode="constant", value=0.0)
cumheights = (top - bottom) * cumheights + bottom cumheights = (top - bottom) * cumheights + bottom
cumheights[..., 0] = bottom cumheights[..., 0] = bottom
cumheights[..., -1] = top cumheights[..., -1] = top
@@ -150,15 +159,13 @@ def rational_quadratic_spline(inputs,
input_heights = heights.gather(-1, bin_idx)[..., 0] input_heights = heights.gather(-1, bin_idx)[..., 0]
if inverse: if inverse:
a = (((inputs - input_cumheights) * (input_derivatives a = (inputs - input_cumheights) * (
+ input_derivatives_plus_one input_derivatives + input_derivatives_plus_one - 2 * input_delta
- 2 * input_delta) ) + input_heights * (input_delta - input_derivatives)
+ input_heights * (input_delta - input_derivatives))) b = input_heights * input_derivatives - (inputs - input_cumheights) * (
b = (input_heights * input_derivatives input_derivatives + input_derivatives_plus_one - 2 * input_delta
- (inputs - input_cumheights) * (input_derivatives )
+ input_derivatives_plus_one c = -input_delta * (inputs - input_cumheights)
- 2 * input_delta))
c = - input_delta * (inputs - input_cumheights)
discriminant = b.pow(2) - 4 * a * c discriminant = b.pow(2) - 4 * a * c
assert (discriminant >= 0).all() assert (discriminant >= 0).all()
@@ -167,11 +174,15 @@ def rational_quadratic_spline(inputs,
outputs = root * input_bin_widths + input_cumwidths outputs = root * input_bin_widths + input_cumwidths
theta_one_minus_theta = root * (1 - root) theta_one_minus_theta = root * (1 - root)
denominator = input_delta + ((input_derivatives + input_derivatives_plus_one - 2 * input_delta) denominator = input_delta + (
* theta_one_minus_theta) (input_derivatives + input_derivatives_plus_one - 2 * input_delta)
derivative_numerator = input_delta.pow(2) * (input_derivatives_plus_one * root.pow(2) * theta_one_minus_theta
+ 2 * input_delta * theta_one_minus_theta )
+ input_derivatives * (1 - root).pow(2)) derivative_numerator = input_delta.pow(2) * (
input_derivatives_plus_one * root.pow(2)
+ 2 * input_delta * theta_one_minus_theta
+ input_derivatives * (1 - root).pow(2)
)
logabsdet = torch.log(derivative_numerator) - 2 * torch.log(denominator) logabsdet = torch.log(derivative_numerator) - 2 * torch.log(denominator)
return outputs, -logabsdet return outputs, -logabsdet
@@ -179,15 +190,20 @@ def rational_quadratic_spline(inputs,
theta = (inputs - input_cumwidths) / input_bin_widths theta = (inputs - input_cumwidths) / input_bin_widths
theta_one_minus_theta = theta * (1 - theta) theta_one_minus_theta = theta * (1 - theta)
numerator = input_heights * (input_delta * theta.pow(2) numerator = input_heights * (
+ input_derivatives * theta_one_minus_theta) input_delta * theta.pow(2) + input_derivatives * theta_one_minus_theta
denominator = input_delta + ((input_derivatives + input_derivatives_plus_one - 2 * input_delta) )
* theta_one_minus_theta) denominator = input_delta + (
(input_derivatives + input_derivatives_plus_one - 2 * input_delta)
* theta_one_minus_theta
)
outputs = input_cumheights + numerator / denominator outputs = input_cumheights + numerator / denominator
derivative_numerator = input_delta.pow(2) * (input_derivatives_plus_one * theta.pow(2) derivative_numerator = input_delta.pow(2) * (
+ 2 * input_delta * theta_one_minus_theta input_derivatives_plus_one * theta.pow(2)
+ input_derivatives * (1 - theta).pow(2)) + 2 * input_delta * theta_one_minus_theta
+ input_derivatives * (1 - theta).pow(2)
)
logabsdet = torch.log(derivative_numerator) - 2 * torch.log(denominator) logabsdet = torch.log(derivative_numerator) - 2 * torch.log(denominator)
return outputs, logabsdet return outputs, logabsdet

171
utils.py
View File

@@ -16,20 +16,24 @@ logger = logging.getLogger(__name__)
def load_checkpoint(checkpoint_path, model, optimizer=None, skip_optimizer=False): def load_checkpoint(checkpoint_path, model, optimizer=None, skip_optimizer=False):
assert os.path.isfile(checkpoint_path) assert os.path.isfile(checkpoint_path)
checkpoint_dict = torch.load(checkpoint_path, map_location='cpu') checkpoint_dict = torch.load(checkpoint_path, map_location="cpu")
iteration = checkpoint_dict['iteration'] iteration = checkpoint_dict["iteration"]
learning_rate = checkpoint_dict['learning_rate'] learning_rate = checkpoint_dict["learning_rate"]
if optimizer is not None and not skip_optimizer and checkpoint_dict['optimizer'] is not None: if (
optimizer.load_state_dict(checkpoint_dict['optimizer']) optimizer is not None
elif optimizer is None and not skip_optimizer: and not skip_optimizer
#else: Disable this line if Infer and resume checkpoint,then enable the line upper and checkpoint_dict["optimizer"] is not None
):
optimizer.load_state_dict(checkpoint_dict["optimizer"])
elif optimizer is None and not skip_optimizer:
# else: Disable this line if Infer and resume checkpoint,then enable the line upper
new_opt_dict = optimizer.state_dict() new_opt_dict = optimizer.state_dict()
new_opt_dict_params = new_opt_dict['param_groups'][0]['params'] new_opt_dict_params = new_opt_dict["param_groups"][0]["params"]
new_opt_dict['param_groups'] = checkpoint_dict['optimizer']['param_groups'] new_opt_dict["param_groups"] = checkpoint_dict["optimizer"]["param_groups"]
new_opt_dict['param_groups'][0]['params'] = new_opt_dict_params new_opt_dict["param_groups"][0]["params"] = new_opt_dict_params
optimizer.load_state_dict(new_opt_dict) optimizer.load_state_dict(new_opt_dict)
saved_state_dict = checkpoint_dict['model'] saved_state_dict = checkpoint_dict["model"]
if hasattr(model, 'module'): if hasattr(model, "module"):
state_dict = model.module.state_dict() state_dict = model.module.state_dict()
else: else:
state_dict = model.state_dict() state_dict = model.state_dict()
@@ -38,39 +42,59 @@ def load_checkpoint(checkpoint_path, model, optimizer=None, skip_optimizer=False
try: try:
# assert "emb_g" not in k # assert "emb_g" not in k
new_state_dict[k] = saved_state_dict[k] new_state_dict[k] = saved_state_dict[k]
assert saved_state_dict[k].shape == v.shape, (saved_state_dict[k].shape, v.shape) assert saved_state_dict[k].shape == v.shape, (
saved_state_dict[k].shape,
v.shape,
)
except: except:
logger.error("%s is not in the checkpoint" % k) logger.error("%s is not in the checkpoint" % k)
new_state_dict[k] = v new_state_dict[k] = v
if hasattr(model, 'module'): if hasattr(model, "module"):
model.module.load_state_dict(new_state_dict, strict=False) model.module.load_state_dict(new_state_dict, strict=False)
else: else:
model.load_state_dict(new_state_dict, strict=False) model.load_state_dict(new_state_dict, strict=False)
logger.info("Loaded checkpoint '{}' (iteration {})".format( logger.info(
checkpoint_path, iteration)) "Loaded checkpoint '{}' (iteration {})".format(checkpoint_path, iteration)
)
return model, optimizer, learning_rate, iteration return model, optimizer, learning_rate, iteration
def save_checkpoint(model, optimizer, learning_rate, iteration, checkpoint_path): def save_checkpoint(model, optimizer, learning_rate, iteration, checkpoint_path):
logger.info("Saving model and optimizer state at iteration {} to {}".format( logger.info(
iteration, checkpoint_path)) "Saving model and optimizer state at iteration {} to {}".format(
if hasattr(model, 'module'): iteration, checkpoint_path
)
)
if hasattr(model, "module"):
state_dict = model.module.state_dict() state_dict = model.module.state_dict()
else: else:
state_dict = model.state_dict() state_dict = model.state_dict()
torch.save({'model': state_dict, torch.save(
'iteration': iteration, {
'optimizer': optimizer.state_dict(), "model": state_dict,
'learning_rate': learning_rate}, checkpoint_path) "iteration": iteration,
"optimizer": optimizer.state_dict(),
"learning_rate": learning_rate,
},
checkpoint_path,
)
def summarize(writer, global_step, scalars={}, histograms={}, images={}, audios={}, audio_sampling_rate=22050): def summarize(
writer,
global_step,
scalars={},
histograms={},
images={},
audios={},
audio_sampling_rate=22050,
):
for k, v in scalars.items(): for k, v in scalars.items():
writer.add_scalar(k, v, global_step) writer.add_scalar(k, v, global_step)
for k, v in histograms.items(): for k, v in histograms.items():
writer.add_histogram(k, v, global_step) writer.add_histogram(k, v, global_step)
for k, v in images.items(): for k, v in images.items():
writer.add_image(k, v, global_step, dataformats='HWC') writer.add_image(k, v, global_step, dataformats="HWC")
for k, v in audios.items(): for k, v in audios.items():
writer.add_audio(k, v, global_step, audio_sampling_rate) writer.add_audio(k, v, global_step, audio_sampling_rate)
@@ -86,23 +110,23 @@ def plot_spectrogram_to_numpy(spectrogram):
global MATPLOTLIB_FLAG global MATPLOTLIB_FLAG
if not MATPLOTLIB_FLAG: if not MATPLOTLIB_FLAG:
import matplotlib import matplotlib
matplotlib.use("Agg") matplotlib.use("Agg")
MATPLOTLIB_FLAG = True MATPLOTLIB_FLAG = True
mpl_logger = logging.getLogger('matplotlib') mpl_logger = logging.getLogger("matplotlib")
mpl_logger.setLevel(logging.WARNING) mpl_logger.setLevel(logging.WARNING)
import matplotlib.pylab as plt import matplotlib.pylab as plt
import numpy as np import numpy as np
fig, ax = plt.subplots(figsize=(10, 2)) fig, ax = plt.subplots(figsize=(10, 2))
im = ax.imshow(spectrogram, aspect="auto", origin="lower", im = ax.imshow(spectrogram, aspect="auto", origin="lower", interpolation="none")
interpolation='none')
plt.colorbar(im, ax=ax) plt.colorbar(im, ax=ax)
plt.xlabel("Frames") plt.xlabel("Frames")
plt.ylabel("Channels") plt.ylabel("Channels")
plt.tight_layout() plt.tight_layout()
fig.canvas.draw() fig.canvas.draw()
data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='') data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep="")
data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,)) data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
plt.close() plt.close()
return data return data
@@ -112,26 +136,28 @@ def plot_alignment_to_numpy(alignment, info=None):
global MATPLOTLIB_FLAG global MATPLOTLIB_FLAG
if not MATPLOTLIB_FLAG: if not MATPLOTLIB_FLAG:
import matplotlib import matplotlib
matplotlib.use("Agg") matplotlib.use("Agg")
MATPLOTLIB_FLAG = True MATPLOTLIB_FLAG = True
mpl_logger = logging.getLogger('matplotlib') mpl_logger = logging.getLogger("matplotlib")
mpl_logger.setLevel(logging.WARNING) mpl_logger.setLevel(logging.WARNING)
import matplotlib.pylab as plt import matplotlib.pylab as plt
import numpy as np import numpy as np
fig, ax = plt.subplots(figsize=(6, 4)) fig, ax = plt.subplots(figsize=(6, 4))
im = ax.imshow(alignment.transpose(), aspect='auto', origin='lower', im = ax.imshow(
interpolation='none') alignment.transpose(), aspect="auto", origin="lower", interpolation="none"
)
fig.colorbar(im, ax=ax) fig.colorbar(im, ax=ax)
xlabel = 'Decoder timestep' xlabel = "Decoder timestep"
if info is not None: if info is not None:
xlabel += '\n\n' + info xlabel += "\n\n" + info
plt.xlabel(xlabel) plt.xlabel(xlabel)
plt.ylabel('Encoder timestep') plt.ylabel("Encoder timestep")
plt.tight_layout() plt.tight_layout()
fig.canvas.draw() fig.canvas.draw()
data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='') data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep="")
data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,)) data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
plt.close() plt.close()
return data return data
@@ -143,17 +169,21 @@ def load_wav_to_torch(full_path):
def load_filepaths_and_text(filename, split="|"): def load_filepaths_and_text(filename, split="|"):
with open(filename, encoding='utf-8') as f: with open(filename, encoding="utf-8") as f:
filepaths_and_text = [line.strip().split(split) for line in f] filepaths_and_text = [line.strip().split(split) for line in f]
return filepaths_and_text return filepaths_and_text
def get_hparams(init=True): def get_hparams(init=True):
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('-c', '--config', type=str, default="./configs/base.json", parser.add_argument(
help='JSON file for configuration') "-c",
parser.add_argument('-m', '--model', type=str, required=True, "--config",
help='Model name') type=str,
default="./configs/base.json",
help="JSON file for configuration",
)
parser.add_argument("-m", "--model", type=str, required=True, help="Model name")
args = parser.parse_args() args = parser.parse_args()
model_dir = os.path.join("./logs", args.model) model_dir = os.path.join("./logs", args.model)
@@ -178,31 +208,41 @@ def get_hparams(init=True):
return hparams return hparams
def clean_checkpoints(path_to_models='logs/44k/', n_ckpts_to_keep=2, sort_by_time=True): def clean_checkpoints(path_to_models="logs/44k/", n_ckpts_to_keep=2, sort_by_time=True):
"""Freeing up space by deleting saved ckpts """Freeing up space by deleting saved ckpts
Arguments: Arguments:
path_to_models -- Path to the model directory path_to_models -- Path to the model directory
n_ckpts_to_keep -- Number of ckpts to keep, excluding G_0.pth and D_0.pth n_ckpts_to_keep -- Number of ckpts to keep, excluding G_0.pth and D_0.pth
sort_by_time -- True -> chronologically delete ckpts sort_by_time -- True -> chronologically delete ckpts
False -> lexicographically delete ckpts False -> lexicographically delete ckpts
""" """
import re import re
ckpts_files = [f for f in os.listdir(path_to_models) if os.path.isfile(os.path.join(path_to_models, f))]
name_key = (lambda _f: int(re.compile('._(\d+)\.pth').match(_f).group(1))) ckpts_files = [
time_key = (lambda _f: os.path.getmtime(os.path.join(path_to_models, _f))) f
for f in os.listdir(path_to_models)
if os.path.isfile(os.path.join(path_to_models, f))
]
name_key = lambda _f: int(re.compile("._(\d+)\.pth").match(_f).group(1))
time_key = lambda _f: os.path.getmtime(os.path.join(path_to_models, _f))
sort_key = time_key if sort_by_time else name_key sort_key = time_key if sort_by_time else name_key
x_sorted = lambda _x: sorted([f for f in ckpts_files if f.startswith(_x) and not f.endswith('_0.pth')], x_sorted = lambda _x: sorted(
key=sort_key) [f for f in ckpts_files if f.startswith(_x) and not f.endswith("_0.pth")],
to_del = [os.path.join(path_to_models, fn) for fn in key=sort_key,
(x_sorted('G')[:-n_ckpts_to_keep] + x_sorted('D')[:-n_ckpts_to_keep])] )
to_del = [
os.path.join(path_to_models, fn)
for fn in (x_sorted("G")[:-n_ckpts_to_keep] + x_sorted("D")[:-n_ckpts_to_keep])
]
del_info = lambda fn: logger.info(f".. Free up space by deleting ckpt {fn}") del_info = lambda fn: logger.info(f".. Free up space by deleting ckpt {fn}")
del_routine = lambda x: [os.remove(x), del_info(x)] del_routine = lambda x: [os.remove(x), del_info(x)]
rs = [del_routine(fn) for fn in to_del] rs = [del_routine(fn) for fn in to_del]
def get_hparams_from_dir(model_dir): def get_hparams_from_dir(model_dir):
config_save_path = os.path.join(model_dir, "config.json") config_save_path = os.path.join(model_dir, "config.json")
with open(config_save_path, "r", encoding='utf-8') as f: with open(config_save_path, "r", encoding="utf-8") as f:
data = f.read() data = f.read()
config = json.loads(data) config = json.loads(data)
@@ -212,7 +252,7 @@ def get_hparams_from_dir(model_dir):
def get_hparams_from_file(config_path): def get_hparams_from_file(config_path):
with open(config_path, "r", encoding='utf-8') as f: with open(config_path, "r", encoding="utf-8") as f:
data = f.read() data = f.read()
config = json.loads(data) config = json.loads(data)
@@ -223,9 +263,11 @@ def get_hparams_from_file(config_path):
def check_git_hash(model_dir): def check_git_hash(model_dir):
source_dir = os.path.dirname(os.path.realpath(__file__)) source_dir = os.path.dirname(os.path.realpath(__file__))
if not os.path.exists(os.path.join(source_dir, ".git")): if not os.path.exists(os.path.join(source_dir, ".git")):
logger.warn("{} is not a git repository, therefore hash value comparison will be ignored.".format( logger.warn(
source_dir "{} is not a git repository, therefore hash value comparison will be ignored.".format(
)) source_dir
)
)
return return
cur_hash = subprocess.getoutput("git rev-parse HEAD") cur_hash = subprocess.getoutput("git rev-parse HEAD")
@@ -234,8 +276,11 @@ def check_git_hash(model_dir):
if os.path.exists(path): if os.path.exists(path):
saved_hash = open(path).read() saved_hash = open(path).read()
if saved_hash != cur_hash: if saved_hash != cur_hash:
logger.warn("git hash values are different. {}(saved) != {}(current)".format( logger.warn(
saved_hash[:8], cur_hash[:8])) "git hash values are different. {}(saved) != {}(current)".format(
saved_hash[:8], cur_hash[:8]
)
)
else: else:
open(path, "w").write(cur_hash) open(path, "w").write(cur_hash)
@@ -255,7 +300,7 @@ def get_logger(model_dir, filename="train.log"):
return logger return logger
class HParams(): class HParams:
def __init__(self, **kwargs): def __init__(self, **kwargs):
for k, v in kwargs.items(): for k, v in kwargs.items():
if type(v) == dict: if type(v) == dict:

128
webui.py
View File

@@ -10,7 +10,9 @@ logging.getLogger("markdown_it").setLevel(logging.WARNING)
logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("matplotlib").setLevel(logging.WARNING) logging.getLogger("matplotlib").setLevel(logging.WARNING)
logging.basicConfig(level=logging.INFO, format="| %(name)s | %(levelname)s | %(message)s") logging.basicConfig(
level=logging.INFO, format="| %(name)s | %(levelname)s | %(message)s"
)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -27,6 +29,7 @@ import webbrowser
net_g = None net_g = None
def get_text(text, language_str, hps): def get_text(text, language_str, hps):
norm_text, phone, tone, word2ph = clean_text(text, language_str) norm_text, phone, tone, word2ph = clean_text(text, language_str)
phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str) phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
@@ -42,52 +45,101 @@ def get_text(text, language_str, hps):
del word2ph del word2ph
assert bert.shape[-1] == len(phone), phone assert bert.shape[-1] == len(phone), phone
if language_str=='ZH': if language_str == "ZH":
bert = bert bert = bert
ja_bert = torch.zeros(768, len(phone)) ja_bert = torch.zeros(768, len(phone))
elif language_str=="JA": elif language_str == "JA":
ja_bert = bert ja_bert = bert
bert = torch.zeros(1024, len(phone)) bert = torch.zeros(1024, len(phone))
else: else:
bert = torch.zeros(1024, len(phone)) bert = torch.zeros(1024, len(phone))
ja_bert = torch.zeros(768, len(phone)) ja_bert = torch.zeros(768, len(phone))
assert bert.shape[-1] == len(phone), ( assert bert.shape[-1] == len(phone), (
bert.shape, len(phone), sum(word2ph), p1, p2, t1, t2, pold, pold2, word2ph, text, w2pho) bert.shape,
len(phone),
sum(word2ph),
p1,
p2,
t1,
t2,
pold,
pold2,
word2ph,
text,
w2pho,
)
phone = torch.LongTensor(phone) phone = torch.LongTensor(phone)
tone = torch.LongTensor(tone) tone = torch.LongTensor(tone)
language = torch.LongTensor(language) language = torch.LongTensor(language)
return bert, ja_bert, phone, tone, language return bert, ja_bert, phone, tone, language
def infer(text, sdp_ratio, noise_scale, noise_scale_w, length_scale, sid, language): def infer(text, sdp_ratio, noise_scale, noise_scale_w, length_scale, sid, language):
global net_g global net_g
bert, ja_bert, phones, tones, lang_ids = get_text(text, language, hps) bert, ja_bert, phones, tones, lang_ids = get_text(text, language, hps)
with torch.no_grad(): with torch.no_grad():
x_tst=phones.to(device).unsqueeze(0) x_tst = phones.to(device).unsqueeze(0)
tones=tones.to(device).unsqueeze(0) tones = tones.to(device).unsqueeze(0)
lang_ids=lang_ids.to(device).unsqueeze(0) lang_ids = lang_ids.to(device).unsqueeze(0)
bert = bert.to(device).unsqueeze(0) bert = bert.to(device).unsqueeze(0)
ja_bert = ja_bert.to(device).unsqueeze(0) ja_bert = ja_bert.to(device).unsqueeze(0)
x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device) x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
del phones del phones
speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device) speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
audio = net_g.infer(x_tst, x_tst_lengths, speakers, tones, lang_ids, bert, ja_bert, sdp_ratio=sdp_ratio audio = (
, noise_scale=noise_scale, noise_scale_w=noise_scale_w, length_scale=length_scale)[0][0,0].data.cpu().float().numpy() net_g.infer(
x_tst,
x_tst_lengths,
speakers,
tones,
lang_ids,
bert,
ja_bert,
sdp_ratio=sdp_ratio,
noise_scale=noise_scale,
noise_scale_w=noise_scale_w,
length_scale=length_scale,
)[0][0, 0]
.data.cpu()
.float()
.numpy()
)
del x_tst, tones, lang_ids, bert, x_tst_lengths, speakers del x_tst, tones, lang_ids, bert, x_tst_lengths, speakers
return audio return audio
def tts_fn(text, speaker, sdp_ratio, noise_scale, noise_scale_w, length_scale, language):
def tts_fn(
text, speaker, sdp_ratio, noise_scale, noise_scale_w, length_scale, language
):
with torch.no_grad(): with torch.no_grad():
audio = infer(text, sdp_ratio=sdp_ratio, noise_scale=noise_scale, noise_scale_w=noise_scale_w, length_scale=length_scale, sid=speaker, language=language) audio = infer(
text,
sdp_ratio=sdp_ratio,
noise_scale=noise_scale,
noise_scale_w=noise_scale_w,
length_scale=length_scale,
sid=speaker,
language=language,
)
torch.cuda.empty_cache() torch.cuda.empty_cache()
return "Success", (hps.data.sampling_rate, audio) return "Success", (hps.data.sampling_rate, audio)
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument("-m", "--model", default="./logs/as/G_8000.pth", help="path of your model") parser.add_argument(
parser.add_argument("-c", "--config", default="./configs/config.json", help="path of your config file") "-m", "--model", default="./logs/as/G_8000.pth", help="path of your model"
)
parser.add_argument(
"-c",
"--config",
default="./configs/config.json",
help="path of your config file",
)
parser.add_argument("--share", default=False, help="make link public") parser.add_argument("--share", default=False, help="make link public")
parser.add_argument("-d", "--debug", action="store_true", help="enable DEBUG-LEVEL log") parser.add_argument(
"-d", "--debug", action="store_true", help="enable DEBUG-LEVEL log"
)
args = parser.parse_args() args = parser.parse_args()
if args.debug: if args.debug:
@@ -109,33 +161,51 @@ if __name__ == "__main__":
hps.data.filter_length // 2 + 1, hps.data.filter_length // 2 + 1,
hps.train.segment_size // hps.data.hop_length, hps.train.segment_size // hps.data.hop_length,
n_speakers=hps.data.n_speakers, n_speakers=hps.data.n_speakers,
**hps.model).to(device) **hps.model
).to(device)
_ = net_g.eval() _ = net_g.eval()
_ = utils.load_checkpoint(args.model, net_g, None, skip_optimizer=True) _ = utils.load_checkpoint(args.model, net_g, None, skip_optimizer=True)
speaker_ids = hps.data.spk2id speaker_ids = hps.data.spk2id
speakers = list(speaker_ids.keys()) speakers = list(speaker_ids.keys())
languages = ["ZH","JA"] languages = ["ZH", "JA"]
with gr.Blocks() as app: with gr.Blocks() as app:
with gr.Row(): with gr.Row():
with gr.Column(): with gr.Column():
text = gr.TextArea(label="Text", placeholder="Input Text Here", text = gr.TextArea(
value="吃葡萄不吐葡萄皮，不吃葡萄倒吐葡萄皮。") label="Text",
speaker = gr.Dropdown(choices=speakers, value=speakers[0], label='Speaker') placeholder="Input Text Here",
sdp_ratio = gr.Slider(minimum=0, maximum=1, value=0.2, step=0.1, label='SDP Ratio') value="吃葡萄不吐葡萄皮，不吃葡萄倒吐葡萄皮。",
noise_scale = gr.Slider(minimum=0.1, maximum=2, value=0.6, step=0.1, label='Noise Scale') )
noise_scale_w = gr.Slider(minimum=0.1, maximum=2, value=0.8, step=0.1, label='Noise Scale W') speaker = gr.Dropdown(
length_scale = gr.Slider(minimum=0.1, maximum=2, value=1, step=0.1, label='Length Scale') choices=speakers, value=speakers[0], label="Speaker"
language = gr.Dropdown(choices=languages, value=languages[0], label='Language') )
sdp_ratio = gr.Slider(
minimum=0, maximum=1, value=0.2, step=0.1, label="SDP Ratio"
)
noise_scale = gr.Slider(
minimum=0.1, maximum=2, value=0.6, step=0.1, label="Noise Scale"
)
noise_scale_w = gr.Slider(
minimum=0.1, maximum=2, value=0.8, step=0.1, label="Noise Scale W"
)
length_scale = gr.Slider(
minimum=0.1, maximum=2, value=1, step=0.1, label="Length Scale"
)
language = gr.Dropdown(
choices=languages, value=languages[0], label="Language"
)
btn = gr.Button("Generate!", variant="primary") btn = gr.Button("Generate!", variant="primary")
with gr.Column(): with gr.Column():
text_output = gr.Textbox(label="Message") text_output = gr.Textbox(label="Message")
audio_output = gr.Audio(label="Output Audio") audio_output = gr.Audio(label="Output Audio")
btn.click(tts_fn, btn.click(
inputs=[text, speaker, sdp_ratio, noise_scale, noise_scale_w, length_scale], tts_fn,
outputs=[text_output, audio_output]) inputs=[text, speaker, sdp_ratio, noise_scale, noise_scale_w, length_scale],
outputs=[text_output, audio_output],
)
webbrowser.open("http://127.0.0.1:7860") webbrowser.open("http://127.0.0.1:7860")
app.launch(share=args.share) app.launch(share=args.share)