tuna2134/sbv2-v2
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Format code

Browse Source
This commit is contained in:
github-actions[bot]
2023-09-06 13:42:08 +00:00
parent d82ba3457a
commit 92f3fdd11f
26 changed files with 2938 additions and 1453 deletions
Download Patch File Download Diff File Expand all files Collapse all files

181
attentions.py
View File

@@ -9,6 +9,7 @@ import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class LayerNorm(nn.Module): class LayerNorm(nn.Module):
def __init__(self, channels, eps=1e-5): def __init__(self, channels, eps=1e-5):
super().__init__() super().__init__()
@@ -24,7 +25,6 @@ class LayerNorm(nn.Module):
return x.transpose(1, -1) return x.transpose(1, -1)
@torch.jit.script @torch.jit.script
def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels): def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
n_channels_int = n_channels[0] n_channels_int = n_channels[0]
@@ -34,8 +34,20 @@ def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
acts = t_act * s_act acts = t_act * s_act
return acts return acts
class Encoder(nn.Module): class Encoder(nn.Module):
def __init__(self, hidden_channels, filter_channels, n_heads, n_layers, kernel_size=1, p_dropout=0., window_size=4, isflow = True, **kwargs): def __init__(
self,
hidden_channels,
filter_channels,
n_heads,
n_layers,
kernel_size=1,
p_dropout=0.0,
window_size=4,
isflow=True,
**kwargs
):
super().__init__() super().__init__()
self.hidden_channels = hidden_channels self.hidden_channels = hidden_channels
self.filter_channels = filter_channels self.filter_channels = filter_channels
@@ -50,24 +62,45 @@ class Encoder(nn.Module):
# self.cond_layer = weight_norm(cond_layer, name='weight') # self.cond_layer = weight_norm(cond_layer, name='weight')
# self.gin_channels = 256 # self.gin_channels = 256
self.cond_layer_idx = self.n_layers self.cond_layer_idx = self.n_layers
if 'gin_channels' in kwargs: if "gin_channels" in kwargs:
self.gin_channels = kwargs['gin_channels'] self.gin_channels = kwargs["gin_channels"]
if self.gin_channels != 0: if self.gin_channels != 0:
self.spk_emb_linear = nn.Linear(self.gin_channels, self.hidden_channels) self.spk_emb_linear = nn.Linear(self.gin_channels, self.hidden_channels)
# vits2 says 3rd block, so idx is 2 by default # vits2 says 3rd block, so idx is 2 by default
self.cond_layer_idx = kwargs['cond_layer_idx'] if 'cond_layer_idx' in kwargs else 2 self.cond_layer_idx = (
kwargs["cond_layer_idx"] if "cond_layer_idx" in kwargs else 2
)
logging.debug(self.gin_channels, self.cond_layer_idx) logging.debug(self.gin_channels, self.cond_layer_idx)
assert self.cond_layer_idx < self.n_layers, 'cond_layer_idx should be less than n_layers' assert (
self.cond_layer_idx < self.n_layers
), "cond_layer_idx should be less than n_layers"
self.drop = nn.Dropout(p_dropout) self.drop = nn.Dropout(p_dropout)
self.attn_layers = nn.ModuleList() self.attn_layers = nn.ModuleList()
self.norm_layers_1 = nn.ModuleList() self.norm_layers_1 = nn.ModuleList()
self.ffn_layers = nn.ModuleList() self.ffn_layers = nn.ModuleList()
self.norm_layers_2 = nn.ModuleList() self.norm_layers_2 = nn.ModuleList()
for i in range(self.n_layers): for i in range(self.n_layers):
self.attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout, window_size=window_size)) self.attn_layers.append(
MultiHeadAttention(
hidden_channels,
hidden_channels,
n_heads,
p_dropout=p_dropout,
window_size=window_size,
)
)
self.norm_layers_1.append(LayerNorm(hidden_channels)) self.norm_layers_1.append(LayerNorm(hidden_channels))
self.ffn_layers.append(FFN(hidden_channels, hidden_channels, filter_channels, kernel_size, p_dropout=p_dropout)) self.ffn_layers.append(
FFN(
hidden_channels,
hidden_channels,
filter_channels,
kernel_size,
p_dropout=p_dropout,
)
)
self.norm_layers_2.append(LayerNorm(hidden_channels)) self.norm_layers_2.append(LayerNorm(hidden_channels))
def forward(self, x, x_mask, g=None): def forward(self, x, x_mask, g=None):
attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1) attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
x = x * x_mask x = x * x_mask
@@ -89,7 +122,18 @@ class Encoder(nn.Module):
class Decoder(nn.Module): class Decoder(nn.Module):
def __init__(self, hidden_channels, filter_channels, n_heads, n_layers, kernel_size=1, p_dropout=0., proximal_bias=False, proximal_init=True, **kwargs): def __init__(
self,
hidden_channels,
filter_channels,
n_heads,
n_layers,
kernel_size=1,
p_dropout=0.0,
proximal_bias=False,
proximal_init=True,
**kwargs
):
super().__init__() super().__init__()
self.hidden_channels = hidden_channels self.hidden_channels = hidden_channels
self.filter_channels = filter_channels self.filter_channels = filter_channels
@@ -108,11 +152,33 @@ class Decoder(nn.Module):
self.ffn_layers = nn.ModuleList() self.ffn_layers = nn.ModuleList()
self.norm_layers_2 = nn.ModuleList() self.norm_layers_2 = nn.ModuleList()
for i in range(self.n_layers): for i in range(self.n_layers):
self.self_attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout, proximal_bias=proximal_bias, proximal_init=proximal_init)) self.self_attn_layers.append(
MultiHeadAttention(
hidden_channels,
hidden_channels,
n_heads,
p_dropout=p_dropout,
proximal_bias=proximal_bias,
proximal_init=proximal_init,
)
)
self.norm_layers_0.append(LayerNorm(hidden_channels)) self.norm_layers_0.append(LayerNorm(hidden_channels))
self.encdec_attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout)) self.encdec_attn_layers.append(
MultiHeadAttention(
hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout
)
)
self.norm_layers_1.append(LayerNorm(hidden_channels)) self.norm_layers_1.append(LayerNorm(hidden_channels))
self.ffn_layers.append(FFN(hidden_channels, hidden_channels, filter_channels, kernel_size, p_dropout=p_dropout, causal=True)) self.ffn_layers.append(
FFN(
hidden_channels,
hidden_channels,
filter_channels,
kernel_size,
p_dropout=p_dropout,
causal=True,
)
)
self.norm_layers_2.append(LayerNorm(hidden_channels)) self.norm_layers_2.append(LayerNorm(hidden_channels))
def forward(self, x, x_mask, h, h_mask): def forward(self, x, x_mask, h, h_mask):
@@ -120,7 +186,9 @@ class Decoder(nn.Module):
x: decoder input x: decoder input
h: encoder output h: encoder output
""" """
self_attn_mask = commons.subsequent_mask(x_mask.size(2)).to(device=x.device, dtype=x.dtype) self_attn_mask = commons.subsequent_mask(x_mask.size(2)).to(
device=x.device, dtype=x.dtype
)
encdec_attn_mask = h_mask.unsqueeze(2) * x_mask.unsqueeze(-1) encdec_attn_mask = h_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
x = x * x_mask x = x * x_mask
for i in range(self.n_layers): for i in range(self.n_layers):
@@ -140,7 +208,18 @@ class Decoder(nn.Module):
class MultiHeadAttention(nn.Module): class MultiHeadAttention(nn.Module):
def __init__(self, channels, out_channels, n_heads, p_dropout=0., window_size=None, heads_share=True, block_length=None, proximal_bias=False, proximal_init=False): def __init__(
self,
channels,
out_channels,
n_heads,
p_dropout=0.0,
window_size=None,
heads_share=True,
block_length=None,
proximal_bias=False,
proximal_init=False,
):
super().__init__() super().__init__()
assert channels % n_heads == 0 assert channels % n_heads == 0
@@ -165,8 +244,14 @@ class MultiHeadAttention(nn.Module):
if window_size is not None: if window_size is not None:
n_heads_rel = 1 if heads_share else n_heads n_heads_rel = 1 if heads_share else n_heads
rel_stddev = self.k_channels**-0.5 rel_stddev = self.k_channels**-0.5
self.emb_rel_k = nn.Parameter(torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) * rel_stddev) self.emb_rel_k = nn.Parameter(
self.emb_rel_v = nn.Parameter(torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) * rel_stddev) torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels)
* rel_stddev
)
self.emb_rel_v = nn.Parameter(
torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels)
* rel_stddev
)
nn.init.xavier_uniform_(self.conv_q.weight) nn.init.xavier_uniform_(self.conv_q.weight)
nn.init.xavier_uniform_(self.conv_k.weight) nn.init.xavier_uniform_(self.conv_k.weight)
@@ -195,28 +280,46 @@ class MultiHeadAttention(nn.Module):
scores = torch.matmul(query / math.sqrt(self.k_channels), key.transpose(-2, -1)) scores = torch.matmul(query / math.sqrt(self.k_channels), key.transpose(-2, -1))
if self.window_size is not None: if self.window_size is not None:
assert t_s == t_t, "Relative attention is only available for self-attention." assert (
t_s == t_t
), "Relative attention is only available for self-attention."
key_relative_embeddings = self._get_relative_embeddings(self.emb_rel_k, t_s) key_relative_embeddings = self._get_relative_embeddings(self.emb_rel_k, t_s)
rel_logits = self._matmul_with_relative_keys(query /math.sqrt(self.k_channels), key_relative_embeddings) rel_logits = self._matmul_with_relative_keys(
query / math.sqrt(self.k_channels), key_relative_embeddings
)
scores_local = self._relative_position_to_absolute_position(rel_logits) scores_local = self._relative_position_to_absolute_position(rel_logits)
scores = scores + scores_local scores = scores + scores_local
if self.proximal_bias: if self.proximal_bias:
assert t_s == t_t, "Proximal bias is only available for self-attention." assert t_s == t_t, "Proximal bias is only available for self-attention."
scores = scores + self._attention_bias_proximal(t_s).to(device=scores.device, dtype=scores.dtype) scores = scores + self._attention_bias_proximal(t_s).to(
device=scores.device, dtype=scores.dtype
)
if mask is not None: if mask is not None:
scores = scores.masked_fill(mask == 0, -1e4) scores = scores.masked_fill(mask == 0, -1e4)
if self.block_length is not None: if self.block_length is not None:
assert t_s == t_t, "Local attention is only available for self-attention." assert (
block_mask = torch.ones_like(scores).triu(-self.block_length).tril(self.block_length) t_s == t_t
), "Local attention is only available for self-attention."
block_mask = (
torch.ones_like(scores)
.triu(-self.block_length)
.tril(self.block_length)
)
scores = scores.masked_fill(block_mask == 0, -1e4) scores = scores.masked_fill(block_mask == 0, -1e4)
p_attn = F.softmax(scores, dim=-1) # [b, n_h, t_t, t_s] p_attn = F.softmax(scores, dim=-1) # [b, n_h, t_t, t_s]
p_attn = self.drop(p_attn) p_attn = self.drop(p_attn)
output = torch.matmul(p_attn, value) output = torch.matmul(p_attn, value)
if self.window_size is not None: if self.window_size is not None:
relative_weights = self._absolute_position_to_relative_position(p_attn) relative_weights = self._absolute_position_to_relative_position(p_attn)
value_relative_embeddings = self._get_relative_embeddings(self.emb_rel_v, t_s) value_relative_embeddings = self._get_relative_embeddings(
output = output + self._matmul_with_relative_values(relative_weights, value_relative_embeddings) self.emb_rel_v, t_s
output = output.transpose(2, 3).contiguous().view(b, d, t_t) # [b, n_h, t_t, d_k] -> [b, d, t_t] )
output = output + self._matmul_with_relative_values(
relative_weights, value_relative_embeddings
)
output = (
output.transpose(2, 3).contiguous().view(b, d, t_t)
) # [b, n_h, t_t, d_k] -> [b, d, t_t]
return output, p_attn return output, p_attn
def _matmul_with_relative_values(self, x, y): def _matmul_with_relative_values(self, x, y):
@@ -246,10 +349,13 @@ class MultiHeadAttention(nn.Module):
if pad_length > 0: if pad_length > 0:
padded_relative_embeddings = F.pad( padded_relative_embeddings = F.pad(
relative_embeddings, relative_embeddings,
commons.convert_pad_shape([[0, 0], [pad_length, pad_length], [0, 0]])) commons.convert_pad_shape([[0, 0], [pad_length, pad_length], [0, 0]]),
)
else: else:
padded_relative_embeddings = relative_embeddings padded_relative_embeddings = relative_embeddings
used_relative_embeddings = padded_relative_embeddings[:,slice_start_position:slice_end_position] used_relative_embeddings = padded_relative_embeddings[
:, slice_start_position:slice_end_position
]
return used_relative_embeddings return used_relative_embeddings
def _relative_position_to_absolute_position(self, x): def _relative_position_to_absolute_position(self, x):
@@ -263,10 +369,14 @@ class MultiHeadAttention(nn.Module):
# Concat extra elements so to add up to shape (len+1, 2*len-1). # Concat extra elements so to add up to shape (len+1, 2*len-1).
x_flat = x.view([batch, heads, length * 2 * length]) x_flat = x.view([batch, heads, length * 2 * length])
x_flat = F.pad(x_flat, commons.convert_pad_shape([[0,0],[0,0],[0,length-1]])) x_flat = F.pad(
x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [0, length - 1]])
)
# Reshape and slice out the padded elements. # Reshape and slice out the padded elements.
x_final = x_flat.view([batch, heads, length+1, 2*length-1])[:, :, :length, length-1:] x_final = x_flat.view([batch, heads, length + 1, 2 * length - 1])[
:, :, :length, length - 1 :
]
return x_final return x_final
def _absolute_position_to_relative_position(self, x): def _absolute_position_to_relative_position(self, x):
@@ -276,7 +386,9 @@ class MultiHeadAttention(nn.Module):
""" """
batch, heads, length, _ = x.size() batch, heads, length, _ = x.size()
# padd along column # padd along column
x = F.pad(x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, length-1]])) x = F.pad(
x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, length - 1]])
)
x_flat = x.view([batch, heads, length**2 + length * (length - 1)]) x_flat = x.view([batch, heads, length**2 + length * (length - 1)])
# add 0's in the beginning that will skew the elements after reshape # add 0's in the beginning that will skew the elements after reshape
x_flat = F.pad(x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [length, 0]])) x_flat = F.pad(x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [length, 0]]))
@@ -296,7 +408,16 @@ class MultiHeadAttention(nn.Module):
class FFN(nn.Module): class FFN(nn.Module):
def __init__(self, in_channels, out_channels, filter_channels, kernel_size, p_dropout=0., activation=None, causal=False): def __init__(
self,
in_channels,
out_channels,
filter_channels,
kernel_size,
p_dropout=0.0,
activation=None,
causal=False,
):
super().__init__() super().__init__()
self.in_channels = in_channels self.in_channels = in_channels
self.out_channels = out_channels self.out_channels = out_channels

6
bert_gen.py
View File

@@ -7,6 +7,7 @@ from text import cleaned_text_to_sequence, get_bert
import argparse import argparse
import torch.multiprocessing as mp import torch.multiprocessing as mp
def process_line(line): def process_line(line):
rank = mp.current_process()._identity rank = mp.current_process()._identity
rank = rank[0] if len(rank) > 0 else 0 rank = rank[0] if len(rank) > 0 else 0
@@ -41,8 +42,8 @@ def process_line(line):
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('-c', '--config', type=str, default="configs/config.json") parser.add_argument("-c", "--config", type=str, default="configs/config.json")
parser.add_argument('--num_processes', type=int, default=2 ) parser.add_argument("--num_processes", type=int, default=2)
args = parser.parse_args() args = parser.parse_args()
config_path = args.config config_path = args.config
hps = utils.get_hparams_from_file(config_path) hps = utils.get_hparams_from_file(config_path)
@@ -53,7 +54,6 @@ if __name__ == "__main__":
with open(hps.data.validation_files, encoding="utf-8") as f: with open(hps.data.validation_files, encoding="utf-8") as f:
lines.extend(f.readlines()) lines.extend(f.readlines())
num_processes = args.num_processes num_processes = args.num_processes
with Pool(processes=num_processes) as pool: with Pool(processes=num_processes) as pool:
for _ in tqdm(pool.imap_unordered(process_line, lines), total=len(lines)): for _ in tqdm(pool.imap_unordered(process_line, lines), total=len(lines)):

18
commons.py
View File

@@ -30,7 +30,9 @@ def intersperse(lst, item):
def kl_divergence(m_p, logs_p, m_q, logs_q): def kl_divergence(m_p, logs_p, m_q, logs_q):
"""KL(P||Q)""" """KL(P||Q)"""
kl = (logs_q - logs_p) - 0.5 kl = (logs_q - logs_p) - 0.5
kl += 0.5 * (torch.exp(2. * logs_p) + ((m_p - m_q)**2)) * torch.exp(-2. * logs_q) kl += (
0.5 * (torch.exp(2.0 * logs_p) + ((m_p - m_q) ** 2)) * torch.exp(-2.0 * logs_q)
)
return kl return kl
@@ -64,15 +66,15 @@ def rand_slice_segments(x, x_lengths=None, segment_size=4):
return ret, ids_str return ret, ids_str
def get_timing_signal_1d( def get_timing_signal_1d(length, channels, min_timescale=1.0, max_timescale=1.0e4):
length, channels, min_timescale=1.0, max_timescale=1.0e4):
position = torch.arange(length, dtype=torch.float) position = torch.arange(length, dtype=torch.float)
num_timescales = channels // 2 num_timescales = channels // 2
log_timescale_increment = ( log_timescale_increment = math.log(float(max_timescale) / float(min_timescale)) / (
math.log(float(max_timescale) / float(min_timescale)) / num_timescales - 1
(num_timescales - 1)) )
inv_timescales = min_timescale * torch.exp( inv_timescales = min_timescale * torch.exp(
torch.arange(num_timescales, dtype=torch.float) * -log_timescale_increment) torch.arange(num_timescales, dtype=torch.float) * -log_timescale_increment
)
scaled_time = position.unsqueeze(0) * inv_timescales.unsqueeze(1) scaled_time = position.unsqueeze(0) * inv_timescales.unsqueeze(1)
signal = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], 0) signal = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], 0)
signal = F.pad(signal, [0, 0, 0, channels % 2]) signal = F.pad(signal, [0, 0, 0, channels % 2])
@@ -157,5 +159,5 @@ def clip_grad_value_(parameters, clip_value, norm_type=2):
total_norm += param_norm.item() ** norm_type total_norm += param_norm.item() ** norm_type
if clip_value is not None: if clip_value is not None:
p.grad.data.clamp_(min=-clip_value, max=clip_value) p.grad.data.clamp_(min=-clip_value, max=clip_value)
total_norm = total_norm ** (1. / norm_type) total_norm = total_norm ** (1.0 / norm_type)
return total_norm return total_norm

127
data_utils.py
View File

@@ -32,7 +32,9 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
self.spk_map = hparams.spk2id self.spk_map = hparams.spk2id
self.hparams = hparams self.hparams = hparams
self.use_mel_spec_posterior = getattr(hparams, "use_mel_posterior_encoder", False) self.use_mel_spec_posterior = getattr(
hparams, "use_mel_posterior_encoder", False
)
if self.use_mel_spec_posterior: if self.use_mel_spec_posterior:
self.n_mel_channels = getattr(hparams, "n_mel_channels", 80) self.n_mel_channels = getattr(hparams, "n_mel_channels", 80)
@@ -58,17 +60,26 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
lengths = [] lengths = []
skipped = 0 skipped = 0
logger.info("Init dataset...") logger.info("Init dataset...")
for _id, spk, language, text, phones, tone, word2ph in tqdm(self.audiopaths_sid_text): for _id, spk, language, text, phones, tone, word2ph in tqdm(
audiopath = f'{_id}' self.audiopaths_sid_text
):
audiopath = f"{_id}"
if self.min_text_len <= len(phones) and len(phones) <= self.max_text_len: if self.min_text_len <= len(phones) and len(phones) <= self.max_text_len:
phones = phones.split(" ") phones = phones.split(" ")
tone = [int(i) for i in tone.split(" ")] tone = [int(i) for i in tone.split(" ")]
word2ph = [int(i) for i in word2ph.split(" ")] word2ph = [int(i) for i in word2ph.split(" ")]
audiopaths_sid_text_new.append([audiopath, spk, language, text, phones, tone, word2ph]) audiopaths_sid_text_new.append(
[audiopath, spk, language, text, phones, tone, word2ph]
)
lengths.append(os.path.getsize(audiopath) // (2 * self.hop_length)) lengths.append(os.path.getsize(audiopath) // (2 * self.hop_length))
else: else:
skipped += 1 skipped += 1
logger.info("skipped: " + str(skipped) + ", total: " + str(len(self.audiopaths_sid_text))) logger.info(
"skipped: "
+ str(skipped)
+ ", total: "
+ str(len(self.audiopaths_sid_text))
)
self.audiopaths_sid_text = audiopaths_sid_text_new self.audiopaths_sid_text = audiopaths_sid_text_new
self.lengths = lengths self.lengths = lengths
@@ -76,7 +87,9 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
# separate filename, speaker_id and text # separate filename, speaker_id and text
audiopath, sid, language, text, phones, tone, word2ph = audiopath_sid_text audiopath, sid, language, text, phones, tone, word2ph = audiopath_sid_text
bert, ja_bert, phones, tone, language = self.get_text(text, word2ph, phones, tone, language, audiopath) bert, ja_bert, phones, tone, language = self.get_text(
text, word2ph, phones, tone, language, audiopath
)
spec, wav = self.get_audio(audiopath) spec, wav = self.get_audio(audiopath)
sid = torch.LongTensor([int(self.spk_map[sid])]) sid = torch.LongTensor([int(self.spk_map[sid])])
@@ -85,8 +98,11 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
def get_audio(self, filename): def get_audio(self, filename):
audio, sampling_rate = load_wav_to_torch(filename) audio, sampling_rate = load_wav_to_torch(filename)
if sampling_rate != self.sampling_rate: if sampling_rate != self.sampling_rate:
raise ValueError("{} {} SR doesn't match target {} SR".format( raise ValueError(
sampling_rate, self.sampling_rate)) "{} {} SR doesn't match target {} SR".format(
sampling_rate, self.sampling_rate
)
)
audio_norm = audio / self.max_wav_value audio_norm = audio / self.max_wav_value
audio_norm = audio_norm.unsqueeze(0) audio_norm = audio_norm.unsqueeze(0)
spec_filename = filename.replace(".wav", ".spec.pt") spec_filename = filename.replace(".wav", ".spec.pt")
@@ -96,13 +112,26 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
spec = torch.load(spec_filename) spec = torch.load(spec_filename)
except: except:
if self.use_mel_spec_posterior: if self.use_mel_spec_posterior:
spec = mel_spectrogram_torch(audio_norm, self.filter_length, spec = mel_spectrogram_torch(
self.n_mel_channels, self.sampling_rate, self.hop_length, audio_norm,
self.win_length, self.hparams.mel_fmin, self.hparams.mel_fmax, center=False) self.filter_length,
self.n_mel_channels,
self.sampling_rate,
self.hop_length,
self.win_length,
self.hparams.mel_fmin,
self.hparams.mel_fmax,
center=False,
)
else: else:
spec = spectrogram_torch(audio_norm, self.filter_length, spec = spectrogram_torch(
self.sampling_rate, self.hop_length, self.win_length, audio_norm,
center=False) self.filter_length,
self.sampling_rate,
self.hop_length,
self.win_length,
center=False,
)
spec = torch.squeeze(spec, 0) spec = torch.squeeze(spec, 0)
torch.save(spec, spec_filename) torch.save(spec, spec_filename)
return spec, audio_norm return spec, audio_norm
@@ -125,7 +154,7 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
torch.save(bert, bert_path) torch.save(bert, bert_path)
assert bert.shape[-1] == len(phone), phone assert bert.shape[-1] == len(phone), phone
if language_str=='ZH': if language_str == "ZH":
bert = bert bert = bert
ja_bert = torch.zeros(768, len(phone)) ja_bert = torch.zeros(768, len(phone))
elif language_str == "JA": elif language_str == "JA":
@@ -135,7 +164,19 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
bert = torch.zeros(1024, len(phone)) bert = torch.zeros(1024, len(phone))
ja_bert = torch.zeros(768, len(phone)) ja_bert = torch.zeros(768, len(phone))
assert bert.shape[-1] == len(phone), ( assert bert.shape[-1] == len(phone), (
bert.shape, len(phone), sum(word2ph), p1, p2, t1, t2, pold, pold2, word2ph, text, w2pho) bert.shape,
len(phone),
sum(word2ph),
p1,
p2,
t1,
t2,
pold,
pold2,
word2ph,
text,
w2pho,
)
phone = torch.LongTensor(phone) phone = torch.LongTensor(phone)
tone = torch.LongTensor(tone) tone = torch.LongTensor(tone)
language = torch.LongTensor(language) language = torch.LongTensor(language)
@@ -152,9 +193,8 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
return len(self.audiopaths_sid_text) return len(self.audiopaths_sid_text)
class TextAudioSpeakerCollate(): class TextAudioSpeakerCollate:
""" Zero-pads model inputs and targets """Zero-pads model inputs and targets"""
"""
def __init__(self, return_ids=False): def __init__(self, return_ids=False):
self.return_ids = return_ids self.return_ids = return_ids
@@ -167,8 +207,8 @@ class TextAudioSpeakerCollate():
""" """
# Right zero-pad all one-hot text sequences to max input length # Right zero-pad all one-hot text sequences to max input length
_, ids_sorted_decreasing = torch.sort( _, ids_sorted_decreasing = torch.sort(
torch.LongTensor([x[1].size(1) for x in batch]), torch.LongTensor([x[1].size(1) for x in batch]), dim=0, descending=True
dim=0, descending=True) )
max_text_len = max([len(x[0]) for x in batch]) max_text_len = max([len(x[0]) for x in batch])
max_spec_len = max([x[1].size(1) for x in batch]) max_spec_len = max([x[1].size(1) for x in batch])
@@ -223,7 +263,19 @@ class TextAudioSpeakerCollate():
ja_bert = row[7] ja_bert = row[7]
ja_bert_padded[i, :, : ja_bert.size(1)] = ja_bert ja_bert_padded[i, :, : ja_bert.size(1)] = ja_bert
return text_padded, text_lengths, spec_padded, spec_lengths, wav_padded, wav_lengths, sid, tone_padded, language_padded, bert_padded, ja_bert_padded return (
text_padded,
text_lengths,
spec_padded,
spec_lengths,
wav_padded,
wav_lengths,
sid,
tone_padded,
language_padded,
bert_padded,
ja_bert_padded,
)
class DistributedBucketSampler(torch.utils.data.distributed.DistributedSampler): class DistributedBucketSampler(torch.utils.data.distributed.DistributedSampler):
@@ -236,7 +288,15 @@ class DistributedBucketSampler(torch.utils.data.distributed.DistributedSampler):
Ex) boundaries = [b1, b2, b3] -> any x s.t. length(x) <= b1 or length(x) > b3 are discarded. Ex) boundaries = [b1, b2, b3] -> any x s.t. length(x) <= b1 or length(x) > b3 are discarded.
""" """
def __init__(self, dataset, batch_size, boundaries, num_replicas=None, rank=None, shuffle=True): def __init__(
self,
dataset,
batch_size,
boundaries,
num_replicas=None,
rank=None,
shuffle=True,
):
super().__init__(dataset, num_replicas=num_replicas, rank=rank, shuffle=shuffle) super().__init__(dataset, num_replicas=num_replicas, rank=rank, shuffle=shuffle)
self.lengths = dataset.lengths self.lengths = dataset.lengths
self.batch_size = batch_size self.batch_size = batch_size
@@ -262,7 +322,7 @@ class DistributedBucketSampler(torch.utils.data.distributed.DistributedSampler):
assert all(len(bucket) > 0 for bucket in buckets) assert all(len(bucket) > 0 for bucket in buckets)
# When one bucket is not traversed # When one bucket is not traversed
except Exception as e: except Exception as e:
print('Bucket warning ', e) print("Bucket warning ", e)
for i in range(len(buckets) - 1, -1, -1): for i in range(len(buckets) - 1, -1, -1):
if len(buckets[i]) == 0: if len(buckets[i]) == 0:
buckets.pop(i) buckets.pop(i)
@@ -272,7 +332,9 @@ class DistributedBucketSampler(torch.utils.data.distributed.DistributedSampler):
for i in range(len(buckets)): for i in range(len(buckets)):
len_bucket = len(buckets[i]) len_bucket = len(buckets[i])
total_batch_size = self.num_replicas * self.batch_size total_batch_size = self.num_replicas * self.batch_size
rem = (total_batch_size - (len_bucket % total_batch_size)) % total_batch_size rem = (
total_batch_size - (len_bucket % total_batch_size)
) % total_batch_size
num_samples_per_bucket.append(len_bucket + rem) num_samples_per_bucket.append(len_bucket + rem)
return buckets, num_samples_per_bucket return buckets, num_samples_per_bucket
@@ -293,21 +355,30 @@ class DistributedBucketSampler(torch.utils.data.distributed.DistributedSampler):
for i in range(len(self.buckets)): for i in range(len(self.buckets)):
bucket = self.buckets[i] bucket = self.buckets[i]
len_bucket = len(bucket) len_bucket = len(bucket)
if (len_bucket == 0): if len_bucket == 0:
continue continue
ids_bucket = indices[i] ids_bucket = indices[i]
num_samples_bucket = self.num_samples_per_bucket[i] num_samples_bucket = self.num_samples_per_bucket[i]
# add extra samples to make it evenly divisible # add extra samples to make it evenly divisible
rem = num_samples_bucket - len_bucket rem = num_samples_bucket - len_bucket
ids_bucket = ids_bucket + ids_bucket * (rem // len_bucket) + ids_bucket[:(rem % len_bucket)] ids_bucket = (
ids_bucket
+ ids_bucket * (rem // len_bucket)
+ ids_bucket[: (rem % len_bucket)]
)
# subsample # subsample
ids_bucket = ids_bucket[self.rank :: self.num_replicas] ids_bucket = ids_bucket[self.rank :: self.num_replicas]
# batching # batching
for j in range(len(ids_bucket) // self.batch_size): for j in range(len(ids_bucket) // self.batch_size):
batch = [bucket[idx] for idx in ids_bucket[j * self.batch_size:(j + 1) * self.batch_size]] batch = [
bucket[idx]
for idx in ids_bucket[
j * self.batch_size : (j + 1) * self.batch_size
]
]
batches.append(batch) batches.append(batch)
if self.shuffle: if self.shuffle:

4
losses.py
View File

@@ -24,7 +24,7 @@ def discriminator_loss(disc_real_outputs, disc_generated_outputs):
dg = dg.float() dg = dg.float()
r_loss = torch.mean((1 - dr) ** 2) r_loss = torch.mean((1 - dr) ** 2)
g_loss = torch.mean(dg**2) g_loss = torch.mean(dg**2)
loss += (r_loss + g_loss) loss += r_loss + g_loss
r_losses.append(r_loss.item()) r_losses.append(r_loss.item())
g_losses.append(g_loss.item()) g_losses.append(g_loss.item())
@@ -55,7 +55,7 @@ def kl_loss(z_p, logs_q, m_p, logs_p, z_mask):
z_mask = z_mask.float() z_mask = z_mask.float()
kl = logs_p - logs_q - 0.5 kl = logs_p - logs_q - 0.5
kl += 0.5 * ((z_p - m_p)**2) * torch.exp(-2. * logs_p) kl += 0.5 * ((z_p - m_p) ** 2) * torch.exp(-2.0 * logs_p)
kl = torch.sum(kl * z_mask) kl = torch.sum(kl * z_mask)
l = kl / torch.sum(z_mask) l = kl / torch.sum(z_mask)
return l return l

90
mel_processing.py
View File

@@ -49,22 +49,38 @@ hann_window = {}
def spectrogram_torch(y, n_fft, sampling_rate, hop_size, win_size, center=False): def spectrogram_torch(y, n_fft, sampling_rate, hop_size, win_size, center=False):
if torch.min(y) < -1.: if torch.min(y) < -1.0:
print('min value is ', torch.min(y)) print("min value is ", torch.min(y))
if torch.max(y) > 1.: if torch.max(y) > 1.0:
print('max value is ', torch.max(y)) print("max value is ", torch.max(y))
global hann_window global hann_window
dtype_device = str(y.dtype) + '_' + str(y.device) dtype_device = str(y.dtype) + "_" + str(y.device)
wnsize_dtype_device = str(win_size) + '_' + dtype_device wnsize_dtype_device = str(win_size) + "_" + dtype_device
if wnsize_dtype_device not in hann_window: if wnsize_dtype_device not in hann_window:
hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(dtype=y.dtype, device=y.device) hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(
dtype=y.dtype, device=y.device
)
y = torch.nn.functional.pad(y.unsqueeze(1), (int((n_fft-hop_size)/2), int((n_fft-hop_size)/2)), mode='reflect') y = torch.nn.functional.pad(
y.unsqueeze(1),
(int((n_fft - hop_size) / 2), int((n_fft - hop_size) / 2)),
mode="reflect",
)
y = y.squeeze(1) y = y.squeeze(1)
spec = torch.stft(y, n_fft, hop_length=hop_size, win_length=win_size, window=hann_window[wnsize_dtype_device], spec = torch.stft(
center=center, pad_mode='reflect', normalized=False, onesided=True, return_complex=False) y,
n_fft,
hop_length=hop_size,
win_length=win_size,
window=hann_window[wnsize_dtype_device],
center=center,
pad_mode="reflect",
normalized=False,
onesided=True,
return_complex=False,
)
spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6) spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6)
return spec return spec
@@ -72,37 +88,59 @@ def spectrogram_torch(y, n_fft, sampling_rate, hop_size, win_size, center=False)
def spec_to_mel_torch(spec, n_fft, num_mels, sampling_rate, fmin, fmax): def spec_to_mel_torch(spec, n_fft, num_mels, sampling_rate, fmin, fmax):
global mel_basis global mel_basis
dtype_device = str(spec.dtype) + '_' + str(spec.device) dtype_device = str(spec.dtype) + "_" + str(spec.device)
fmax_dtype_device = str(fmax) + '_' + dtype_device fmax_dtype_device = str(fmax) + "_" + dtype_device
if fmax_dtype_device not in mel_basis: if fmax_dtype_device not in mel_basis:
mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax) mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax)
mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(dtype=spec.dtype, device=spec.device) mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(
dtype=spec.dtype, device=spec.device
)
spec = torch.matmul(mel_basis[fmax_dtype_device], spec) spec = torch.matmul(mel_basis[fmax_dtype_device], spec)
spec = spectral_normalize_torch(spec) spec = spectral_normalize_torch(spec)
return spec return spec
def mel_spectrogram_torch(y, n_fft, num_mels, sampling_rate, hop_size, win_size, fmin, fmax, center=False): def mel_spectrogram_torch(
if torch.min(y) < -1.: y, n_fft, num_mels, sampling_rate, hop_size, win_size, fmin, fmax, center=False
print('min value is ', torch.min(y)) ):
if torch.max(y) > 1.: if torch.min(y) < -1.0:
print('max value is ', torch.max(y)) print("min value is ", torch.min(y))
if torch.max(y) > 1.0:
print("max value is ", torch.max(y))
global mel_basis, hann_window global mel_basis, hann_window
dtype_device = str(y.dtype) + '_' + str(y.device) dtype_device = str(y.dtype) + "_" + str(y.device)
fmax_dtype_device = str(fmax) + '_' + dtype_device fmax_dtype_device = str(fmax) + "_" + dtype_device
wnsize_dtype_device = str(win_size) + '_' + dtype_device wnsize_dtype_device = str(win_size) + "_" + dtype_device
if fmax_dtype_device not in mel_basis: if fmax_dtype_device not in mel_basis:
mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax) mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax)
mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(dtype=y.dtype, device=y.device) mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(
dtype=y.dtype, device=y.device
)
if wnsize_dtype_device not in hann_window: if wnsize_dtype_device not in hann_window:
hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(dtype=y.dtype, device=y.device) hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(
dtype=y.dtype, device=y.device
)
y = torch.nn.functional.pad(y.unsqueeze(1), (int((n_fft-hop_size)/2), int((n_fft-hop_size)/2)), mode='reflect') y = torch.nn.functional.pad(
y.unsqueeze(1),
(int((n_fft - hop_size) / 2), int((n_fft - hop_size) / 2)),
mode="reflect",
)
y = y.squeeze(1) y = y.squeeze(1)
spec = torch.stft(y, n_fft, hop_length=hop_size, win_length=win_size, window=hann_window[wnsize_dtype_device], spec = torch.stft(
center=center, pad_mode='reflect', normalized=False, onesided=True, return_complex=False) y,
n_fft,
hop_length=hop_size,
win_length=win_size,
window=hann_window[wnsize_dtype_device],
center=center,
pad_mode="reflect",
normalized=False,
onesided=True,
return_complex=False,
)
spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6) spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6)

3
models.py
View File

@@ -104,7 +104,6 @@ class TransformerCouplingBlock(nn.Module):
gin_channels=0, gin_channels=0,
share_parameter=False, share_parameter=False,
): ):
super().__init__() super().__init__()
self.channels = channels self.channels = channels
self.hidden_channels = hidden_channels self.hidden_channels = hidden_channels
@@ -685,7 +684,6 @@ class ReferenceEncoder(nn.Module):
""" """
def __init__(self, spec_channels, gin_channels=0): def __init__(self, spec_channels, gin_channels=0):
super().__init__() super().__init__()
self.spec_channels = spec_channels self.spec_channels = spec_channels
ref_enc_filters = [32, 32, 64, 64, 128, 128] ref_enc_filters = [32, 32, 64, 64, 128, 128]
@@ -770,7 +768,6 @@ class SynthesizerTrn(nn.Module):
use_transformer_flow=True, use_transformer_flow=True,
**kwargs **kwargs
): ):
super().__init__() super().__init__()
self.n_vocab = n_vocab self.n_vocab = n_vocab
self.spec_channels = spec_channels self.spec_channels = spec_channels

270
modules.py
View File

@@ -16,6 +16,7 @@ from attentions import Encoder
LRELU_SLOPE = 0.1 LRELU_SLOPE = 0.1
class LayerNorm(nn.Module): class LayerNorm(nn.Module):
def __init__(self, channels, eps=1e-5): def __init__(self, channels, eps=1e-5):
super().__init__() super().__init__()
@@ -30,8 +31,17 @@ class LayerNorm(nn.Module):
x = F.layer_norm(x, (self.channels,), self.gamma, self.beta, self.eps) x = F.layer_norm(x, (self.channels,), self.gamma, self.beta, self.eps)
return x.transpose(1, -1) return x.transpose(1, -1)
class ConvReluNorm(nn.Module): class ConvReluNorm(nn.Module):
def __init__(self, in_channels, hidden_channels, out_channels, kernel_size, n_layers, p_dropout): def __init__(
self,
in_channels,
hidden_channels,
out_channels,
kernel_size,
n_layers,
p_dropout,
):
super().__init__() super().__init__()
self.in_channels = in_channels self.in_channels = in_channels
self.hidden_channels = hidden_channels self.hidden_channels = hidden_channels
@@ -43,13 +53,22 @@ class ConvReluNorm(nn.Module):
self.conv_layers = nn.ModuleList() self.conv_layers = nn.ModuleList()
self.norm_layers = nn.ModuleList() self.norm_layers = nn.ModuleList()
self.conv_layers.append(nn.Conv1d(in_channels, hidden_channels, kernel_size, padding=kernel_size//2)) self.conv_layers.append(
nn.Conv1d(
in_channels, hidden_channels, kernel_size, padding=kernel_size // 2
)
)
self.norm_layers.append(LayerNorm(hidden_channels)) self.norm_layers.append(LayerNorm(hidden_channels))
self.relu_drop = nn.Sequential( self.relu_drop = nn.Sequential(nn.ReLU(), nn.Dropout(p_dropout))
nn.ReLU(),
nn.Dropout(p_dropout))
for _ in range(n_layers - 1): for _ in range(n_layers - 1):
self.conv_layers.append(nn.Conv1d(hidden_channels, hidden_channels, kernel_size, padding=kernel_size//2)) self.conv_layers.append(
nn.Conv1d(
hidden_channels,
hidden_channels,
kernel_size,
padding=kernel_size // 2,
)
)
self.norm_layers.append(LayerNorm(hidden_channels)) self.norm_layers.append(LayerNorm(hidden_channels))
self.proj = nn.Conv1d(hidden_channels, out_channels, 1) self.proj = nn.Conv1d(hidden_channels, out_channels, 1)
self.proj.weight.data.zero_() self.proj.weight.data.zero_()
@@ -69,7 +88,8 @@ class DDSConv(nn.Module):
""" """
Dialted and Depth-Separable Convolution Dialted and Depth-Separable Convolution
""" """
def __init__(self, channels, kernel_size, n_layers, p_dropout=0.):
def __init__(self, channels, kernel_size, n_layers, p_dropout=0.0):
super().__init__() super().__init__()
self.channels = channels self.channels = channels
self.kernel_size = kernel_size self.kernel_size = kernel_size
@@ -84,9 +104,16 @@ class DDSConv(nn.Module):
for i in range(n_layers): for i in range(n_layers):
dilation = kernel_size**i dilation = kernel_size**i
padding = (kernel_size * dilation - dilation) // 2 padding = (kernel_size * dilation - dilation) // 2
self.convs_sep.append(nn.Conv1d(channels, channels, kernel_size, self.convs_sep.append(
groups=channels, dilation=dilation, padding=padding nn.Conv1d(
)) channels,
channels,
kernel_size,
groups=channels,
dilation=dilation,
padding=padding,
)
)
self.convs_1x1.append(nn.Conv1d(channels, channels, 1)) self.convs_1x1.append(nn.Conv1d(channels, channels, 1))
self.norms_1.append(LayerNorm(channels)) self.norms_1.append(LayerNorm(channels))
self.norms_2.append(LayerNorm(channels)) self.norms_2.append(LayerNorm(channels))
@@ -107,11 +134,19 @@ class DDSConv(nn.Module):
class WN(torch.nn.Module): class WN(torch.nn.Module):
def __init__(self, hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=0, p_dropout=0): def __init__(
self,
hidden_channels,
kernel_size,
dilation_rate,
n_layers,
gin_channels=0,
p_dropout=0,
):
super(WN, self).__init__() super(WN, self).__init__()
assert(kernel_size % 2 == 1) assert kernel_size % 2 == 1
self.hidden_channels = hidden_channels self.hidden_channels = hidden_channels
self.kernel_size = kernel_size, self.kernel_size = (kernel_size,)
self.dilation_rate = dilation_rate self.dilation_rate = dilation_rate
self.n_layers = n_layers self.n_layers = n_layers
self.gin_channels = gin_channels self.gin_channels = gin_channels
@@ -122,15 +157,22 @@ class WN(torch.nn.Module):
self.drop = nn.Dropout(p_dropout) self.drop = nn.Dropout(p_dropout)
if gin_channels != 0: if gin_channels != 0:
cond_layer = torch.nn.Conv1d(gin_channels, 2*hidden_channels*n_layers, 1) cond_layer = torch.nn.Conv1d(
self.cond_layer = torch.nn.utils.weight_norm(cond_layer, name='weight') gin_channels, 2 * hidden_channels * n_layers, 1
)
self.cond_layer = torch.nn.utils.weight_norm(cond_layer, name="weight")
for i in range(n_layers): for i in range(n_layers):
dilation = dilation_rate**i dilation = dilation_rate**i
padding = int((kernel_size * dilation - dilation) / 2) padding = int((kernel_size * dilation - dilation) / 2)
in_layer = torch.nn.Conv1d(hidden_channels, 2*hidden_channels, kernel_size, in_layer = torch.nn.Conv1d(
dilation=dilation, padding=padding) hidden_channels,
in_layer = torch.nn.utils.weight_norm(in_layer, name='weight') 2 * hidden_channels,
kernel_size,
dilation=dilation,
padding=padding,
)
in_layer = torch.nn.utils.weight_norm(in_layer, name="weight")
self.in_layers.append(in_layer) self.in_layers.append(in_layer)
# last one is not necessary # last one is not necessary
@@ -140,7 +182,7 @@ class WN(torch.nn.Module):
res_skip_channels = hidden_channels res_skip_channels = hidden_channels
res_skip_layer = torch.nn.Conv1d(hidden_channels, res_skip_channels, 1) res_skip_layer = torch.nn.Conv1d(hidden_channels, res_skip_channels, 1)
res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name='weight') res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name="weight")
self.res_skip_layers.append(res_skip_layer) self.res_skip_layers.append(res_skip_layer)
def forward(self, x, x_mask, g=None, **kwargs): def forward(self, x, x_mask, g=None, **kwargs):
@@ -158,10 +200,7 @@ class WN(torch.nn.Module):
else: else:
g_l = torch.zeros_like(x_in) g_l = torch.zeros_like(x_in)
acts = commons.fused_add_tanh_sigmoid_multiply( acts = commons.fused_add_tanh_sigmoid_multiply(x_in, g_l, n_channels_tensor)
x_in,
g_l,
n_channels_tensor)
acts = self.drop(acts) acts = self.drop(acts)
res_skip_acts = self.res_skip_layers[i](acts) res_skip_acts = self.res_skip_layers[i](acts)
@@ -185,24 +224,76 @@ class WN(torch.nn.Module):
class ResBlock1(torch.nn.Module): class ResBlock1(torch.nn.Module):
def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5)): def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5)):
super(ResBlock1, self).__init__() super(ResBlock1, self).__init__()
self.convs1 = nn.ModuleList([ self.convs1 = nn.ModuleList(
weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0], [
padding=get_padding(kernel_size, dilation[0]))), weight_norm(
weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1], Conv1d(
padding=get_padding(kernel_size, dilation[1]))), channels,
weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[2], channels,
padding=get_padding(kernel_size, dilation[2]))) kernel_size,
]) 1,
dilation=dilation[0],
padding=get_padding(kernel_size, dilation[0]),
)
),
weight_norm(
Conv1d(
channels,
channels,
kernel_size,
1,
dilation=dilation[1],
padding=get_padding(kernel_size, dilation[1]),
)
),
weight_norm(
Conv1d(
channels,
channels,
kernel_size,
1,
dilation=dilation[2],
padding=get_padding(kernel_size, dilation[2]),
)
),
]
)
self.convs1.apply(init_weights) self.convs1.apply(init_weights)
self.convs2 = nn.ModuleList([ self.convs2 = nn.ModuleList(
weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1, [
padding=get_padding(kernel_size, 1))), weight_norm(
weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1, Conv1d(
padding=get_padding(kernel_size, 1))), channels,
weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1, channels,
padding=get_padding(kernel_size, 1))) kernel_size,
]) 1,
dilation=1,
padding=get_padding(kernel_size, 1),
)
),
weight_norm(
Conv1d(
channels,
channels,
kernel_size,
1,
dilation=1,
padding=get_padding(kernel_size, 1),
)
),
weight_norm(
Conv1d(
channels,
channels,
kernel_size,
1,
dilation=1,
padding=get_padding(kernel_size, 1),
)
),
]
)
self.convs2.apply(init_weights) self.convs2.apply(init_weights)
def forward(self, x, x_mask=None): def forward(self, x, x_mask=None):
@@ -230,12 +321,30 @@ class ResBlock1(torch.nn.Module):
class ResBlock2(torch.nn.Module): class ResBlock2(torch.nn.Module):
def __init__(self, channels, kernel_size=3, dilation=(1, 3)): def __init__(self, channels, kernel_size=3, dilation=(1, 3)):
super(ResBlock2, self).__init__() super(ResBlock2, self).__init__()
self.convs = nn.ModuleList([ self.convs = nn.ModuleList(
weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0], [
padding=get_padding(kernel_size, dilation[0]))), weight_norm(
weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1], Conv1d(
padding=get_padding(kernel_size, dilation[1]))) channels,
]) channels,
kernel_size,
1,
dilation=dilation[0],
padding=get_padding(kernel_size, dilation[0]),
)
),
weight_norm(
Conv1d(
channels,
channels,
kernel_size,
1,
dilation=dilation[1],
padding=get_padding(kernel_size, dilation[1]),
)
),
]
)
self.convs.apply(init_weights) self.convs.apply(init_weights)
def forward(self, x, x_mask=None): def forward(self, x, x_mask=None):
@@ -294,7 +403,8 @@ class ElementwiseAffine(nn.Module):
class ResidualCouplingLayer(nn.Module): class ResidualCouplingLayer(nn.Module):
def __init__(self, def __init__(
self,
channels, channels,
hidden_channels, hidden_channels,
kernel_size, kernel_size,
@@ -302,7 +412,8 @@ class ResidualCouplingLayer(nn.Module):
n_layers, n_layers,
p_dropout=0, p_dropout=0,
gin_channels=0, gin_channels=0,
mean_only=False): mean_only=False,
):
assert channels % 2 == 0, "channels should be divisible by 2" assert channels % 2 == 0, "channels should be divisible by 2"
super().__init__() super().__init__()
self.channels = channels self.channels = channels
@@ -314,7 +425,14 @@ class ResidualCouplingLayer(nn.Module):
self.mean_only = mean_only self.mean_only = mean_only
self.pre = nn.Conv1d(self.half_channels, hidden_channels, 1) self.pre = nn.Conv1d(self.half_channels, hidden_channels, 1)
self.enc = WN(hidden_channels, kernel_size, dilation_rate, n_layers, p_dropout=p_dropout, gin_channels=gin_channels) self.enc = WN(
hidden_channels,
kernel_size,
dilation_rate,
n_layers,
p_dropout=p_dropout,
gin_channels=gin_channels,
)
self.post = nn.Conv1d(hidden_channels, self.half_channels * (2 - mean_only), 1) self.post = nn.Conv1d(hidden_channels, self.half_channels * (2 - mean_only), 1)
self.post.weight.data.zero_() self.post.weight.data.zero_()
self.post.bias.data.zero_() self.post.bias.data.zero_()
@@ -342,7 +460,15 @@ class ResidualCouplingLayer(nn.Module):
class ConvFlow(nn.Module): class ConvFlow(nn.Module):
def __init__(self, in_channels, filter_channels, kernel_size, n_layers, num_bins=10, tail_bound=5.0): def __init__(
self,
in_channels,
filter_channels,
kernel_size,
n_layers,
num_bins=10,
tail_bound=5.0,
):
super().__init__() super().__init__()
self.in_channels = in_channels self.in_channels = in_channels
self.filter_channels = filter_channels self.filter_channels = filter_channels
@@ -353,8 +479,10 @@ class ConvFlow(nn.Module):
self.half_channels = in_channels // 2 self.half_channels = in_channels // 2
self.pre = nn.Conv1d(self.half_channels, filter_channels, 1) self.pre = nn.Conv1d(self.half_channels, filter_channels, 1)
self.convs = DDSConv(filter_channels, kernel_size, n_layers, p_dropout=0.) self.convs = DDSConv(filter_channels, kernel_size, n_layers, p_dropout=0.0)
self.proj = nn.Conv1d(filter_channels, self.half_channels * (num_bins * 3 - 1), 1) self.proj = nn.Conv1d(
filter_channels, self.half_channels * (num_bins * 3 - 1), 1
)
self.proj.weight.data.zero_() self.proj.weight.data.zero_()
self.proj.bias.data.zero_() self.proj.bias.data.zero_()
@@ -368,16 +496,19 @@ class ConvFlow(nn.Module):
h = h.reshape(b, c, -1, t).permute(0, 1, 3, 2) # [b, cx?, t] -> [b, c, t, ?] h = h.reshape(b, c, -1, t).permute(0, 1, 3, 2) # [b, cx?, t] -> [b, c, t, ?]
unnormalized_widths = h[..., : self.num_bins] / math.sqrt(self.filter_channels) unnormalized_widths = h[..., : self.num_bins] / math.sqrt(self.filter_channels)
unnormalized_heights = h[..., self.num_bins:2*self.num_bins] / math.sqrt(self.filter_channels) unnormalized_heights = h[..., self.num_bins : 2 * self.num_bins] / math.sqrt(
self.filter_channels
)
unnormalized_derivatives = h[..., 2 * self.num_bins :] unnormalized_derivatives = h[..., 2 * self.num_bins :]
x1, logabsdet = piecewise_rational_quadratic_transform(x1, x1, logabsdet = piecewise_rational_quadratic_transform(
x1,
unnormalized_widths, unnormalized_widths,
unnormalized_heights, unnormalized_heights,
unnormalized_derivatives, unnormalized_derivatives,
inverse=reverse, inverse=reverse,
tails='linear', tails="linear",
tail_bound=self.tail_bound tail_bound=self.tail_bound,
) )
x = torch.cat([x0, x1], 1) * x_mask x = torch.cat([x0, x1], 1) * x_mask
@@ -386,8 +517,11 @@ class ConvFlow(nn.Module):
return x, logdet return x, logdet
else: else:
return x return x
class TransformerCouplingLayer(nn.Module): class TransformerCouplingLayer(nn.Module):
def __init__(self, def __init__(
self,
channels, channels,
hidden_channels, hidden_channels,
kernel_size, kernel_size,
@@ -397,7 +531,7 @@ class TransformerCouplingLayer(nn.Module):
filter_channels=0, filter_channels=0,
mean_only=False, mean_only=False,
wn_sharing_parameter=None, wn_sharing_parameter=None,
gin_channels = 0 gin_channels=0,
): ):
assert channels % 2 == 0, "channels should be divisible by 2" assert channels % 2 == 0, "channels should be divisible by 2"
super().__init__() super().__init__()
@@ -409,7 +543,20 @@ class TransformerCouplingLayer(nn.Module):
self.mean_only = mean_only self.mean_only = mean_only
self.pre = nn.Conv1d(self.half_channels, hidden_channels, 1) self.pre = nn.Conv1d(self.half_channels, hidden_channels, 1)
self.enc = Encoder(hidden_channels, filter_channels, n_heads, n_layers, kernel_size, p_dropout, isflow = True, gin_channels = gin_channels) if wn_sharing_parameter is None else wn_sharing_parameter self.enc = (
Encoder(
hidden_channels,
filter_channels,
n_heads,
n_layers,
kernel_size,
p_dropout,
isflow=True,
gin_channels=gin_channels,
)
if wn_sharing_parameter is None
else wn_sharing_parameter
)
self.post = nn.Conv1d(hidden_channels, self.half_channels * (2 - mean_only), 1) self.post = nn.Conv1d(hidden_channels, self.half_channels * (2 - mean_only), 1)
self.post.weight.data.zero_() self.post.weight.data.zero_()
self.post.bias.data.zero_() self.post.bias.data.zero_()
@@ -435,13 +582,14 @@ class TransformerCouplingLayer(nn.Module):
x = torch.cat([x0, x1], 1) x = torch.cat([x0, x1], 1)
return x return x
x1, logabsdet = piecewise_rational_quadratic_transform(x1, x1, logabsdet = piecewise_rational_quadratic_transform(
x1,
unnormalized_widths, unnormalized_widths,
unnormalized_heights, unnormalized_heights,
unnormalized_derivatives, unnormalized_derivatives,
inverse=reverse, inverse=reverse,
tails='linear', tails="linear",
tail_bound=self.tail_bound tail_bound=self.tail_bound,
) )
x = torch.cat([x0, x1], 1) * x_mask x = torch.cat([x0, x1], 1) * x_mask

1
monotonic_align/__init__.py
View File

@@ -3,6 +3,7 @@ from torch import from_numpy
from .core import maximum_path_jit from .core import maximum_path_jit
def maximum_path(neg_cent, mask): def maximum_path(neg_cent, mask):
device = neg_cent.device device = neg_cent.device
dtype = neg_cent.dtype dtype = neg_cent.dtype

17
monotonic_align/core.py
View File

@@ -1,7 +1,16 @@
import numba import numba
@numba.jit(numba.void(numba.int32[:,:,::1], numba.float32[:,:,::1], numba.int32[::1], numba.int32[::1]), nopython=True, nogil=True) @numba.jit(
numba.void(
numba.int32[:, :, ::1],
numba.float32[:, :, ::1],
numba.int32[::1],
numba.int32[::1],
),
nopython=True,
nogil=True,
)
def maximum_path_jit(paths, values, t_ys, t_xs): def maximum_path_jit(paths, values, t_ys, t_xs):
b = paths.shape[0] b = paths.shape[0]
max_neg_val = -1e9 max_neg_val = -1e9
@@ -22,7 +31,7 @@ def maximum_path_jit(paths, values, t_ys, t_xs):
v_cur = value[y - 1, x] v_cur = value[y - 1, x]
if x == 0: if x == 0:
if y == 0: if y == 0:
v_prev = 0. v_prev = 0.0
else: else:
v_prev = max_neg_val v_prev = max_neg_val
else: else:
@@ -31,5 +40,7 @@ def maximum_path_jit(paths, values, t_ys, t_xs):
for y in range(t_y - 1, -1, -1): for y in range(t_y - 1, -1, -1):
path[y, index] = 1 path[y, index] = 1
if index != 0 and (index == y or value[y-1, index] < value[y-1, index-1]): if index != 0 and (
index == y or value[y - 1, index] < value[y - 1, index - 1]
):
index = index - 1 index = index - 1

1
preprocess_text.py
View File

@@ -35,7 +35,6 @@ def main(
max_val_total: int, max_val_total: int,
clean: bool, clean: bool,
): ):
if cleaned_path is None: if cleaned_path is None:
cleaned_path = transcription_path + ".cleaned" cleaned_path = transcription_path + ".cleaned"

28
resample.py
View File

@@ -13,22 +13,21 @@ def process(item):
spkdir, wav_name, args = item spkdir, wav_name, args = item
speaker = spkdir.replace("\\", "/").split("/")[-1] speaker = spkdir.replace("\\", "/").split("/")[-1]
wav_path = os.path.join(args.in_dir, speaker, wav_name) wav_path = os.path.join(args.in_dir, speaker, wav_name)
if os.path.exists(wav_path) and '.wav' in wav_path: if os.path.exists(wav_path) and ".wav" in wav_path:
os.makedirs(os.path.join(args.out_dir, speaker), exist_ok=True) os.makedirs(os.path.join(args.out_dir, speaker), exist_ok=True)
wav, sr = librosa.load(wav_path, sr=args.sr) wav, sr = librosa.load(wav_path, sr=args.sr)
soundfile.write( soundfile.write(os.path.join(args.out_dir, speaker, wav_name), wav, sr)
os.path.join(args.out_dir, speaker, wav_name),
wav,
sr
)
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument("--sr", type=int, default=44100, help="sampling rate") parser.add_argument("--sr", type=int, default=44100, help="sampling rate")
parser.add_argument("--in_dir", type=str, default="./raw", help="path to source dir") parser.add_argument(
parser.add_argument("--out_dir", type=str, default="./dataset", help="path to target dir") "--in_dir", type=str, default="./raw", help="path to source dir"
)
parser.add_argument(
"--out_dir", type=str, default="./dataset", help="path to target dir"
)
args = parser.parse_args() args = parser.parse_args()
# processs = 8 # processs = 8
processs = cpu_count() - 2 if cpu_count() > 4 else 1 processs = cpu_count() - 2 if cpu_count() > 4 else 1
@@ -38,5 +37,14 @@ if __name__ == "__main__":
spk_dir = os.path.join(args.in_dir, speaker) spk_dir = os.path.join(args.in_dir, speaker)
if os.path.isdir(spk_dir): if os.path.isdir(spk_dir):
print(spk_dir) print(spk_dir)
for _ in tqdm(pool.imap_unordered(process, [(spk_dir, i, args) for i in os.listdir(spk_dir) if i.endswith("wav")])): for _ in tqdm(
pool.imap_unordered(
process,
[
(spk_dir, i, args)
for i in os.listdir(spk_dir)
if i.endswith("wav")
],
)
):
pass pass

81
server.py
View File

@@ -13,7 +13,8 @@ from scipy.io import wavfile
# Flask Init # Flask Init
app = Flask(__name__) app = Flask(__name__)
app.config['JSON_AS_ASCII'] = False app.config["JSON_AS_ASCII"] = False
def get_text(text, language_str, hps): def get_text(text, language_str, hps):
norm_text, phone, tone, word2ph = clean_text(text, language_str) norm_text, phone, tone, word2ph = clean_text(text, language_str)
@@ -30,7 +31,7 @@ def get_text(text, language_str, hps):
del word2ph del word2ph
assert bert.shape[-1] == len(phone), phone assert bert.shape[-1] == len(phone), phone
if language_str=='ZH': if language_str == "ZH":
bert = bert bert = bert
ja_bert = torch.zeros(768, len(phone)) ja_bert = torch.zeros(768, len(phone))
elif language_str == "JA": elif language_str == "JA":
@@ -40,12 +41,25 @@ def get_text(text, language_str, hps):
bert = torch.zeros(1024, len(phone)) bert = torch.zeros(1024, len(phone))
ja_bert = torch.zeros(768, len(phone)) ja_bert = torch.zeros(768, len(phone))
assert bert.shape[-1] == len(phone), ( assert bert.shape[-1] == len(phone), (
bert.shape, len(phone), sum(word2ph), p1, p2, t1, t2, pold, pold2, word2ph, text, w2pho) bert.shape,
len(phone),
sum(word2ph),
p1,
p2,
t1,
t2,
pold,
pold2,
word2ph,
text,
w2pho,
)
phone = torch.LongTensor(phone) phone = torch.LongTensor(phone)
tone = torch.LongTensor(tone) tone = torch.LongTensor(tone)
language = torch.LongTensor(language) language = torch.LongTensor(language)
return bert, ja_bert, phone, tone, language return bert, ja_bert, phone, tone, language
def infer(text, sdp_ratio, noise_scale, noise_scale_w, length_scale, sid, language): def infer(text, sdp_ratio, noise_scale, noise_scale_w, length_scale, sid, language):
bert, ja_bert, phones, tones, lang_ids = get_text(text, language, hps) bert, ja_bert, phones, tones, lang_ids = get_text(text, language, hps)
with torch.no_grad(): with torch.no_grad():
@@ -56,55 +70,79 @@ def infer(text, sdp_ratio, noise_scale, noise_scale_w, length_scale, sid, langua
ja_bert = ja_bert.to(device).unsqueeze(0) ja_bert = ja_bert.to(device).unsqueeze(0)
x_tst_lengths = torch.LongTensor([phones.size(0)]).to(dev) x_tst_lengths = torch.LongTensor([phones.size(0)]).to(dev)
speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(dev) speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(dev)
audio = net_g.infer(x_tst, x_tst_lengths, speakers, tones, lang_ids, bert, ja_bert, sdp_ratio=sdp_ratio audio = (
, noise_scale=noise_scale, noise_scale_w=noise_scale_w, length_scale=length_scale)[0][0,0].data.cpu().float().numpy() net_g.infer(
x_tst,
x_tst_lengths,
speakers,
tones,
lang_ids,
bert,
ja_bert,
sdp_ratio=sdp_ratio,
noise_scale=noise_scale,
noise_scale_w=noise_scale_w,
length_scale=length_scale,
)[0][0, 0]
.data.cpu()
.float()
.numpy()
)
return audio return audio
def replace_punctuation(text, i=2): def replace_punctuation(text, i=2):
punctuation = "，。？！" punctuation = "，。？！"
for char in punctuation: for char in punctuation:
text = text.replace(char, char * i) text = text.replace(char, char * i)
return text return text
def wav2(i, o, format): def wav2(i, o, format):
inp = avopen(i, 'rb') inp = avopen(i, "rb")
out = avopen(o, 'wb', format=format) out = avopen(o, "wb", format=format)
if format == "ogg": format = "libvorbis" if format == "ogg":
format = "libvorbis"
ostream = out.add_stream(format) ostream = out.add_stream(format)
for frame in inp.decode(audio=0): for frame in inp.decode(audio=0):
for p in ostream.encode(frame): out.mux(p) for p in ostream.encode(frame):
out.mux(p)
for p in ostream.encode(None): out.mux(p) for p in ostream.encode(None):
out.mux(p)
out.close() out.close()
inp.close() inp.close()
# Load Generator # Load Generator
hps = utils.get_hparams_from_file("./configs/config.json") hps = utils.get_hparams_from_file("./configs/config.json")
dev='cuda' dev = "cuda"
net_g = SynthesizerTrn( net_g = SynthesizerTrn(
len(symbols), len(symbols),
hps.data.filter_length // 2 + 1, hps.data.filter_length // 2 + 1,
hps.train.segment_size // hps.data.hop_length, hps.train.segment_size // hps.data.hop_length,
n_speakers=hps.data.n_speakers, n_speakers=hps.data.n_speakers,
**hps.model).to(dev) **hps.model
).to(dev)
_ = net_g.eval() _ = net_g.eval()
_ = utils.load_checkpoint("logs/G_649000.pth", net_g, None, skip_optimizer=True) _ = utils.load_checkpoint("logs/G_649000.pth", net_g, None, skip_optimizer=True)
@app.route("/") @app.route("/")
def main(): def main():
try: try:
speaker = request.args.get('speaker') speaker = request.args.get("speaker")
text = request.args.get('text').replace("/n","") text = request.args.get("text").replace("/n", "")
sdp_ratio = float(request.args.get("sdp_ratio", 0.2)) sdp_ratio = float(request.args.get("sdp_ratio", 0.2))
noise = float(request.args.get("noise", 0.5)) noise = float(request.args.get("noise", 0.5))
noisew = float(request.args.get("noisew", 0.6)) noisew = float(request.args.get("noisew", 0.6))
length = float(request.args.get("length", 1.2)) length = float(request.args.get("length", 1.2))
language = request.args.get('language') language = request.args.get("language")
if length >= 2: if length >= 2:
return "Too big length" return "Too big length"
if len(text) >= 250: if len(text) >= 250:
@@ -120,7 +158,15 @@ def main():
return "Invalid Parameter" return "Invalid Parameter"
with torch.no_grad(): with torch.no_grad():
audio = infer(text, sdp_ratio=sdp_ratio, noise_scale=noise, noise_scale_w=noisew, length_scale=length, sid=speaker,language = language) audio = infer(
text,
sdp_ratio=sdp_ratio,
noise_scale=noise,
noise_scale_w=noisew,
length_scale=length,
sid=speaker,
language=language,
)
with BytesIO() as wav: with BytesIO() as wav:
wavfile.write(wav, hps.data.sampling_rate, audio) wavfile.write(wav, hps.data.sampling_rate, audio)
@@ -131,6 +177,5 @@ def main():
with BytesIO() as ofp: with BytesIO() as ofp:
wav2(wav, ofp, fmt) wav2(wav, ofp, fmt)
return Response( return Response(
ofp.getvalue(), ofp.getvalue(), mimetype="audio/mpeg" if fmt == "mp3" else "audio/ogg"
mimetype="audio/mpeg" if fmt == "mp3" else "audio/ogg"
) )

13
text/__init__.py
View File

@@ -3,13 +3,14 @@ from text.symbols import *
_symbol_to_id = {s: i for i, s in enumerate(symbols)} _symbol_to_id = {s: i for i, s in enumerate(symbols)}
def cleaned_text_to_sequence(cleaned_text, tones, language): def cleaned_text_to_sequence(cleaned_text, tones, language):
'''Converts a string of text to a sequence of IDs corresponding to the symbols in the text. """Converts a string of text to a sequence of IDs corresponding to the symbols in the text.
Args: Args:
text: string to convert to a sequence text: string to convert to a sequence
Returns: Returns:
List of integers corresponding to the symbols in the text List of integers corresponding to the symbols in the text
''' """
phones = [_symbol_to_id[symbol] for symbol in cleaned_text] phones = [_symbol_to_id[symbol] for symbol in cleaned_text]
tone_start = language_tone_start_map[language] tone_start = language_tone_start_map[language]
tones = [i + tone_start for i in tones] tones = [i + tone_start for i in tones]
@@ -17,14 +18,12 @@ def cleaned_text_to_sequence(cleaned_text, tones, language):
lang_ids = [lang_id for i in phones] lang_ids = [lang_id for i in phones]
return phones, tones, lang_ids return phones, tones, lang_ids
def get_bert(norm_text, word2ph, language, device): def get_bert(norm_text, word2ph, language, device):
from .chinese_bert import get_bert_feature as zh_bert from .chinese_bert import get_bert_feature as zh_bert
from .english_bert_mock import get_bert_feature as en_bert from .english_bert_mock import get_bert_feature as en_bert
from .japanese_bert import get_bert_feature as jp_bert from .japanese_bert import get_bert_feature as jp_bert
lang_bert_func_map = {
'ZH': zh_bert, lang_bert_func_map = {"ZH": zh_bert, "EN": en_bert, "JP": jp_bert}
'EN': en_bert,
'JP': jp_bert
}
bert = lang_bert_func_map[language](norm_text, word2ph, device) bert = lang_bert_func_map[language](norm_text, word2ph, device)
return bert return bert

129
text/chinese.py
View File

@@ -9,65 +9,70 @@ from text.symbols import punctuation
from text.tone_sandhi import ToneSandhi from text.tone_sandhi import ToneSandhi
current_file_path = os.path.dirname(__file__) current_file_path = os.path.dirname(__file__)
pinyin_to_symbol_map = {line.split("\t")[0]: line.strip().split("\t")[1] for line in pinyin_to_symbol_map = {
open(os.path.join(current_file_path, 'opencpop-strict.txt')).readlines()} line.split("\t")[0]: line.strip().split("\t")[1]
for line in open(os.path.join(current_file_path, "opencpop-strict.txt")).readlines()
}
import jieba.posseg as psg import jieba.posseg as psg
rep_map = { rep_map = {
'：': ',', "：": ",",
'；': ',', "；": ",",
'，': ',', "，": ",",
'。': '.', "。": ".",
'！': '!', "！": "!",
'？': '?', "？": "?",
'\n': '.', "\n": ".",
"·": ",", "·": ",",
'、': ",", "、": ",",
'...': '…', "...": "…",
'$': '.', "$": ".",
'“': "'", "“": "'",
'”': "'", "”": "'",
'‘': "'", "‘": "'",
'’': "'", "’": "'",
'（': "'", "（": "'",
'）': "'", "）": "'",
'(': "'", "(": "'",
')': "'", ")": "'",
'《': "'", "《": "'",
'》': "'", "》": "'",
'【': "'", "【": "'",
'】': "'", "】": "'",
'[': "'", "[": "'",
']': "'", "]": "'",
'—': "-", "—": "-",
'～': "-", "～": "-",
'~': "-", "~": "-",
'「': "'", "「": "'",
'」': "'", "」": "'",
} }
tone_modifier = ToneSandhi() tone_modifier = ToneSandhi()
def replace_punctuation(text): def replace_punctuation(text):
text = text.replace("嗯", "恩").replace("呣", "母") text = text.replace("嗯", "恩").replace("呣", "母")
pattern = re.compile('|'.join(re.escape(p) for p in rep_map.keys())) pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys()))
replaced_text = pattern.sub(lambda x: rep_map[x.group()], text) replaced_text = pattern.sub(lambda x: rep_map[x.group()], text)
replaced_text = re.sub(r'[^\u4e00-\u9fa5'+"".join(punctuation)+r']+', '', replaced_text) replaced_text = re.sub(
r"[^\u4e00-\u9fa5" + "".join(punctuation) + r"]+", "", replaced_text
)
return replaced_text return replaced_text
def g2p(text): def g2p(text):
pattern = r'(?<=[{0}])\s*'.format(''.join(punctuation)) pattern = r"(?<=[{0}])\s*".format("".join(punctuation))
sentences = [i for i in re.split(pattern, text) if i.strip()!=''] sentences = [i for i in re.split(pattern, text) if i.strip() != ""]
phones, tones, word2ph = _g2p(sentences) phones, tones, word2ph = _g2p(sentences)
assert sum(word2ph) == len(phones) assert sum(word2ph) == len(phones)
assert len(word2ph) == len(text) # Sometimes it will crash,you can add a try-catch. assert len(word2ph) == len(text) # Sometimes it will crash,you can add a try-catch.
phones = ['_'] + phones + ["_"] phones = ["_"] + phones + ["_"]
tones = [0] + tones + [0] tones = [0] + tones + [0]
word2ph = [1] + word2ph + [1] word2ph = [1] + word2ph + [1]
return phones, tones, word2ph return phones, tones, word2ph
@@ -76,10 +81,10 @@ def g2p(text):
def _get_initials_finals(word): def _get_initials_finals(word):
initials = [] initials = []
finals = [] finals = []
orig_initials = lazy_pinyin( orig_initials = lazy_pinyin(word, neutral_tone_with_five=True, style=Style.INITIALS)
word, neutral_tone_with_five=True, style=Style.INITIALS)
orig_finals = lazy_pinyin( orig_finals = lazy_pinyin(
word, neutral_tone_with_five=True, style=Style.FINALS_TONE3) word, neutral_tone_with_five=True, style=Style.FINALS_TONE3
)
for c, v in zip(orig_initials, orig_finals): for c, v in zip(orig_initials, orig_finals):
initials.append(c) initials.append(c)
finals.append(v) finals.append(v)
@@ -93,17 +98,16 @@ def _g2p(segments):
for seg in segments: for seg in segments:
pinyins = [] pinyins = []
# Replace all English words in the sentence # Replace all English words in the sentence
seg = re.sub('[a-zA-Z]+', '', seg) seg = re.sub("[a-zA-Z]+", "", seg)
seg_cut = psg.lcut(seg) seg_cut = psg.lcut(seg)
initials = [] initials = []
finals = [] finals = []
seg_cut = tone_modifier.pre_merge_for_modify(seg_cut) seg_cut = tone_modifier.pre_merge_for_modify(seg_cut)
for word, pos in seg_cut: for word, pos in seg_cut:
if pos == 'eng': if pos == "eng":
continue continue
sub_initials, sub_finals = _get_initials_finals(word) sub_initials, sub_finals = _get_initials_finals(word)
sub_finals = tone_modifier.modified_tone(word, pos, sub_finals = tone_modifier.modified_tone(word, pos, sub_finals)
sub_finals)
initials.append(sub_initials) initials.append(sub_initials)
finals.append(sub_finals) finals.append(sub_finals)
@@ -118,46 +122,46 @@ def _g2p(segments):
if c == v: if c == v:
assert c in punctuation assert c in punctuation
phone = [c] phone = [c]
tone = '0' tone = "0"
word2ph.append(1) word2ph.append(1)
else: else:
v_without_tone = v[:-1] v_without_tone = v[:-1]
tone = v[-1] tone = v[-1]
pinyin = c + v_without_tone pinyin = c + v_without_tone
assert tone in '12345' assert tone in "12345"
if c: if c:
# 多音节 # 多音节
v_rep_map = { v_rep_map = {
"uei": 'ui', "uei": "ui",
'iou': 'iu', "iou": "iu",
'uen': 'un', "uen": "un",
} }
if v_without_tone in v_rep_map.keys(): if v_without_tone in v_rep_map.keys():
pinyin = c + v_rep_map[v_without_tone] pinyin = c + v_rep_map[v_without_tone]
else: else:
# 单音节 # 单音节
pinyin_rep_map = { pinyin_rep_map = {
'ing': 'ying', "ing": "ying",
'i': 'yi', "i": "yi",
'in': 'yin', "in": "yin",
'u': 'wu', "u": "wu",
} }
if pinyin in pinyin_rep_map.keys(): if pinyin in pinyin_rep_map.keys():
pinyin = pinyin_rep_map[pinyin] pinyin = pinyin_rep_map[pinyin]
else: else:
single_rep_map = { single_rep_map = {
'v': 'yu', "v": "yu",
'e': 'e', "e": "e",
'i': 'y', "i": "y",
'u': 'w', "u": "w",
} }
if pinyin[0] in single_rep_map.keys(): if pinyin[0] in single_rep_map.keys():
pinyin = single_rep_map[pinyin[0]] + pinyin[1:] pinyin = single_rep_map[pinyin[0]] + pinyin[1:]
assert pinyin in pinyin_to_symbol_map.keys(), (pinyin, seg, raw_pinyin) assert pinyin in pinyin_to_symbol_map.keys(), (pinyin, seg, raw_pinyin)
phone = pinyin_to_symbol_map[pinyin].split(' ') phone = pinyin_to_symbol_map[pinyin].split(" ")
word2ph.append(len(phone)) word2ph.append(len(phone))
phones_list += phone phones_list += phone
@@ -165,20 +169,23 @@ def _g2p(segments):
return phones_list, tones_list, word2ph return phones_list, tones_list, word2ph
def text_normalize(text): def text_normalize(text):
numbers = re.findall(r'\d+(?:\.?\d+)?', text) numbers = re.findall(r"\d+(?:\.?\d+)?", text)
for number in numbers: for number in numbers:
text = text.replace(number, cn2an.an2cn(number), 1) text = text.replace(number, cn2an.an2cn(number), 1)
text = replace_punctuation(text) text = replace_punctuation(text)
return text return text
def get_bert_feature(text, word2ph): def get_bert_feature(text, word2ph):
from text import chinese_bert from text import chinese_bert
return chinese_bert.get_bert_feature(text, word2ph) return chinese_bert.get_bert_feature(text, word2ph)
if __name__ == '__main__':
if __name__ == "__main__":
from text.chinese_bert import get_bert_feature from text.chinese_bert import get_bert_feature
text = "啊！但是《原神》是由,米哈\游自主， [研发]的一款全.新开放世界.冒险游戏" text = "啊！但是《原神》是由,米哈\游自主， [研发]的一款全.新开放世界.冒险游戏"
text = text_normalize(text) text = text_normalize(text)
print(text) print(text)

61
text/chinese_bert.py
View File

@@ -4,18 +4,25 @@ from transformers import AutoTokenizer, AutoModelForMaskedLM
tokenizer = AutoTokenizer.from_pretrained("./bert/chinese-roberta-wwm-ext-large") tokenizer = AutoTokenizer.from_pretrained("./bert/chinese-roberta-wwm-ext-large")
def get_bert_feature(text, word2ph, device=None): def get_bert_feature(text, word2ph, device=None):
if sys.platform == "darwin" and torch.backends.mps.is_available() and device == "cpu": if (
sys.platform == "darwin"
and torch.backends.mps.is_available()
and device == "cpu"
):
device = "mps" device = "mps"
if not device: if not device:
device = "cuda" device = "cuda"
model = AutoModelForMaskedLM.from_pretrained("./bert/chinese-roberta-wwm-ext-large").to(device) model = AutoModelForMaskedLM.from_pretrained(
"./bert/chinese-roberta-wwm-ext-large"
).to(device)
with torch.no_grad(): with torch.no_grad():
inputs = tokenizer(text, return_tensors='pt') inputs = tokenizer(text, return_tensors="pt")
for i in inputs: for i in inputs:
inputs[i] = inputs[i].to(device) inputs[i] = inputs[i].to(device)
res = model(**inputs, output_hidden_states=True) res = model(**inputs, output_hidden_states=True)
res = torch.cat(res['hidden_states'][-3:-2], -1)[0].cpu() res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
assert len(word2ph) == len(text) + 2 assert len(word2ph) == len(text) + 2
word2phone = word2ph word2phone = word2ph
@@ -26,14 +33,53 @@ def get_bert_feature(text, word2ph, device=None):
phone_level_feature = torch.cat(phone_level_feature, dim=0) phone_level_feature = torch.cat(phone_level_feature, dim=0)
return phone_level_feature.T return phone_level_feature.T
if __name__ == '__main__':
if __name__ == "__main__":
import torch import torch
word_level_feature = torch.rand(38, 1024) # 12个词,每个词1024维特征 word_level_feature = torch.rand(38, 1024) # 12个词,每个词1024维特征
word2phone = [1, 2, 1, 2, 2, 1, 2, 2, 1, 2, 2, 1, 2, 2, 2, 2, 2, 1, 1, 2, 2, 1, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 1] word2phone = [
1,
2,
1,
2,
2,
1,
2,
2,
1,
2,
2,
1,
2,
2,
2,
2,
2,
1,
1,
2,
2,
1,
2,
2,
2,
2,
1,
2,
2,
2,
2,
2,
1,
2,
2,
2,
2,
1,
]
# 计算总帧数 # 计算总帧数
total_frames = sum(word2phone) total_frames = sum(word2phone)
@@ -49,4 +95,3 @@ if __name__ == '__main__':
phone_level_feature = torch.cat(phone_level_feature, dim=0) phone_level_feature = torch.cat(phone_level_feature, dim=0)
print(phone_level_feature.shape) # torch.Size([36, 1024]) print(phone_level_feature.shape) # torch.Size([36, 1024])

10
text/cleaner.py
View File

@@ -1,10 +1,7 @@
from text import chinese, japanese, cleaned_text_to_sequence from text import chinese, japanese, cleaned_text_to_sequence
language_module_map = { language_module_map = {"ZH": chinese, "JP": japanese}
'ZH': chinese,
'JP': japanese
}
def clean_text(text, language): def clean_text(text, language):
@@ -13,6 +10,7 @@ def clean_text(text, language):
phones, tones, word2ph = language_module.g2p(norm_text) phones, tones, word2ph = language_module.g2p(norm_text)
return norm_text, phones, tones, word2ph return norm_text, phones, tones, word2ph
def clean_text_bert(text, language): def clean_text_bert(text, language):
language_module = language_module_map[language] language_module = language_module_map[language]
norm_text = language_module.text_normalize(text) norm_text = language_module.text_normalize(text)
@@ -20,9 +18,11 @@ def clean_text_bert(text, language):
bert = language_module.get_bert_feature(norm_text, word2ph) bert = language_module.get_bert_feature(norm_text, word2ph)
return phones, tones, bert return phones, tones, bert
def text_to_sequence(text, language): def text_to_sequence(text, language):
norm_text, phones, tones, word2ph = clean_text(text, language) norm_text, phones, tones, word2ph = clean_text(text, language)
return cleaned_text_to_sequence(phones, tones, language) return cleaned_text_to_sequence(phones, tones, language)
if __name__ == '__main__':
if __name__ == "__main__":
pass pass

119
text/english.py
View File

@@ -7,35 +7,108 @@ from string import punctuation
from text import symbols from text import symbols
current_file_path = os.path.dirname(__file__) current_file_path = os.path.dirname(__file__)
CMU_DICT_PATH = os.path.join(current_file_path, 'cmudict.rep') CMU_DICT_PATH = os.path.join(current_file_path, "cmudict.rep")
CACHE_PATH = os.path.join(current_file_path, 'cmudict_cache.pickle') CACHE_PATH = os.path.join(current_file_path, "cmudict_cache.pickle")
_g2p = G2p() _g2p = G2p()
arpa = {'AH0', 'S', 'AH1', 'EY2', 'AE2', 'EH0', 'OW2', 'UH0', 'NG', 'B', 'G', 'AY0', 'M', 'AA0', 'F', 'AO0', 'ER2', 'UH1', 'IY1', 'AH2', 'DH', 'IY0', 'EY1', 'IH0', 'K', 'N', 'W', 'IY2', 'T', 'AA1', 'ER1', 'EH2', 'OY0', 'UH2', 'UW1', 'Z', 'AW2', 'AW1', 'V', 'UW2', 'AA2', 'ER', 'AW0', 'UW0', 'R', 'OW1', 'EH1', 'ZH', 'AE0', 'IH2', 'IH', 'Y', 'JH', 'P', 'AY1', 'EY0', 'OY2', 'TH', 'HH', 'D', 'ER0', 'CH', 'AO1', 'AE1', 'AO2', 'OY1', 'AY2', 'IH1', 'OW0', 'L', 'SH'} arpa = {
"AH0",
"S",
"AH1",
"EY2",
"AE2",
"EH0",
"OW2",
"UH0",
"NG",
"B",
"G",
"AY0",
"M",
"AA0",
"F",
"AO0",
"ER2",
"UH1",
"IY1",
"AH2",
"DH",
"IY0",
"EY1",
"IH0",
"K",
"N",
"W",
"IY2",
"T",
"AA1",
"ER1",
"EH2",
"OY0",
"UH2",
"UW1",
"Z",
"AW2",
"AW1",
"V",
"UW2",
"AA2",
"ER",
"AW0",
"UW0",
"R",
"OW1",
"EH1",
"ZH",
"AE0",
"IH2",
"IH",
"Y",
"JH",
"P",
"AY1",
"EY0",
"OY2",
"TH",
"HH",
"D",
"ER0",
"CH",
"AO1",
"AE1",
"AO2",
"OY1",
"AY2",
"IH1",
"OW0",
"L",
"SH",
}
def post_replace_ph(ph): def post_replace_ph(ph):
rep_map = { rep_map = {
'：': ',', "：": ",",
'；': ',', "；": ",",
'，': ',', "，": ",",
'。': '.', "。": ".",
'！': '!', "！": "!",
'？': '?', "？": "?",
'\n': '.', "\n": ".",
"·": ",", "·": ",",
'、': ",", "、": ",",
'...': '…', "...": "…",
'v': "V" "v": "V",
} }
if ph in rep_map.keys(): if ph in rep_map.keys():
ph = rep_map[ph] ph = rep_map[ph]
if ph in symbols: if ph in symbols:
return ph return ph
if ph not in symbols: if ph not in symbols:
ph = 'UNK' ph = "UNK"
return ph return ph
def read_dict(): def read_dict():
g2p_dict = {} g2p_dict = {}
start_line = 49 start_line = 49
@@ -45,13 +118,13 @@ def read_dict():
while line: while line:
if line_index >= start_line: if line_index >= start_line:
line = line.strip() line = line.strip()
word_split = line.split(' ') word_split = line.split(" ")
word = word_split[0] word = word_split[0]
syllable_split = word_split[1].split(' - ') syllable_split = word_split[1].split(" - ")
g2p_dict[word] = [] g2p_dict[word] = []
for syllable in syllable_split: for syllable in syllable_split:
phone_split = syllable.split(' ') phone_split = syllable.split(" ")
g2p_dict[word].append(phone_split) g2p_dict[word].append(phone_split)
line_index = line_index + 1 line_index = line_index + 1
@@ -61,13 +134,13 @@ def read_dict():
def cache_dict(g2p_dict, file_path): def cache_dict(g2p_dict, file_path):
with open(file_path, 'wb') as pickle_file: with open(file_path, "wb") as pickle_file:
pickle.dump(g2p_dict, pickle_file) pickle.dump(g2p_dict, pickle_file)
def get_dict(): def get_dict():
if os.path.exists(CACHE_PATH): if os.path.exists(CACHE_PATH):
with open(CACHE_PATH, 'rb') as pickle_file: with open(CACHE_PATH, "rb") as pickle_file:
g2p_dict = pickle.load(pickle_file) g2p_dict = pickle.load(pickle_file)
else: else:
g2p_dict = read_dict() g2p_dict = read_dict()
@@ -75,15 +148,18 @@ def get_dict():
return g2p_dict return g2p_dict
eng_dict = get_dict() eng_dict = get_dict()
def refine_ph(phn): def refine_ph(phn):
tone = 0 tone = 0
if re.search(r'\d$', phn): if re.search(r"\d$", phn):
tone = int(phn[-1]) + 1 tone = int(phn[-1]) + 1
phn = phn[:-1] phn = phn[:-1]
return phn.lower(), tone return phn.lower(), tone
def refine_syllables(syllables): def refine_syllables(syllables):
tones = [] tones = []
phonemes = [] phonemes = []
@@ -100,8 +176,8 @@ def text_normalize(text):
# todo: eng text normalize # todo: eng text normalize
return text return text
def g2p(text):
def g2p(text):
phones = [] phones = []
tones = [] tones = []
words = re.split(r"([,;.\-\?\!\s+])", text) words = re.split(r"([,;.\-\?\!\s+])", text)
@@ -126,6 +202,7 @@ def g2p(text):
phones = [post_replace_ph(i) for i in phones] phones = [post_replace_ph(i) for i in phones]
return phones, tones, word2ph return phones, tones, word2ph
if __name__ == "__main__": if __name__ == "__main__":
# print(get_dict()) # print(get_dict())
# print(eng_word_to_phoneme("hello")) # print(eng_word_to_phoneme("hello"))

54
text/japanese.py
View File

@@ -331,12 +331,12 @@ def kata2phoneme(text: str) -> str:
x = _RULEMAP2.get(text[:2]) x = _RULEMAP2.get(text[:2])
if x is not None: if x is not None:
text = text[2:] text = text[2:]
res += x.split(' ')[1:] res += x.split(" ")[1:]
continue continue
x = _RULEMAP1.get(text[0]) x = _RULEMAP1.get(text[0])
if x is not None: if x is not None:
text = text[1:] text = text[1:]
res += x.split(' ')[1:] res += x.split(" ")[1:]
continue continue
res.append(text[0]) res.append(text[0])
text = text[1:] text = text[1:]
@@ -358,6 +358,7 @@ _SYMBOL_TOKENS = set(list("・、。？！"))
_NO_YOMI_TOKENS = set(list("「」『』―（）［］[]")) _NO_YOMI_TOKENS = set(list("「」『』―（）［］[]"))
_TAGGER = MeCab.Tagger() _TAGGER = MeCab.Tagger()
def text2kata(text: str) -> str: def text2kata(text: str) -> str:
parsed = _TAGGER.parse(text) parsed = _TAGGER.parse(text)
res = [] res = []
@@ -472,6 +473,7 @@ def japanese_text_to_phonemes(text: str) -> str:
res = kata2phoneme(res) res = kata2phoneme(res)
return res return res
def is_japanese_character(char): def is_japanese_character(char):
# 定义日语文字系统的 Unicode 范围 # 定义日语文字系统的 Unicode 范围
japanese_ranges = [ japanese_ranges = [
@@ -493,27 +495,37 @@ def is_japanese_character(char):
return False return False
rep_map = { rep_map = {
'：': ',', "：": ",",
'；': ',', "；": ",",
'，': ',', "，": ",",
'。': '.', "。": ".",
'！': '!', "！": "!",
'？': '?', "？": "?",
'\n': '.', "\n": ".",
"·": ",", "·": ",",
'、': ",", "、": ",",
'...': '…' "...": "…",
} }
def replace_punctuation(text): def replace_punctuation(text):
pattern = re.compile('|'.join(re.escape(p) for p in rep_map.keys())) pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys()))
replaced_text = pattern.sub(lambda x: rep_map[x.group()], text) replaced_text = pattern.sub(lambda x: rep_map[x.group()], text)
replaced_text = re.sub(r'[^\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\u3400-\u4DBF'+"".join(punctuation)+r']+', '', replaced_text) replaced_text = re.sub(
r"[^\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\u3400-\u4DBF"
+ "".join(punctuation)
+ r"]+",
"",
replaced_text,
)
return replaced_text return replaced_text
def text_normalize(text): def text_normalize(text):
res = unicodedata.normalize("NFKC", text) res = unicodedata.normalize("NFKC", text)
res = japanese_convert_numbers_to_words(res) res = japanese_convert_numbers_to_words(res)
@@ -521,6 +533,7 @@ def text_normalize(text):
res = replace_punctuation(res) res = replace_punctuation(res)
return res return res
def distribute_phone(n_phone, n_word): def distribute_phone(n_phone, n_word):
phones_per_word = [0] * n_word phones_per_word = [0] * n_word
for task in range(n_phone): for task in range(n_phone):
@@ -529,16 +542,19 @@ def distribute_phone(n_phone, n_word):
phones_per_word[min_index] += 1 phones_per_word[min_index] += 1
return phones_per_word return phones_per_word
tokenizer = AutoTokenizer.from_pretrained("./bert/bert-base-japanese-v3") tokenizer = AutoTokenizer.from_pretrained("./bert/bert-base-japanese-v3")
def g2p(norm_text): def g2p(norm_text):
tokenized = tokenizer.tokenize(norm_text) tokenized = tokenizer.tokenize(norm_text)
phs = [] phs = []
ph_groups = [] ph_groups = []
for t in tokenized: for t in tokenized:
if not t.startswith('#'): if not t.startswith("#"):
ph_groups.append([t]) ph_groups.append([t])
else: else:
ph_groups[-1].append(t.replace("#", '')) ph_groups[-1].append(t.replace("#", ""))
word2ph = [] word2ph = []
for group in ph_groups: for group in ph_groups:
phonemes = kata2phoneme(text2kata("".join(group))) phonemes = kata2phoneme(text2kata("".join(group)))
@@ -552,12 +568,13 @@ def g2p(norm_text):
word2ph += aaa word2ph += aaa
phs += phonemes phs += phonemes
phones = ['_'] + phs + ["_"] phones = ["_"] + phs + ["_"]
tones = [0 for i in phones] tones = [0 for i in phones]
word2ph = [1] + word2ph + [1] word2ph = [1] + word2ph + [1]
return phones, tones, word2ph return phones, tones, word2ph
if __name__ == '__main__':
if __name__ == "__main__":
tokenizer = AutoTokenizer.from_pretrained("./bert/bert-base-japanese-v3") tokenizer = AutoTokenizer.from_pretrained("./bert/bert-base-japanese-v3")
text = "hello,こんにちは、世界！……" text = "hello,こんにちは、世界！……"
from text.japanese_bert import get_bert_feature from text.japanese_bert import get_bert_feature
@@ -568,6 +585,3 @@ if __name__ == '__main__':
bert = get_bert_feature(text, word2ph) bert = get_bert_feature(text, word2ph)
print(phones, tones, word2ph, bert.shape) print(phones, tones, word2ph, bert.shape)

17
text/japanese_bert.py
View File

@@ -4,19 +4,26 @@ import sys
tokenizer = AutoTokenizer.from_pretrained("./bert/bert-base-japanese-v3") tokenizer = AutoTokenizer.from_pretrained("./bert/bert-base-japanese-v3")
def get_bert_feature(text, word2ph, device=None): def get_bert_feature(text, word2ph, device=None):
if sys.platform == "darwin" and torch.backends.mps.is_available() and device == "cpu": if (
sys.platform == "darwin"
and torch.backends.mps.is_available()
and device == "cpu"
):
device = "mps" device = "mps"
if not device: if not device:
device = "cuda" device = "cuda"
model = AutoModelForMaskedLM.from_pretrained("./bert/bert-base-japanese-v3").to(device) model = AutoModelForMaskedLM.from_pretrained("./bert/bert-base-japanese-v3").to(
device
)
with torch.no_grad(): with torch.no_grad():
inputs = tokenizer(text, return_tensors='pt') inputs = tokenizer(text, return_tensors="pt")
for i in inputs: for i in inputs:
inputs[i] = inputs[i].to(device) inputs[i] = inputs[i].to(device)
res = model(**inputs, output_hidden_states=True) res = model(**inputs, output_hidden_states=True)
res = torch.cat(res['hidden_states'][-3:-2], -1)[0].cpu() res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
assert inputs['input_ids'].shape[-1] == len(word2ph) assert inputs["input_ids"].shape[-1] == len(word2ph)
word2phone = word2ph word2phone = word2ph
phone_level_feature = [] phone_level_feature = []
for i in range(len(word2phone)): for i in range(len(word2phone)):

181
text/symbols.py
View File

@@ -1,26 +1,166 @@
punctuation = ['!', '?', '…', ",", ".", "'", '-'] punctuation = ["!", "?", "…", ",", ".", "'", "-"]
pu_symbols = punctuation + ["SP", "UNK"] pu_symbols = punctuation + ["SP", "UNK"]
pad = '_' pad = "_"
# chinese # chinese
zh_symbols = ['E', 'En', 'a', 'ai', 'an', 'ang', 'ao', 'b', 'c', 'ch', 'd', 'e', 'ei', 'en', 'eng', 'er', 'f', 'g', 'h', zh_symbols = [
'i', 'i0', 'ia', 'ian', 'iang', 'iao', 'ie', 'in', 'ing', 'iong', 'ir', 'iu', 'j', 'k', 'l', 'm', 'n', 'o', "E",
'ong', "En",
'ou', 'p', 'q', 'r', 's', 'sh', 't', 'u', 'ua', 'uai', 'uan', 'uang', 'ui', 'un', 'uo', 'v', 'van', 've', 'vn', "a",
'w', 'x', 'y', 'z', 'zh', "ai",
"AA", "EE", "OO"] "an",
"ang",
"ao",
"b",
"c",
"ch",
"d",
"e",
"ei",
"en",
"eng",
"er",
"f",
"g",
"h",
"i",
"i0",
"ia",
"ian",
"iang",
"iao",
"ie",
"in",
"ing",
"iong",
"ir",
"iu",
"j",
"k",
"l",
"m",
"n",
"o",
"ong",
"ou",
"p",
"q",
"r",
"s",
"sh",
"t",
"u",
"ua",
"uai",
"uan",
"uang",
"ui",
"un",
"uo",
"v",
"van",
"ve",
"vn",
"w",
"x",
"y",
"z",
"zh",
"AA",
"EE",
"OO",
]
num_zh_tones = 6 num_zh_tones = 6
# japanese # japanese
ja_symbols = ['N', 'a', 'a:', 'b', 'by', 'ch', 'd', 'dy', 'e', 'e:', 'f', 'g', 'gy', 'h', 'hy', 'i', 'i:', 'j', 'k', 'ky', ja_symbols = [
'm', 'my', 'n', 'ny', 'o', 'o:', 'p', 'py', 'q', 'r', 'ry', 's', 'sh', 't', 'ts', 'ty', 'u', 'u:', "N",
'w', 'y', 'z', 'zy'] "a",
"a:",
"b",
"by",
"ch",
"d",
"dy",
"e",
"e:",
"f",
"g",
"gy",
"h",
"hy",
"i",
"i:",
"j",
"k",
"ky",
"m",
"my",
"n",
"ny",
"o",
"o:",
"p",
"py",
"q",
"r",
"ry",
"s",
"sh",
"t",
"ts",
"ty",
"u",
"u:",
"w",
"y",
"z",
"zy",
]
num_ja_tones = 1 num_ja_tones = 1
# English # English
en_symbols = ['aa', 'ae', 'ah', 'ao', 'aw', 'ay', 'b', 'ch', 'd', 'dh', 'eh', 'er', 'ey', 'f', 'g', 'hh', 'ih', 'iy', en_symbols = [
'jh', 'k', 'l', 'm', 'n', 'ng', 'ow', 'oy', 'p', 'r', 's', "aa",
'sh', 't', 'th', 'uh', 'uw', 'V', 'w', 'y', 'z', 'zh'] "ae",
"ah",
"ao",
"aw",
"ay",
"b",
"ch",
"d",
"dh",
"eh",
"er",
"ey",
"f",
"g",
"hh",
"ih",
"iy",
"jh",
"k",
"l",
"m",
"n",
"ng",
"ow",
"oy",
"p",
"r",
"s",
"sh",
"t",
"th",
"uh",
"uw",
"V",
"w",
"y",
"z",
"zh",
]
num_en_tones = 4 num_en_tones = 4
# combine all symbols # combine all symbols
@@ -32,21 +172,16 @@ sil_phonemes_ids = [symbols.index(i) for i in pu_symbols]
num_tones = num_zh_tones + num_ja_tones + num_en_tones num_tones = num_zh_tones + num_ja_tones + num_en_tones
# language maps # language maps
language_id_map = { language_id_map = {"ZH": 0, "JP": 1, "EN": 2}
'ZH': 0,
"JP": 1,
"EN": 2
}
num_languages = len(language_id_map.keys()) num_languages = len(language_id_map.keys())
language_tone_start_map = { language_tone_start_map = {
'ZH': 0, "ZH": 0,
"JP": num_zh_tones, "JP": num_zh_tones,
"EN": num_zh_tones + num_ja_tones "EN": num_zh_tones + num_ja_tones,
} }
if __name__ == '__main__': if __name__ == "__main__":
a = set(zh_symbols) a = set(zh_symbols)
b = set(en_symbols) b = set(en_symbols)
print(sorted(a & b)) print(sorted(a & b))

602
text/tone_sandhi.py
View File

@@ -19,51 +19,442 @@ from pypinyin import lazy_pinyin
from pypinyin import Style from pypinyin import Style
class ToneSandhi(): class ToneSandhi:
def __init__(self): def __init__(self):
self.must_neural_tone_words = { self.must_neural_tone_words = {
'麻烦', '麻利', '鸳鸯', '高粱', '骨头', '骆驼', '马虎', '首饰', '馒头', '馄饨', '风筝', "麻烦",
'难为', '队伍', '阔气', '闺女', '门道', '锄头', '铺盖', '铃铛', '铁匠', '钥匙', '里脊', "麻利",
'里头', '部分', '那么', '道士', '造化', '迷糊', '连累', '这么', '这个', '运气', '过去', "鸳鸯",
'软和', '转悠', '踏实', '跳蚤', '跟头', '趔趄', '财主', '豆腐', '讲究', '记性', '记号', "高粱",
'认识', '规矩', '见识', '裁缝', '补丁', '衣裳', '衣服', '衙门', '街坊', '行李', '行当', "骨头",
'蛤蟆', '蘑菇', '薄荷', '葫芦', '葡萄', '萝卜', '荸荠', '苗条', '苗头', '苍蝇', '芝麻', "骆驼",
'舒服', '舒坦', '舌头', '自在', '膏药', '脾气', '脑袋', '脊梁', '能耐', '胳膊', '胭脂', "马虎",
'胡萝', '胡琴', '胡同', '聪明', '耽误', '耽搁', '耷拉', '耳朵', '老爷', '老实', '老婆', "首饰",
'老头', '老太', '翻腾', '罗嗦', '罐头', '编辑', '结实', '红火', '累赘', '糨糊', '糊涂', "馒头",
'精神', '粮食', '簸箕', '篱笆', '算计', '算盘', '答应', '笤帚', '笑语', '笑话', '窟窿', "馄饨",
'窝囊', '窗户', '稳当', '稀罕', '称呼', '秧歌', '秀气', '秀才', '福气', '祖宗', '砚台', "风筝",
'码头', '石榴', '石头', '石匠', '知识', '眼睛', '眯缝', '眨巴', '眉毛', '相声', '盘算', "难为",
'白净', '痢疾', '痛快', '疟疾', '疙瘩', '疏忽', '畜生', '生意', '甘蔗', '琵琶', '琢磨', "队伍",
'琉璃', '玻璃', '玫瑰', '玄乎', '狐狸', '状元', '特务', '牲口', '牙碜', '牌楼', '爽快', "阔气",
'爱人', '热闹', '烧饼', '烟筒', '烂糊', '点心', '炊帚', '灯笼', '火候', '漂亮', '滑溜', "闺女",
'溜达', '温和', '清楚', '消息', '浪头', '活泼', '比方', '正经', '欺负', '模糊', '槟榔', "门道",
'棺材', '棒槌', '棉花', '核桃', '栅栏', '柴火', '架势', '枕头', '枇杷', '机灵', '本事', "锄头",
'木头', '木匠', '朋友', '月饼', '月亮', '暖和', '明白', '时候', '新鲜', '故事', '收拾', "铺盖",
'收成', '提防', '挖苦', '挑剔', '指甲', '指头', '拾掇', '拳头', '拨弄', '招牌', '招呼', "铃铛",
'抬举', '护士', '折腾', '扫帚', '打量', '打算', '打点', '打扮', '打听', '打发', '扎实', "铁匠",
'扁担', '戒指', '懒得', '意识', '意思', '情形', '悟性', '怪物', '思量', '怎么', '念头', "钥匙",
'念叨', '快活', '忙活', '志气', '心思', '得罪', '张罗', '弟兄', '开通', '应酬', '庄稼', "里脊",
'干事', '帮手', '帐篷', '希罕', '师父', '师傅', '巴结', '巴掌', '差事', '工夫', '岁数', "里头",
'屁股', '尾巴', '少爷', '小气', '小伙', '将就', '对头', '对付', '寡妇', '家伙', '客气', "部分",
'实在', '官司', '学问', '学生', '字号', '嫁妆', '媳妇', '媒人', '婆家', '娘家', '委屈', "那么",
'姑娘', '姐夫', '妯娌', '妥当', '妖精', '奴才', '女婿', '头发', '太阳', '大爷', '大方', "道士",
'大意', '大夫', '多少', '多么', '外甥', '壮实', '地道', '地方', '在乎', '困难', '嘴巴', "造化",
'嘱咐', '嘟囔', '嘀咕', '喜欢', '喇嘛', '喇叭', '商量', '唾沫', '哑巴', '哈欠', '哆嗦', "迷糊",
'咳嗽', '和尚', '告诉', '告示', '含糊', '吓唬', '后头', '名字', '名堂', '合同', '吆喝', "连累",
'叫唤', '口袋', '厚道', '厉害', '千斤', '包袱', '包涵', '匀称', '勤快', '动静', '动弹', "这么",
'功夫', '力气', '前头', '刺猬', '刺激', '别扭', '利落', '利索', '利害', '分析', '出息', "这个",
'凑合', '凉快', '冷战', '冤枉', '冒失', '养活', '关系', '先生', '兄弟', '便宜', '使唤', "运气",
'佩服', '作坊', '体面', '位置', '似的', '伙计', '休息', '什么', '人家', '亲戚', '亲家', "过去",
'交情', '云彩', '事情', '买卖', '主意', '丫头', '丧气', '两口', '东西', '东家', '世故', "软和",
'不由', '不在', '下水', '下巴', '上头', '上司', '丈夫', '丈人', '一辈', '那个', '菩萨', "转悠",
'父亲', '母亲', '咕噜', '邋遢', '费用', '冤家', '甜头', '介绍', '荒唐', '大人', '泥鳅', "踏实",
'幸福', '熟悉', '计划', '扑腾', '蜡烛', '姥爷', '照顾', '喉咙', '吉他', '弄堂', '蚂蚱', "跳蚤",
'凤凰', '拖沓', '寒碜', '糟蹋', '倒腾', '报复', '逻辑', '盘缠', '喽啰', '牢骚', '咖喱', "跟头",
'扫把', '惦记' "趔趄",
"财主",
"豆腐",
"讲究",
"记性",
"记号",
"认识",
"规矩",
"见识",
"裁缝",
"补丁",
"衣裳",
"衣服",
"衙门",
"街坊",
"行李",
"行当",
"蛤蟆",
"蘑菇",
"薄荷",
"葫芦",
"葡萄",
"萝卜",
"荸荠",
"苗条",
"苗头",
"苍蝇",
"芝麻",
"舒服",
"舒坦",
"舌头",
"自在",
"膏药",
"脾气",
"脑袋",
"脊梁",
"能耐",
"胳膊",
"胭脂",
"胡萝",
"胡琴",
"胡同",
"聪明",
"耽误",
"耽搁",
"耷拉",
"耳朵",
"老爷",
"老实",
"老婆",
"老头",
"老太",
"翻腾",
"罗嗦",
"罐头",
"编辑",
"结实",
"红火",
"累赘",
"糨糊",
"糊涂",
"精神",
"粮食",
"簸箕",
"篱笆",
"算计",
"算盘",
"答应",
"笤帚",
"笑语",
"笑话",
"窟窿",
"窝囊",
"窗户",
"稳当",
"稀罕",
"称呼",
"秧歌",
"秀气",
"秀才",
"福气",
"祖宗",
"砚台",
"码头",
"石榴",
"石头",
"石匠",
"知识",
"眼睛",
"眯缝",
"眨巴",
"眉毛",
"相声",
"盘算",
"白净",
"痢疾",
"痛快",
"疟疾",
"疙瘩",
"疏忽",
"畜生",
"生意",
"甘蔗",
"琵琶",
"琢磨",
"琉璃",
"玻璃",
"玫瑰",
"玄乎",
"狐狸",
"状元",
"特务",
"牲口",
"牙碜",
"牌楼",
"爽快",
"爱人",
"热闹",
"烧饼",
"烟筒",
"烂糊",
"点心",
"炊帚",
"灯笼",
"火候",
"漂亮",
"滑溜",
"溜达",
"温和",
"清楚",
"消息",
"浪头",
"活泼",
"比方",
"正经",
"欺负",
"模糊",
"槟榔",
"棺材",
"棒槌",
"棉花",
"核桃",
"栅栏",
"柴火",
"架势",
"枕头",
"枇杷",
"机灵",
"本事",
"木头",
"木匠",
"朋友",
"月饼",
"月亮",
"暖和",
"明白",
"时候",
"新鲜",
"故事",
"收拾",
"收成",
"提防",
"挖苦",
"挑剔",
"指甲",
"指头",
"拾掇",
"拳头",
"拨弄",
"招牌",
"招呼",
"抬举",
"护士",
"折腾",
"扫帚",
"打量",
"打算",
"打点",
"打扮",
"打听",
"打发",
"扎实",
"扁担",
"戒指",
"懒得",
"意识",
"意思",
"情形",
"悟性",
"怪物",
"思量",
"怎么",
"念头",
"念叨",
"快活",
"忙活",
"志气",
"心思",
"得罪",
"张罗",
"弟兄",
"开通",
"应酬",
"庄稼",
"干事",
"帮手",
"帐篷",
"希罕",
"师父",
"师傅",
"巴结",
"巴掌",
"差事",
"工夫",
"岁数",
"屁股",
"尾巴",
"少爷",
"小气",
"小伙",
"将就",
"对头",
"对付",
"寡妇",
"家伙",
"客气",
"实在",
"官司",
"学问",
"学生",
"字号",
"嫁妆",
"媳妇",
"媒人",
"婆家",
"娘家",
"委屈",
"姑娘",
"姐夫",
"妯娌",
"妥当",
"妖精",
"奴才",
"女婿",
"头发",
"太阳",
"大爷",
"大方",
"大意",
"大夫",
"多少",
"多么",
"外甥",
"壮实",
"地道",
"地方",
"在乎",
"困难",
"嘴巴",
"嘱咐",
"嘟囔",
"嘀咕",
"喜欢",
"喇嘛",
"喇叭",
"商量",
"唾沫",
"哑巴",
"哈欠",
"哆嗦",
"咳嗽",
"和尚",
"告诉",
"告示",
"含糊",
"吓唬",
"后头",
"名字",
"名堂",
"合同",
"吆喝",
"叫唤",
"口袋",
"厚道",
"厉害",
"千斤",
"包袱",
"包涵",
"匀称",
"勤快",
"动静",
"动弹",
"功夫",
"力气",
"前头",
"刺猬",
"刺激",
"别扭",
"利落",
"利索",
"利害",
"分析",
"出息",
"凑合",
"凉快",
"冷战",
"冤枉",
"冒失",
"养活",
"关系",
"先生",
"兄弟",
"便宜",
"使唤",
"佩服",
"作坊",
"体面",
"位置",
"似的",
"伙计",
"休息",
"什么",
"人家",
"亲戚",
"亲家",
"交情",
"云彩",
"事情",
"买卖",
"主意",
"丫头",
"丧气",
"两口",
"东西",
"东家",
"世故",
"不由",
"不在",
"下水",
"下巴",
"上头",
"上司",
"丈夫",
"丈人",
"一辈",
"那个",
"菩萨",
"父亲",
"母亲",
"咕噜",
"邋遢",
"费用",
"冤家",
"甜头",
"介绍",
"荒唐",
"大人",
"泥鳅",
"幸福",
"熟悉",
"计划",
"扑腾",
"蜡烛",
"姥爷",
"照顾",
"喉咙",
"吉他",
"弄堂",
"蚂蚱",
"凤凰",
"拖沓",
"寒碜",
"糟蹋",
"倒腾",
"报复",
"逻辑",
"盘缠",
"喽啰",
"牢骚",
"咖喱",
"扫把",
"惦记",
} }
self.must_not_neural_tone_words = { self.must_not_neural_tone_words = {
"男子", "女子", "分子", "原子", "量子", "莲子", "石子", "瓜子", "电子", "人人", "虎虎" "男子",
"女子",
"分子",
"原子",
"量子",
"莲子",
"石子",
"瓜子",
"电子",
"人人",
"虎虎",
} }
self.punc = "：，；。？！“”‘’':,;.?!" self.punc = "：，；。？！“”‘’':,;.?!"
@@ -72,14 +463,15 @@ class ToneSandhi():
# word: "家里" # word: "家里"
# pos: "s" # pos: "s"
# finals: ['ia1', 'i3'] # finals: ['ia1', 'i3']
def _neural_sandhi(self, word: str, pos: str, def _neural_sandhi(self, word: str, pos: str, finals: List[str]) -> List[str]:
finals: List[str]) -> List[str]:
# reduplication words for n. and v. e.g. 奶奶, 试试, 旺旺 # reduplication words for n. and v. e.g. 奶奶, 试试, 旺旺
for j, item in enumerate(word): for j, item in enumerate(word):
if j - 1 >= 0 and item == word[j - 1] and pos[0] in { if (
"n", "v", "a" j - 1 >= 0
} and word not in self.must_not_neural_tone_words: and item == word[j - 1]
and pos[0] in {"n", "v", "a"}
and word not in self.must_not_neural_tone_words
):
finals[j] = finals[j][:-1] + "5" finals[j] = finals[j][:-1] + "5"
ge_idx = word.find("个") ge_idx = word.find("个")
if len(word) >= 1 and word[-1] in "吧呢啊呐噻嘛吖嗨呐哦哒额滴哩哟喽啰耶喔诶": if len(word) >= 1 and word[-1] in "吧呢啊呐噻嘛吖嗨呐哦哒额滴哩哟喽啰耶喔诶":
@@ -89,9 +481,12 @@ class ToneSandhi():
# e.g. 走了, 看着, 去过 # e.g. 走了, 看着, 去过
# elif len(word) == 1 and word in "了着过" and pos in {"ul", "uz", "ug"}: # elif len(word) == 1 and word in "了着过" and pos in {"ul", "uz", "ug"}:
# finals[-1] = finals[-1][:-1] + "5" # finals[-1] = finals[-1][:-1] + "5"
elif len(word) > 1 and word[-1] in "们子" and pos in { elif (
"r", "n" len(word) > 1
} and word not in self.must_not_neural_tone_words: and word[-1] in "们子"
and pos in {"r", "n"}
and word not in self.must_not_neural_tone_words
):
finals[-1] = finals[-1][:-1] + "5" finals[-1] = finals[-1][:-1] + "5"
# e.g. 桌上, 地下, 家里 # e.g. 桌上, 地下, 家里
elif len(word) > 1 and word[-1] in "上下里" and pos in {"s", "l", "f"}: elif len(word) > 1 and word[-1] in "上下里" and pos in {"s", "l", "f"}:
@@ -100,21 +495,26 @@ class ToneSandhi():
elif len(word) > 1 and word[-1] in "来去" and word[-2] in "上下进出回过起开": elif len(word) > 1 and word[-1] in "来去" and word[-2] in "上下进出回过起开":
finals[-1] = finals[-1][:-1] + "5" finals[-1] = finals[-1][:-1] + "5"
# 个做量词 # 个做量词
elif (ge_idx >= 1 and elif (
(word[ge_idx - 1].isnumeric() or ge_idx >= 1
word[ge_idx - 1] in "几有两半多各整每做是")) or word == '个': and (word[ge_idx - 1].isnumeric() or word[ge_idx - 1] in "几有两半多各整每做是")
) or word == "个":
finals[ge_idx] = finals[ge_idx][:-1] + "5" finals[ge_idx] = finals[ge_idx][:-1] + "5"
else: else:
if word in self.must_neural_tone_words or word[ if (
-2:] in self.must_neural_tone_words: word in self.must_neural_tone_words
or word[-2:] in self.must_neural_tone_words
):
finals[-1] = finals[-1][:-1] + "5" finals[-1] = finals[-1][:-1] + "5"
word_list = self._split_word(word) word_list = self._split_word(word)
finals_list = [finals[: len(word_list[0])], finals[len(word_list[0]) :]] finals_list = [finals[: len(word_list[0])], finals[len(word_list[0]) :]]
for i, word in enumerate(word_list): for i, word in enumerate(word_list):
# conventional neural in Chinese # conventional neural in Chinese
if word in self.must_neural_tone_words or word[ if (
-2:] in self.must_neural_tone_words: word in self.must_neural_tone_words
or word[-2:] in self.must_neural_tone_words
):
finals_list[i][-1] = finals_list[i][-1][:-1] + "5" finals_list[i][-1] = finals_list[i][-1][:-1] + "5"
finals = sum(finals_list, []) finals = sum(finals_list, [])
return finals return finals
@@ -126,15 +526,15 @@ class ToneSandhi():
else: else:
for i, char in enumerate(word): for i, char in enumerate(word):
# "不" before tone4 should be bu2, e.g. 不怕 # "不" before tone4 should be bu2, e.g. 不怕
if char == "不" and i + 1 < len(word) and finals[i + if char == "不" and i + 1 < len(word) and finals[i + 1][-1] == "4":
1][-1] == "4":
finals[i] = finals[i][:-1] + "2" finals[i] = finals[i][:-1] + "2"
return finals return finals
def _yi_sandhi(self, word: str, finals: List[str]) -> List[str]: def _yi_sandhi(self, word: str, finals: List[str]) -> List[str]:
# "一" in number sequences, e.g. 一零零, 二一零 # "一" in number sequences, e.g. 一零零, 二一零
if word.find("一") != -1 and all( if word.find("一") != -1 and all(
[item.isnumeric() for item in word if item != "一"]): [item.isnumeric() for item in word if item != "一"]
):
return finals return finals
# "一" between reduplication words shold be yi5, e.g. 看一看 # "一" between reduplication words shold be yi5, e.g. 看一看
elif len(word) == 3 and word[1] == "一" and word[0] == word[-1]: elif len(word) == 3 and word[1] == "一" and word[0] == word[-1]:
@@ -182,18 +582,19 @@ class ToneSandhi():
elif len(word_list[0]) == 1: elif len(word_list[0]) == 1:
finals[1] = finals[1][:-1] + "2" finals[1] = finals[1][:-1] + "2"
else: else:
finals_list = [ finals_list = [finals[: len(word_list[0])], finals[len(word_list[0]) :]]
finals[:len(word_list[0])], finals[len(word_list[0]):]
]
if len(finals_list) == 2: if len(finals_list) == 2:
for i, sub in enumerate(finals_list): for i, sub in enumerate(finals_list):
# e.g. 所有/人 # e.g. 所有/人
if self._all_tone_three(sub) and len(sub) == 2: if self._all_tone_three(sub) and len(sub) == 2:
finals_list[i][0] = finals_list[i][0][:-1] + "2" finals_list[i][0] = finals_list[i][0][:-1] + "2"
# e.g. 好/喜欢 # e.g. 好/喜欢
elif i == 1 and not self._all_tone_three(sub) and finals_list[i][0][-1] == "3" and \ elif (
finals_list[0][-1][-1] == "3": i == 1
and not self._all_tone_three(sub)
and finals_list[i][0][-1] == "3"
and finals_list[0][-1][-1] == "3"
):
finals_list[0][-1] = finals_list[0][-1][:-1] + "2" finals_list[0][-1] = finals_list[0][-1][:-1] + "2"
finals = sum(finals_list, []) finals = sum(finals_list, [])
# split idiom into two words who's length is 2 # split idiom into two words who's length is 2
@@ -222,7 +623,7 @@ class ToneSandhi():
new_seg.append((word, pos)) new_seg.append((word, pos))
last_word = word[:] last_word = word[:]
if last_word == "不": if last_word == "不":
new_seg.append((last_word, 'd')) new_seg.append((last_word, "d"))
last_word = "" last_word = ""
return new_seg return new_seg
@@ -236,12 +637,21 @@ class ToneSandhi():
new_seg = [] new_seg = []
# function 1 # function 1
for i, (word, pos) in enumerate(seg): for i, (word, pos) in enumerate(seg):
if i - 1 >= 0 and word == "一" and i + 1 < len(seg) and seg[i - 1][ if (
0] == seg[i + 1][0] and seg[i - 1][1] == "v": i - 1 >= 0
and word == "一"
and i + 1 < len(seg)
and seg[i - 1][0] == seg[i + 1][0]
and seg[i - 1][1] == "v"
):
new_seg[i - 1][0] = new_seg[i - 1][0] + "一" + new_seg[i - 1][0] new_seg[i - 1][0] = new_seg[i - 1][0] + "一" + new_seg[i - 1][0]
else: else:
if i - 2 >= 0 and seg[i - 1][0] == "一" and seg[i - 2][ if (
0] == word and pos == "v": i - 2 >= 0
and seg[i - 1][0] == "一"
and seg[i - 2][0] == word
and pos == "v"
):
continue continue
else: else:
new_seg.append([word, pos]) new_seg.append([word, pos])
@@ -257,22 +667,27 @@ class ToneSandhi():
# the first and the second words are all_tone_three # the first and the second words are all_tone_three
def _merge_continuous_three_tones( def _merge_continuous_three_tones(
self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]: self, seg: List[Tuple[str, str]]
) -> List[Tuple[str, str]]:
new_seg = [] new_seg = []
sub_finals_list = [ sub_finals_list = [
lazy_pinyin( lazy_pinyin(word, neutral_tone_with_five=True, style=Style.FINALS_TONE3)
word, neutral_tone_with_five=True, style=Style.FINALS_TONE3)
for (word, pos) in seg for (word, pos) in seg
] ]
assert len(sub_finals_list) == len(seg) assert len(sub_finals_list) == len(seg)
merge_last = [False] * len(seg) merge_last = [False] * len(seg)
for i, (word, pos) in enumerate(seg): for i, (word, pos) in enumerate(seg):
if i - 1 >= 0 and self._all_tone_three( if (
sub_finals_list[i - 1]) and self._all_tone_three( i - 1 >= 0
sub_finals_list[i]) and not merge_last[i - 1]: and self._all_tone_three(sub_finals_list[i - 1])
and self._all_tone_three(sub_finals_list[i])
and not merge_last[i - 1]
):
# if the last word is reduplication, not merge, because reduplication need to be _neural_sandhi # if the last word is reduplication, not merge, because reduplication need to be _neural_sandhi
if not self._is_reduplication(seg[i - 1][0]) and len( if (
seg[i - 1][0]) + len(seg[i][0]) <= 3: not self._is_reduplication(seg[i - 1][0])
and len(seg[i - 1][0]) + len(seg[i][0]) <= 3
):
new_seg[-1][0] = new_seg[-1][0] + seg[i][0] new_seg[-1][0] = new_seg[-1][0] + seg[i][0]
merge_last[i] = True merge_last[i] = True
else: else:
@@ -287,21 +702,27 @@ class ToneSandhi():
# the last char of first word and the first char of second word is tone_three # the last char of first word and the first char of second word is tone_three
def _merge_continuous_three_tones_2( def _merge_continuous_three_tones_2(
self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]: self, seg: List[Tuple[str, str]]
) -> List[Tuple[str, str]]:
new_seg = [] new_seg = []
sub_finals_list = [ sub_finals_list = [
lazy_pinyin( lazy_pinyin(word, neutral_tone_with_five=True, style=Style.FINALS_TONE3)
word, neutral_tone_with_five=True, style=Style.FINALS_TONE3)
for (word, pos) in seg for (word, pos) in seg
] ]
assert len(sub_finals_list) == len(seg) assert len(sub_finals_list) == len(seg)
merge_last = [False] * len(seg) merge_last = [False] * len(seg)
for i, (word, pos) in enumerate(seg): for i, (word, pos) in enumerate(seg):
if i - 1 >= 0 and sub_finals_list[i - 1][-1][-1] == "3" and sub_finals_list[i][0][-1] == "3" and not \ if (
merge_last[i - 1]: i - 1 >= 0
and sub_finals_list[i - 1][-1][-1] == "3"
and sub_finals_list[i][0][-1] == "3"
and not merge_last[i - 1]
):
# if the last word is reduplication, not merge, because reduplication need to be _neural_sandhi # if the last word is reduplication, not merge, because reduplication need to be _neural_sandhi
if not self._is_reduplication(seg[i - 1][0]) and len( if (
seg[i - 1][0]) + len(seg[i][0]) <= 3: not self._is_reduplication(seg[i - 1][0])
and len(seg[i - 1][0]) + len(seg[i][0]) <= 3
):
new_seg[-1][0] = new_seg[-1][0] + seg[i][0] new_seg[-1][0] = new_seg[-1][0] + seg[i][0]
merge_last[i] = True merge_last[i] = True
else: else:
@@ -319,8 +740,7 @@ class ToneSandhi():
new_seg.append([word, pos]) new_seg.append([word, pos])
return new_seg return new_seg
def _merge_reduplication( def _merge_reduplication(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
new_seg = [] new_seg = []
for i, (word, pos) in enumerate(seg): for i, (word, pos) in enumerate(seg):
if new_seg and word == new_seg[-1][0]: if new_seg and word == new_seg[-1][0]:
@@ -329,8 +749,7 @@ class ToneSandhi():
new_seg.append([word, pos]) new_seg.append([word, pos])
return new_seg return new_seg
def pre_merge_for_modify( def pre_merge_for_modify(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
seg = self._merge_bu(seg) seg = self._merge_bu(seg)
try: try:
seg = self._merge_yi(seg) seg = self._merge_yi(seg)
@@ -342,8 +761,7 @@ class ToneSandhi():
seg = self._merge_er(seg) seg = self._merge_er(seg)
return seg return seg
def modified_tone(self, word: str, pos: str, def modified_tone(self, word: str, pos: str, finals: List[str]) -> List[str]:
finals: List[str]) -> List[str]:
finals = self._bu_sandhi(word, finals) finals = self._bu_sandhi(word, finals)
finals = self._yi_sandhi(word, finals) finals = self._yi_sandhi(word, finals)
finals = self._neural_sandhi(word, pos, finals) finals = self._neural_sandhi(word, pos, finals)

389
train_ms.py
View File

@@ -14,37 +14,38 @@ from torch.nn.parallel import DistributedDataParallel as DDP
from torch.cuda.amp import autocast, GradScaler from torch.cuda.amp import autocast, GradScaler
from tqdm import tqdm from tqdm import tqdm
import logging import logging
logging.getLogger('numba').setLevel(logging.WARNING)
logging.getLogger("numba").setLevel(logging.WARNING)
import commons import commons
import utils import utils
from data_utils import ( from data_utils import (
TextAudioSpeakerLoader, TextAudioSpeakerLoader,
TextAudioSpeakerCollate, TextAudioSpeakerCollate,
DistributedBucketSampler DistributedBucketSampler,
) )
from models import ( from models import (
SynthesizerTrn, SynthesizerTrn,
MultiPeriodDiscriminator, MultiPeriodDiscriminator,
DurationDiscriminator, DurationDiscriminator,
) )
from losses import ( from losses import generator_loss, discriminator_loss, feature_loss, kl_loss
generator_loss,
discriminator_loss,
feature_loss,
kl_loss
)
from mel_processing import mel_spectrogram_torch, spec_to_mel_torch from mel_processing import mel_spectrogram_torch, spec_to_mel_torch
from text.symbols import symbols from text.symbols import symbols
torch.backends.cudnn.benchmark = True torch.backends.cudnn.benchmark = True
torch.backends.cuda.sdp_kernel("flash") torch.backends.cuda.sdp_kernel("flash")
torch.backends.cuda.enable_flash_sdp(True) torch.backends.cuda.enable_flash_sdp(True)
torch.backends.cuda.enable_mem_efficient_sdp(True) # Not avaliable if torch version is lower than 2.0 torch.backends.cuda.enable_mem_efficient_sdp(
True
) # Not avaliable if torch version is lower than 2.0
torch.backends.cuda.enable_math_sdp(True) torch.backends.cuda.enable_math_sdp(True)
global_step = 0 global_step = 0
def run(): def run():
dist.init_process_group(backend="nccl", init_method="env://") # Use torchrun instead of mp.spawn dist.init_process_group(
backend="nccl", init_method="env://"
) # Use torchrun instead of mp.spawn
rank = dist.get_rank() rank = dist.get_rank()
n_gpus = dist.get_world_size() n_gpus = dist.get_world_size()
hps = utils.get_hparams() hps = utils.get_hparams()
@@ -64,17 +65,34 @@ def run():
[32, 300, 400, 500, 600, 700, 800, 900, 1000], [32, 300, 400, 500, 600, 700, 800, 900, 1000],
num_replicas=n_gpus, num_replicas=n_gpus,
rank=rank, rank=rank,
shuffle=True) shuffle=True,
)
collate_fn = TextAudioSpeakerCollate() collate_fn = TextAudioSpeakerCollate()
train_loader = DataLoader(train_dataset, num_workers=16, shuffle=False, pin_memory=True, train_loader = DataLoader(
collate_fn=collate_fn, batch_sampler=train_sampler, train_dataset,
persistent_workers=True,prefetch_factor=4) #128G Memory suitable loader. num_workers=16,
shuffle=False,
pin_memory=True,
collate_fn=collate_fn,
batch_sampler=train_sampler,
persistent_workers=True,
prefetch_factor=4,
) # 128G Memory suitable loader.
if rank == 0: if rank == 0:
eval_dataset = TextAudioSpeakerLoader(hps.data.validation_files, hps.data) eval_dataset = TextAudioSpeakerLoader(hps.data.validation_files, hps.data)
eval_loader = DataLoader(eval_dataset, num_workers=0, shuffle=False, eval_loader = DataLoader(
batch_size=1, pin_memory=True, eval_dataset,
drop_last=False, collate_fn=collate_fn) num_workers=0,
if "use_noise_scaled_mas" in hps.model.keys() and hps.model.use_noise_scaled_mas == True: shuffle=False,
batch_size=1,
pin_memory=True,
drop_last=False,
collate_fn=collate_fn,
)
if (
"use_noise_scaled_mas" in hps.model.keys()
and hps.model.use_noise_scaled_mas == True
):
print("Using noise scaled MAS for VITS2") print("Using noise scaled MAS for VITS2")
use_noise_scaled_mas = True use_noise_scaled_mas = True
mas_noise_scale_initial = 0.01 mas_noise_scale_initial = 0.01
@@ -84,7 +102,10 @@ def run():
use_noise_scaled_mas = False use_noise_scaled_mas = False
mas_noise_scale_initial = 0.0 mas_noise_scale_initial = 0.0
noise_scale_delta = 0.0 noise_scale_delta = 0.0
if "use_duration_discriminator" in hps.model.keys() and hps.model.use_duration_discriminator == True: if (
"use_duration_discriminator" in hps.model.keys()
and hps.model.use_duration_discriminator == True
):
print("Using duration discriminator for VITS2") print("Using duration discriminator for VITS2")
use_duration_discriminator = True use_duration_discriminator = True
net_dur_disc = DurationDiscriminator( net_dur_disc = DurationDiscriminator(
@@ -94,9 +115,14 @@ def run():
0.1, 0.1,
gin_channels=hps.model.gin_channels if hps.data.n_speakers != 0 else 0, gin_channels=hps.model.gin_channels if hps.data.n_speakers != 0 else 0,
).cuda(rank) ).cuda(rank)
if "use_spk_conditioned_encoder" in hps.model.keys() and hps.model.use_spk_conditioned_encoder == True: if (
"use_spk_conditioned_encoder" in hps.model.keys()
and hps.model.use_spk_conditioned_encoder == True
):
if hps.data.n_speakers == 0: if hps.data.n_speakers == 0:
raise ValueError("n_speakers must be > 0 when using spk conditioned encoder to train multi-speaker model") raise ValueError(
"n_speakers must be > 0 when using spk conditioned encoder to train multi-speaker model"
)
use_spk_conditioned_encoder = True use_spk_conditioned_encoder = True
else: else:
print("Using normal encoder for VITS1") print("Using normal encoder for VITS1")
@@ -109,25 +135,29 @@ def run():
n_speakers=hps.data.n_speakers, n_speakers=hps.data.n_speakers,
mas_noise_scale_initial=mas_noise_scale_initial, mas_noise_scale_initial=mas_noise_scale_initial,
noise_scale_delta=noise_scale_delta, noise_scale_delta=noise_scale_delta,
**hps.model).cuda(rank) **hps.model,
).cuda(rank)
net_d = MultiPeriodDiscriminator(hps.model.use_spectral_norm).cuda(rank) net_d = MultiPeriodDiscriminator(hps.model.use_spectral_norm).cuda(rank)
optim_g = torch.optim.AdamW( optim_g = torch.optim.AdamW(
filter(lambda p: p.requires_grad, net_g.parameters()), filter(lambda p: p.requires_grad, net_g.parameters()),
hps.train.learning_rate, hps.train.learning_rate,
betas=hps.train.betas, betas=hps.train.betas,
eps=hps.train.eps) eps=hps.train.eps,
)
optim_d = torch.optim.AdamW( optim_d = torch.optim.AdamW(
net_d.parameters(), net_d.parameters(),
hps.train.learning_rate, hps.train.learning_rate,
betas=hps.train.betas, betas=hps.train.betas,
eps=hps.train.eps) eps=hps.train.eps,
)
if net_dur_disc is not None: if net_dur_disc is not None:
optim_dur_disc = torch.optim.AdamW( optim_dur_disc = torch.optim.AdamW(
net_dur_disc.parameters(), net_dur_disc.parameters(),
hps.train.learning_rate, hps.train.learning_rate,
betas=hps.train.betas, betas=hps.train.betas,
eps=hps.train.eps) eps=hps.train.eps,
)
else: else:
optim_dur_disc = None optim_dur_disc = None
net_g = DDP(net_g, device_ids=[rank], find_unused_parameters=True) net_g = DDP(net_g, device_ids=[rank], find_unused_parameters=True)
@@ -136,11 +166,24 @@ def run():
net_dur_disc = DDP(net_dur_disc, device_ids=[rank], find_unused_parameters=True) net_dur_disc = DDP(net_dur_disc, device_ids=[rank], find_unused_parameters=True)
try: try:
if net_dur_disc is not None: if net_dur_disc is not None:
_, _, _, epoch_str = utils.load_checkpoint(utils.latest_checkpoint_path(hps.model_dir, "DUR_*.pth"), net_dur_disc, optim_dur_disc, skip_optimizer=True) _, _, _, epoch_str = utils.load_checkpoint(
_, optim_g, _, epoch_str = utils.load_checkpoint(utils.latest_checkpoint_path(hps.model_dir, "G_*.pth"), net_g, utils.latest_checkpoint_path(hps.model_dir, "DUR_*.pth"),
optim_g, skip_optimizer=True) net_dur_disc,
_, optim_d, _, epoch_str = utils.load_checkpoint(utils.latest_checkpoint_path(hps.model_dir, "D_*.pth"), net_d, optim_dur_disc,
optim_d, skip_optimizer=True) skip_optimizer=True,
)
_, optim_g, _, epoch_str = utils.load_checkpoint(
utils.latest_checkpoint_path(hps.model_dir, "G_*.pth"),
net_g,
optim_g,
skip_optimizer=True,
)
_, optim_d, _, epoch_str = utils.load_checkpoint(
utils.latest_checkpoint_path(hps.model_dir, "D_*.pth"),
net_d,
optim_d,
skip_optimizer=True,
)
epoch_str = max(epoch_str, 1) epoch_str = max(epoch_str, 1)
global_step = (epoch_str - 1) * len(train_loader) global_step = (epoch_str - 1) * len(train_loader)
@@ -149,27 +192,56 @@ def run():
epoch_str = 1 epoch_str = 1
global_step = 0 global_step = 0
scheduler_g = torch.optim.lr_scheduler.ExponentialLR(
scheduler_g = torch.optim.lr_scheduler.ExponentialLR(optim_g, gamma=hps.train.lr_decay, last_epoch=epoch_str - 2) optim_g, gamma=hps.train.lr_decay, last_epoch=epoch_str - 2
scheduler_d = torch.optim.lr_scheduler.ExponentialLR(optim_d, gamma=hps.train.lr_decay, last_epoch=epoch_str - 2) )
scheduler_d = torch.optim.lr_scheduler.ExponentialLR(
optim_d, gamma=hps.train.lr_decay, last_epoch=epoch_str - 2
)
if net_dur_disc is not None: if net_dur_disc is not None:
scheduler_dur_disc = torch.optim.lr_scheduler.ExponentialLR(optim_dur_disc, gamma=hps.train.lr_decay, last_epoch=epoch_str-2) scheduler_dur_disc = torch.optim.lr_scheduler.ExponentialLR(
optim_dur_disc, gamma=hps.train.lr_decay, last_epoch=epoch_str - 2
)
else: else:
scheduler_dur_disc = None scheduler_dur_disc = None
scaler = GradScaler(enabled=hps.train.fp16_run) scaler = GradScaler(enabled=hps.train.fp16_run)
for epoch in range(epoch_str, hps.train.epochs + 1): for epoch in range(epoch_str, hps.train.epochs + 1):
if rank == 0: if rank == 0:
train_and_evaluate(rank, epoch, hps, [net_g, net_d, net_dur_disc], [optim_g, optim_d, optim_dur_disc], [scheduler_g, scheduler_d, scheduler_dur_disc], scaler, [train_loader, eval_loader], logger, [writer, writer_eval]) train_and_evaluate(
rank,
epoch,
hps,
[net_g, net_d, net_dur_disc],
[optim_g, optim_d, optim_dur_disc],
[scheduler_g, scheduler_d, scheduler_dur_disc],
scaler,
[train_loader, eval_loader],
logger,
[writer, writer_eval],
)
else: else:
train_and_evaluate(rank, epoch, hps, [net_g, net_d, net_dur_disc], [optim_g, optim_d, optim_dur_disc], [scheduler_g, scheduler_d, scheduler_dur_disc], scaler, [train_loader, None], None, None) train_and_evaluate(
rank,
epoch,
hps,
[net_g, net_d, net_dur_disc],
[optim_g, optim_d, optim_dur_disc],
[scheduler_g, scheduler_d, scheduler_dur_disc],
scaler,
[train_loader, None],
None,
None,
)
scheduler_g.step() scheduler_g.step()
scheduler_d.step() scheduler_d.step()
if net_dur_disc is not None: if net_dur_disc is not None:
scheduler_dur_disc.step() scheduler_dur_disc.step()
def train_and_evaluate(rank, epoch, hps, nets, optims, schedulers, scaler, loaders, logger, writers): def train_and_evaluate(
rank, epoch, hps, nets, optims, schedulers, scaler, loaders, logger, writers
):
net_g, net_d, net_dur_disc = nets net_g, net_d, net_dur_disc = nets
optim_g, optim_d, optim_dur_disc = optims optim_g, optim_d, optim_dur_disc = optims
scheduler_g, scheduler_d, scheduler_dur_disc = schedulers scheduler_g, scheduler_d, scheduler_dur_disc = schedulers
@@ -184,13 +256,34 @@ def train_and_evaluate(rank, epoch, hps, nets, optims, schedulers, scaler, loade
net_d.train() net_d.train()
if net_dur_disc is not None: if net_dur_disc is not None:
net_dur_disc.train() net_dur_disc.train()
for batch_idx, (x, x_lengths, spec, spec_lengths, y, y_lengths, speakers, tone, language, bert, ja_bert) in tqdm(enumerate(train_loader)): for batch_idx, (
x,
x_lengths,
spec,
spec_lengths,
y,
y_lengths,
speakers,
tone,
language,
bert,
ja_bert,
) in tqdm(enumerate(train_loader)):
if net_g.module.use_noise_scaled_mas: if net_g.module.use_noise_scaled_mas:
current_mas_noise_scale = net_g.module.mas_noise_scale_initial - net_g.module.noise_scale_delta * global_step current_mas_noise_scale = (
net_g.module.mas_noise_scale_initial
- net_g.module.noise_scale_delta * global_step
)
net_g.module.current_mas_noise_scale = max(current_mas_noise_scale, 0.0) net_g.module.current_mas_noise_scale = max(current_mas_noise_scale, 0.0)
x, x_lengths = x.cuda(rank, non_blocking=True), x_lengths.cuda(rank, non_blocking=True) x, x_lengths = x.cuda(rank, non_blocking=True), x_lengths.cuda(
spec, spec_lengths = spec.cuda(rank, non_blocking=True), spec_lengths.cuda(rank, non_blocking=True) rank, non_blocking=True
y, y_lengths = y.cuda(rank, non_blocking=True), y_lengths.cuda(rank, non_blocking=True) )
spec, spec_lengths = spec.cuda(rank, non_blocking=True), spec_lengths.cuda(
rank, non_blocking=True
)
y, y_lengths = y.cuda(rank, non_blocking=True), y_lengths.cuda(
rank, non_blocking=True
)
speakers = speakers.cuda(rank, non_blocking=True) speakers = speakers.cuda(rank, non_blocking=True)
tone = tone.cuda(rank, non_blocking=True) tone = tone.cuda(rank, non_blocking=True)
language = language.cuda(rank, non_blocking=True) language = language.cuda(rank, non_blocking=True)
@@ -198,16 +291,37 @@ def train_and_evaluate(rank, epoch, hps, nets, optims, schedulers, scaler, loade
ja_bert = ja_bert.cuda(rank, non_blocking=True) ja_bert = ja_bert.cuda(rank, non_blocking=True)
with autocast(enabled=hps.train.fp16_run): with autocast(enabled=hps.train.fp16_run):
y_hat, l_length, attn, ids_slice, x_mask, z_mask, \ (
(z, z_p, m_p, logs_p, m_q, logs_q), (hidden_x, logw, logw_) = net_g(x, x_lengths, spec, spec_lengths, speakers, tone, language, bert, ja_bert) y_hat,
l_length,
attn,
ids_slice,
x_mask,
z_mask,
(z, z_p, m_p, logs_p, m_q, logs_q),
(hidden_x, logw, logw_),
) = net_g(
x,
x_lengths,
spec,
spec_lengths,
speakers,
tone,
language,
bert,
ja_bert,
)
mel = spec_to_mel_torch( mel = spec_to_mel_torch(
spec, spec,
hps.data.filter_length, hps.data.filter_length,
hps.data.n_mel_channels, hps.data.n_mel_channels,
hps.data.sampling_rate, hps.data.sampling_rate,
hps.data.mel_fmin, hps.data.mel_fmin,
hps.data.mel_fmax) hps.data.mel_fmax,
y_mel = commons.slice_segments(mel, ids_slice, hps.train.segment_size // hps.data.hop_length) )
y_mel = commons.slice_segments(
mel, ids_slice, hps.train.segment_size // hps.data.hop_length
)
y_hat_mel = mel_spectrogram_torch( y_hat_mel = mel_spectrogram_torch(
y_hat.squeeze(1), y_hat.squeeze(1),
hps.data.filter_length, hps.data.filter_length,
@@ -216,26 +330,38 @@ def train_and_evaluate(rank, epoch, hps, nets, optims, schedulers, scaler, loade
hps.data.hop_length, hps.data.hop_length,
hps.data.win_length, hps.data.win_length,
hps.data.mel_fmin, hps.data.mel_fmin,
hps.data.mel_fmax hps.data.mel_fmax,
) )
y = commons.slice_segments(y, ids_slice * hps.data.hop_length, hps.train.segment_size) # slice y = commons.slice_segments(
y, ids_slice * hps.data.hop_length, hps.train.segment_size
) # slice
# Discriminator # Discriminator
y_d_hat_r, y_d_hat_g, _, _ = net_d(y, y_hat.detach()) y_d_hat_r, y_d_hat_g, _, _ = net_d(y, y_hat.detach())
with autocast(enabled=False): with autocast(enabled=False):
loss_disc, losses_disc_r, losses_disc_g = discriminator_loss(y_d_hat_r, y_d_hat_g) loss_disc, losses_disc_r, losses_disc_g = discriminator_loss(
y_d_hat_r, y_d_hat_g
)
loss_disc_all = loss_disc loss_disc_all = loss_disc
if net_dur_disc is not None: if net_dur_disc is not None:
y_dur_hat_r, y_dur_hat_g = net_dur_disc(hidden_x.detach(), x_mask.detach(), logw.detach(), logw_.detach()) y_dur_hat_r, y_dur_hat_g = net_dur_disc(
hidden_x.detach(), x_mask.detach(), logw.detach(), logw_.detach()
)
with autocast(enabled=False): with autocast(enabled=False):
# TODO: I think need to mean using the mask, but for now, just mean all # TODO: I think need to mean using the mask, but for now, just mean all
loss_dur_disc, losses_dur_disc_r, losses_dur_disc_g = discriminator_loss(y_dur_hat_r, y_dur_hat_g) (
loss_dur_disc,
losses_dur_disc_r,
losses_dur_disc_g,
) = discriminator_loss(y_dur_hat_r, y_dur_hat_g)
loss_dur_disc_all = loss_dur_disc loss_dur_disc_all = loss_dur_disc
optim_dur_disc.zero_grad() optim_dur_disc.zero_grad()
scaler.scale(loss_dur_disc_all).backward() scaler.scale(loss_dur_disc_all).backward()
scaler.unscale_(optim_dur_disc) scaler.unscale_(optim_dur_disc)
grad_norm_dur_disc = commons.clip_grad_value_(net_dur_disc.parameters(), None) grad_norm_dur_disc = commons.clip_grad_value_(
net_dur_disc.parameters(), None
)
scaler.step(optim_dur_disc) scaler.step(optim_dur_disc)
optim_d.zero_grad() optim_d.zero_grad()
@@ -269,51 +395,97 @@ def train_and_evaluate(rank, epoch, hps, nets, optims, schedulers, scaler, loade
if rank == 0: if rank == 0:
if global_step % hps.train.log_interval == 0: if global_step % hps.train.log_interval == 0:
lr = optim_g.param_groups[0]['lr'] lr = optim_g.param_groups[0]["lr"]
losses = [loss_disc, loss_gen, loss_fm, loss_mel, loss_dur, loss_kl] losses = [loss_disc, loss_gen, loss_fm, loss_mel, loss_dur, loss_kl]
logger.info('Train Epoch: {} [{:.0f}%]'.format( logger.info(
epoch, "Train Epoch: {} [{:.0f}%]".format(
100. * batch_idx / len(train_loader))) epoch, 100.0 * batch_idx / len(train_loader)
)
)
logger.info([x.item() for x in losses] + [global_step, lr]) logger.info([x.item() for x in losses] + [global_step, lr])
scalar_dict = {"loss/g/total": loss_gen_all, "loss/d/total": loss_disc_all, "learning_rate": lr, scalar_dict = {
"grad_norm_d": grad_norm_d, "grad_norm_g": grad_norm_g} "loss/g/total": loss_gen_all,
"loss/d/total": loss_disc_all,
"learning_rate": lr,
"grad_norm_d": grad_norm_d,
"grad_norm_g": grad_norm_g,
}
scalar_dict.update( scalar_dict.update(
{"loss/g/fm": loss_fm, "loss/g/mel": loss_mel, "loss/g/dur": loss_dur, "loss/g/kl": loss_kl}) {
scalar_dict.update({"loss/g/{}".format(i): v for i, v in enumerate(losses_gen)}) "loss/g/fm": loss_fm,
scalar_dict.update({"loss/d_r/{}".format(i): v for i, v in enumerate(losses_disc_r)}) "loss/g/mel": loss_mel,
scalar_dict.update({"loss/d_g/{}".format(i): v for i, v in enumerate(losses_disc_g)}) "loss/g/dur": loss_dur,
"loss/g/kl": loss_kl,
}
)
scalar_dict.update(
{"loss/g/{}".format(i): v for i, v in enumerate(losses_gen)}
)
scalar_dict.update(
{"loss/d_r/{}".format(i): v for i, v in enumerate(losses_disc_r)}
)
scalar_dict.update(
{"loss/d_g/{}".format(i): v for i, v in enumerate(losses_disc_g)}
)
image_dict = { image_dict = {
"slice/mel_org": utils.plot_spectrogram_to_numpy(y_mel[0].data.cpu().numpy()), "slice/mel_org": utils.plot_spectrogram_to_numpy(
"slice/mel_gen": utils.plot_spectrogram_to_numpy(y_hat_mel[0].data.cpu().numpy()), y_mel[0].data.cpu().numpy()
"all/mel": utils.plot_spectrogram_to_numpy(mel[0].data.cpu().numpy()), ),
"all/attn": utils.plot_alignment_to_numpy(attn[0, 0].data.cpu().numpy()) "slice/mel_gen": utils.plot_spectrogram_to_numpy(
y_hat_mel[0].data.cpu().numpy()
),
"all/mel": utils.plot_spectrogram_to_numpy(
mel[0].data.cpu().numpy()
),
"all/attn": utils.plot_alignment_to_numpy(
attn[0, 0].data.cpu().numpy()
),
} }
utils.summarize( utils.summarize(
writer=writer, writer=writer,
global_step=global_step, global_step=global_step,
images=image_dict, images=image_dict,
scalars=scalar_dict) scalars=scalar_dict,
)
if global_step % hps.train.eval_interval == 0: if global_step % hps.train.eval_interval == 0:
evaluate(hps, net_g, eval_loader, writer_eval) evaluate(hps, net_g, eval_loader, writer_eval)
utils.save_checkpoint(net_g, optim_g, hps.train.learning_rate, epoch, utils.save_checkpoint(
os.path.join(hps.model_dir, "G_{}.pth".format(global_step))) net_g,
utils.save_checkpoint(net_d, optim_d, hps.train.learning_rate, epoch, optim_g,
os.path.join(hps.model_dir, "D_{}.pth".format(global_step))) hps.train.learning_rate,
epoch,
os.path.join(hps.model_dir, "G_{}.pth".format(global_step)),
)
utils.save_checkpoint(
net_d,
optim_d,
hps.train.learning_rate,
epoch,
os.path.join(hps.model_dir, "D_{}.pth".format(global_step)),
)
if net_dur_disc is not None: if net_dur_disc is not None:
utils.save_checkpoint(net_dur_disc, optim_dur_disc, hps.train.learning_rate, epoch, os.path.join(hps.model_dir, "DUR_{}.pth".format(global_step))) utils.save_checkpoint(
keep_ckpts = getattr(hps.train, 'keep_ckpts', 5) net_dur_disc,
optim_dur_disc,
hps.train.learning_rate,
epoch,
os.path.join(hps.model_dir, "DUR_{}.pth".format(global_step)),
)
keep_ckpts = getattr(hps.train, "keep_ckpts", 5)
if keep_ckpts > 0: if keep_ckpts > 0:
utils.clean_checkpoints(path_to_models=hps.model_dir, n_ckpts_to_keep=keep_ckpts, sort_by_time=True) utils.clean_checkpoints(
path_to_models=hps.model_dir,
n_ckpts_to_keep=keep_ckpts,
sort_by_time=True,
)
global_step += 1 global_step += 1
if rank == 0: if rank == 0:
logger.info('====> Epoch: {}'.format(epoch)) logger.info("====> Epoch: {}".format(epoch))
def evaluate(hps, generator, eval_loader, writer_eval): def evaluate(hps, generator, eval_loader, writer_eval):
@@ -322,7 +494,19 @@ def evaluate(hps, generator, eval_loader, writer_eval):
audio_dict = {} audio_dict = {}
print("Evaluating ...") print("Evaluating ...")
with torch.no_grad(): with torch.no_grad():
for batch_idx, (x, x_lengths, spec, spec_lengths, y, y_lengths, speakers, tone, language, bert, ja_bert) in enumerate(eval_loader): for batch_idx, (
x,
x_lengths,
spec,
spec_lengths,
y,
y_lengths,
speakers,
tone,
language,
bert,
ja_bert,
) in enumerate(eval_loader):
x, x_lengths = x.cuda(), x_lengths.cuda() x, x_lengths = x.cuda(), x_lengths.cuda()
spec, spec_lengths = spec.cuda(), spec_lengths.cuda() spec, spec_lengths = spec.cuda(), spec_lengths.cuda()
y, y_lengths = y.cuda(), y_lengths.cuda() y, y_lengths = y.cuda(), y_lengths.cuda()
@@ -332,7 +516,18 @@ def evaluate(hps, generator, eval_loader, writer_eval):
tone = tone.cuda() tone = tone.cuda()
language = language.cuda() language = language.cuda()
for use_sdp in [True, False]: for use_sdp in [True, False]:
y_hat, attn, mask, *_ = generator.module.infer(x, x_lengths, speakers, tone, language, bert, ja_bert, y=spec, max_len=1000, sdp_ratio=0.0 if not use_sdp else 1.0) y_hat, attn, mask, *_ = generator.module.infer(
x,
x_lengths,
speakers,
tone,
language,
bert,
ja_bert,
y=spec,
max_len=1000,
sdp_ratio=0.0 if not use_sdp else 1.0,
)
y_hat_lengths = mask.sum([1, 2]).long() * hps.data.hop_length y_hat_lengths = mask.sum([1, 2]).long() * hps.data.hop_length
mel = spec_to_mel_torch( mel = spec_to_mel_torch(
@@ -341,7 +536,8 @@ def evaluate(hps, generator, eval_loader, writer_eval):
hps.data.n_mel_channels, hps.data.n_mel_channels,
hps.data.sampling_rate, hps.data.sampling_rate,
hps.data.mel_fmin, hps.data.mel_fmin,
hps.data.mel_fmax) hps.data.mel_fmax,
)
y_hat_mel = mel_spectrogram_torch( y_hat_mel = mel_spectrogram_torch(
y_hat.squeeze(1).float(), y_hat.squeeze(1).float(),
hps.data.filter_length, hps.data.filter_length,
@@ -350,15 +546,29 @@ def evaluate(hps, generator, eval_loader, writer_eval):
hps.data.hop_length, hps.data.hop_length,
hps.data.win_length, hps.data.win_length,
hps.data.mel_fmin, hps.data.mel_fmin,
hps.data.mel_fmax hps.data.mel_fmax,
)
image_dict.update(
{
f"gen/mel_{batch_idx}": utils.plot_spectrogram_to_numpy(
y_hat_mel[0].cpu().numpy()
)
}
)
audio_dict.update(
{
f"gen/audio_{batch_idx}_{use_sdp}": y_hat[
0, :, : y_hat_lengths[0]
]
}
)
image_dict.update(
{
f"gt/mel_{batch_idx}": utils.plot_spectrogram_to_numpy(
mel[0].cpu().numpy()
)
}
) )
image_dict.update({
f"gen/mel_{batch_idx}": utils.plot_spectrogram_to_numpy(y_hat_mel[0].cpu().numpy())
})
audio_dict.update({
f"gen/audio_{batch_idx}_{use_sdp}": y_hat[0, :, :y_hat_lengths[0]]
})
image_dict.update({f"gt/mel_{batch_idx}": utils.plot_spectrogram_to_numpy(mel[0].cpu().numpy())})
audio_dict.update({f"gt/audio_{batch_idx}": y[0, :, : y_lengths[0]]}) audio_dict.update({f"gt/audio_{batch_idx}": y[0, :, : y_lengths[0]]})
utils.summarize( utils.summarize(
@@ -366,9 +576,10 @@ def evaluate(hps, generator, eval_loader, writer_eval):
global_step=global_step, global_step=global_step,
images=image_dict, images=image_dict,
audios=audio_dict, audios=audio_dict,
audio_sampling_rate=hps.data.sampling_rate audio_sampling_rate=hps.data.sampling_rate,
) )
generator.train() generator.train()
if __name__ == "__main__": if __name__ == "__main__":
run() run()

110
transforms.py
View File

@@ -9,26 +9,24 @@ DEFAULT_MIN_BIN_HEIGHT = 1e-3
DEFAULT_MIN_DERIVATIVE = 1e-3 DEFAULT_MIN_DERIVATIVE = 1e-3
def piecewise_rational_quadratic_transform(inputs, def piecewise_rational_quadratic_transform(
inputs,
unnormalized_widths, unnormalized_widths,
unnormalized_heights, unnormalized_heights,
unnormalized_derivatives, unnormalized_derivatives,
inverse=False, inverse=False,
tails=None, tails=None,
tail_bound=1., tail_bound=1.0,
min_bin_width=DEFAULT_MIN_BIN_WIDTH, min_bin_width=DEFAULT_MIN_BIN_WIDTH,
min_bin_height=DEFAULT_MIN_BIN_HEIGHT, min_bin_height=DEFAULT_MIN_BIN_HEIGHT,
min_derivative=DEFAULT_MIN_DERIVATIVE): min_derivative=DEFAULT_MIN_DERIVATIVE,
):
if tails is None: if tails is None:
spline_fn = rational_quadratic_spline spline_fn = rational_quadratic_spline
spline_kwargs = {} spline_kwargs = {}
else: else:
spline_fn = unconstrained_rational_quadratic_spline spline_fn = unconstrained_rational_quadratic_spline
spline_kwargs = { spline_kwargs = {"tails": tails, "tail_bound": tail_bound}
'tails': tails,
'tail_bound': tail_bound
}
outputs, logabsdet = spline_fn( outputs, logabsdet = spline_fn(
inputs=inputs, inputs=inputs,
@@ -46,29 +44,28 @@ def piecewise_rational_quadratic_transform(inputs,
def searchsorted(bin_locations, inputs, eps=1e-6): def searchsorted(bin_locations, inputs, eps=1e-6):
bin_locations[..., -1] += eps bin_locations[..., -1] += eps
return torch.sum( return torch.sum(inputs[..., None] >= bin_locations, dim=-1) - 1
inputs[..., None] >= bin_locations,
dim=-1
) - 1
def unconstrained_rational_quadratic_spline(inputs, def unconstrained_rational_quadratic_spline(
inputs,
unnormalized_widths, unnormalized_widths,
unnormalized_heights, unnormalized_heights,
unnormalized_derivatives, unnormalized_derivatives,
inverse=False, inverse=False,
tails='linear', tails="linear",
tail_bound=1., tail_bound=1.0,
min_bin_width=DEFAULT_MIN_BIN_WIDTH, min_bin_width=DEFAULT_MIN_BIN_WIDTH,
min_bin_height=DEFAULT_MIN_BIN_HEIGHT, min_bin_height=DEFAULT_MIN_BIN_HEIGHT,
min_derivative=DEFAULT_MIN_DERIVATIVE): min_derivative=DEFAULT_MIN_DERIVATIVE,
):
inside_interval_mask = (inputs >= -tail_bound) & (inputs <= tail_bound) inside_interval_mask = (inputs >= -tail_bound) & (inputs <= tail_bound)
outside_interval_mask = ~inside_interval_mask outside_interval_mask = ~inside_interval_mask
outputs = torch.zeros_like(inputs) outputs = torch.zeros_like(inputs)
logabsdet = torch.zeros_like(inputs) logabsdet = torch.zeros_like(inputs)
if tails == 'linear': if tails == "linear":
unnormalized_derivatives = F.pad(unnormalized_derivatives, pad=(1, 1)) unnormalized_derivatives = F.pad(unnormalized_derivatives, pad=(1, 1))
constant = np.log(np.exp(1 - min_derivative) - 1) constant = np.log(np.exp(1 - min_derivative) - 1)
unnormalized_derivatives[..., 0] = constant unnormalized_derivatives[..., 0] = constant
@@ -77,45 +74,57 @@ def unconstrained_rational_quadratic_spline(inputs,
outputs[outside_interval_mask] = inputs[outside_interval_mask] outputs[outside_interval_mask] = inputs[outside_interval_mask]
logabsdet[outside_interval_mask] = 0 logabsdet[outside_interval_mask] = 0
else: else:
raise RuntimeError('{} tails are not implemented.'.format(tails)) raise RuntimeError("{} tails are not implemented.".format(tails))
outputs[inside_interval_mask], logabsdet[inside_interval_mask] = rational_quadratic_spline( (
outputs[inside_interval_mask],
logabsdet[inside_interval_mask],
) = rational_quadratic_spline(
inputs=inputs[inside_interval_mask], inputs=inputs[inside_interval_mask],
unnormalized_widths=unnormalized_widths[inside_interval_mask, :], unnormalized_widths=unnormalized_widths[inside_interval_mask, :],
unnormalized_heights=unnormalized_heights[inside_interval_mask, :], unnormalized_heights=unnormalized_heights[inside_interval_mask, :],
unnormalized_derivatives=unnormalized_derivatives[inside_interval_mask, :], unnormalized_derivatives=unnormalized_derivatives[inside_interval_mask, :],
inverse=inverse, inverse=inverse,
left=-tail_bound, right=tail_bound, bottom=-tail_bound, top=tail_bound, left=-tail_bound,
right=tail_bound,
bottom=-tail_bound,
top=tail_bound,
min_bin_width=min_bin_width, min_bin_width=min_bin_width,
min_bin_height=min_bin_height, min_bin_height=min_bin_height,
min_derivative=min_derivative min_derivative=min_derivative,
) )
return outputs, logabsdet return outputs, logabsdet
def rational_quadratic_spline(inputs,
def rational_quadratic_spline(
inputs,
unnormalized_widths, unnormalized_widths,
unnormalized_heights, unnormalized_heights,
unnormalized_derivatives, unnormalized_derivatives,
inverse=False, inverse=False,
left=0., right=1., bottom=0., top=1., left=0.0,
right=1.0,
bottom=0.0,
top=1.0,
min_bin_width=DEFAULT_MIN_BIN_WIDTH, min_bin_width=DEFAULT_MIN_BIN_WIDTH,
min_bin_height=DEFAULT_MIN_BIN_HEIGHT, min_bin_height=DEFAULT_MIN_BIN_HEIGHT,
min_derivative=DEFAULT_MIN_DERIVATIVE): min_derivative=DEFAULT_MIN_DERIVATIVE,
):
if torch.min(inputs) < left or torch.max(inputs) > right: if torch.min(inputs) < left or torch.max(inputs) > right:
raise ValueError('Input to a transform is not within its domain') raise ValueError("Input to a transform is not within its domain")
num_bins = unnormalized_widths.shape[-1] num_bins = unnormalized_widths.shape[-1]
if min_bin_width * num_bins > 1.0: if min_bin_width * num_bins > 1.0:
raise ValueError('Minimal bin width too large for the number of bins') raise ValueError("Minimal bin width too large for the number of bins")
if min_bin_height * num_bins > 1.0: if min_bin_height * num_bins > 1.0:
raise ValueError('Minimal bin height too large for the number of bins') raise ValueError("Minimal bin height too large for the number of bins")
widths = F.softmax(unnormalized_widths, dim=-1) widths = F.softmax(unnormalized_widths, dim=-1)
widths = min_bin_width + (1 - min_bin_width * num_bins) * widths widths = min_bin_width + (1 - min_bin_width * num_bins) * widths
cumwidths = torch.cumsum(widths, dim=-1) cumwidths = torch.cumsum(widths, dim=-1)
cumwidths = F.pad(cumwidths, pad=(1, 0), mode='constant', value=0.0) cumwidths = F.pad(cumwidths, pad=(1, 0), mode="constant", value=0.0)
cumwidths = (right - left) * cumwidths + left cumwidths = (right - left) * cumwidths + left
cumwidths[..., 0] = left cumwidths[..., 0] = left
cumwidths[..., -1] = right cumwidths[..., -1] = right
@@ -126,7 +135,7 @@ def rational_quadratic_spline(inputs,
heights = F.softmax(unnormalized_heights, dim=-1) heights = F.softmax(unnormalized_heights, dim=-1)
heights = min_bin_height + (1 - min_bin_height * num_bins) * heights heights = min_bin_height + (1 - min_bin_height * num_bins) * heights
cumheights = torch.cumsum(heights, dim=-1) cumheights = torch.cumsum(heights, dim=-1)
cumheights = F.pad(cumheights, pad=(1, 0), mode='constant', value=0.0) cumheights = F.pad(cumheights, pad=(1, 0), mode="constant", value=0.0)
cumheights = (top - bottom) * cumheights + bottom cumheights = (top - bottom) * cumheights + bottom
cumheights[..., 0] = bottom cumheights[..., 0] = bottom
cumheights[..., -1] = top cumheights[..., -1] = top
@@ -150,14 +159,12 @@ def rational_quadratic_spline(inputs,
input_heights = heights.gather(-1, bin_idx)[..., 0] input_heights = heights.gather(-1, bin_idx)[..., 0]
if inverse: if inverse:
a = (((inputs - input_cumheights) * (input_derivatives a = (inputs - input_cumheights) * (
+ input_derivatives_plus_one input_derivatives + input_derivatives_plus_one - 2 * input_delta
- 2 * input_delta) ) + input_heights * (input_delta - input_derivatives)
+ input_heights * (input_delta - input_derivatives))) b = input_heights * input_derivatives - (inputs - input_cumheights) * (
b = (input_heights * input_derivatives input_derivatives + input_derivatives_plus_one - 2 * input_delta
- (inputs - input_cumheights) * (input_derivatives )
+ input_derivatives_plus_one
- 2 * input_delta))
c = -input_delta * (inputs - input_cumheights) c = -input_delta * (inputs - input_cumheights)
discriminant = b.pow(2) - 4 * a * c discriminant = b.pow(2) - 4 * a * c
@@ -167,11 +174,15 @@ def rational_quadratic_spline(inputs,
outputs = root * input_bin_widths + input_cumwidths outputs = root * input_bin_widths + input_cumwidths
theta_one_minus_theta = root * (1 - root) theta_one_minus_theta = root * (1 - root)
denominator = input_delta + ((input_derivatives + input_derivatives_plus_one - 2 * input_delta) denominator = input_delta + (
* theta_one_minus_theta) (input_derivatives + input_derivatives_plus_one - 2 * input_delta)
derivative_numerator = input_delta.pow(2) * (input_derivatives_plus_one * root.pow(2) * theta_one_minus_theta
)
derivative_numerator = input_delta.pow(2) * (
input_derivatives_plus_one * root.pow(2)
+ 2 * input_delta * theta_one_minus_theta + 2 * input_delta * theta_one_minus_theta
+ input_derivatives * (1 - root).pow(2)) + input_derivatives * (1 - root).pow(2)
)
logabsdet = torch.log(derivative_numerator) - 2 * torch.log(denominator) logabsdet = torch.log(derivative_numerator) - 2 * torch.log(denominator)
return outputs, -logabsdet return outputs, -logabsdet
@@ -179,15 +190,20 @@ def rational_quadratic_spline(inputs,
theta = (inputs - input_cumwidths) / input_bin_widths theta = (inputs - input_cumwidths) / input_bin_widths
theta_one_minus_theta = theta * (1 - theta) theta_one_minus_theta = theta * (1 - theta)
numerator = input_heights * (input_delta * theta.pow(2) numerator = input_heights * (
+ input_derivatives * theta_one_minus_theta) input_delta * theta.pow(2) + input_derivatives * theta_one_minus_theta
denominator = input_delta + ((input_derivatives + input_derivatives_plus_one - 2 * input_delta) )
* theta_one_minus_theta) denominator = input_delta + (
(input_derivatives + input_derivatives_plus_one - 2 * input_delta)
* theta_one_minus_theta
)
outputs = input_cumheights + numerator / denominator outputs = input_cumheights + numerator / denominator
derivative_numerator = input_delta.pow(2) * (input_derivatives_plus_one * theta.pow(2) derivative_numerator = input_delta.pow(2) * (
input_derivatives_plus_one * theta.pow(2)
+ 2 * input_delta * theta_one_minus_theta + 2 * input_delta * theta_one_minus_theta
+ input_derivatives * (1 - theta).pow(2)) + input_derivatives * (1 - theta).pow(2)
)
logabsdet = torch.log(derivative_numerator) - 2 * torch.log(denominator) logabsdet = torch.log(derivative_numerator) - 2 * torch.log(denominator)
return outputs, logabsdet return outputs, logabsdet

153
utils.py
View File

@@ -16,20 +16,24 @@ logger = logging.getLogger(__name__)
def load_checkpoint(checkpoint_path, model, optimizer=None, skip_optimizer=False): def load_checkpoint(checkpoint_path, model, optimizer=None, skip_optimizer=False):
assert os.path.isfile(checkpoint_path) assert os.path.isfile(checkpoint_path)
checkpoint_dict = torch.load(checkpoint_path, map_location='cpu') checkpoint_dict = torch.load(checkpoint_path, map_location="cpu")
iteration = checkpoint_dict['iteration'] iteration = checkpoint_dict["iteration"]
learning_rate = checkpoint_dict['learning_rate'] learning_rate = checkpoint_dict["learning_rate"]
if optimizer is not None and not skip_optimizer and checkpoint_dict['optimizer'] is not None: if (
optimizer.load_state_dict(checkpoint_dict['optimizer']) optimizer is not None
and not skip_optimizer
and checkpoint_dict["optimizer"] is not None
):
optimizer.load_state_dict(checkpoint_dict["optimizer"])
elif optimizer is None and not skip_optimizer: elif optimizer is None and not skip_optimizer:
# else: Disable this line if Infer and resume checkpoint,then enable the line upper # else: Disable this line if Infer and resume checkpoint,then enable the line upper
new_opt_dict = optimizer.state_dict() new_opt_dict = optimizer.state_dict()
new_opt_dict_params = new_opt_dict['param_groups'][0]['params'] new_opt_dict_params = new_opt_dict["param_groups"][0]["params"]
new_opt_dict['param_groups'] = checkpoint_dict['optimizer']['param_groups'] new_opt_dict["param_groups"] = checkpoint_dict["optimizer"]["param_groups"]
new_opt_dict['param_groups'][0]['params'] = new_opt_dict_params new_opt_dict["param_groups"][0]["params"] = new_opt_dict_params
optimizer.load_state_dict(new_opt_dict) optimizer.load_state_dict(new_opt_dict)
saved_state_dict = checkpoint_dict['model'] saved_state_dict = checkpoint_dict["model"]
if hasattr(model, 'module'): if hasattr(model, "module"):
state_dict = model.module.state_dict() state_dict = model.module.state_dict()
else: else:
state_dict = model.state_dict() state_dict = model.state_dict()
@@ -38,39 +42,59 @@ def load_checkpoint(checkpoint_path, model, optimizer=None, skip_optimizer=False
try: try:
# assert "emb_g" not in k # assert "emb_g" not in k
new_state_dict[k] = saved_state_dict[k] new_state_dict[k] = saved_state_dict[k]
assert saved_state_dict[k].shape == v.shape, (saved_state_dict[k].shape, v.shape) assert saved_state_dict[k].shape == v.shape, (
saved_state_dict[k].shape,
v.shape,
)
except: except:
logger.error("%s is not in the checkpoint" % k) logger.error("%s is not in the checkpoint" % k)
new_state_dict[k] = v new_state_dict[k] = v
if hasattr(model, 'module'): if hasattr(model, "module"):
model.module.load_state_dict(new_state_dict, strict=False) model.module.load_state_dict(new_state_dict, strict=False)
else: else:
model.load_state_dict(new_state_dict, strict=False) model.load_state_dict(new_state_dict, strict=False)
logger.info("Loaded checkpoint '{}' (iteration {})".format( logger.info(
checkpoint_path, iteration)) "Loaded checkpoint '{}' (iteration {})".format(checkpoint_path, iteration)
)
return model, optimizer, learning_rate, iteration return model, optimizer, learning_rate, iteration
def save_checkpoint(model, optimizer, learning_rate, iteration, checkpoint_path): def save_checkpoint(model, optimizer, learning_rate, iteration, checkpoint_path):
logger.info("Saving model and optimizer state at iteration {} to {}".format( logger.info(
iteration, checkpoint_path)) "Saving model and optimizer state at iteration {} to {}".format(
if hasattr(model, 'module'): iteration, checkpoint_path
)
)
if hasattr(model, "module"):
state_dict = model.module.state_dict() state_dict = model.module.state_dict()
else: else:
state_dict = model.state_dict() state_dict = model.state_dict()
torch.save({'model': state_dict, torch.save(
'iteration': iteration, {
'optimizer': optimizer.state_dict(), "model": state_dict,
'learning_rate': learning_rate}, checkpoint_path) "iteration": iteration,
"optimizer": optimizer.state_dict(),
"learning_rate": learning_rate,
},
checkpoint_path,
)
def summarize(writer, global_step, scalars={}, histograms={}, images={}, audios={}, audio_sampling_rate=22050): def summarize(
writer,
global_step,
scalars={},
histograms={},
images={},
audios={},
audio_sampling_rate=22050,
):
for k, v in scalars.items(): for k, v in scalars.items():
writer.add_scalar(k, v, global_step) writer.add_scalar(k, v, global_step)
for k, v in histograms.items(): for k, v in histograms.items():
writer.add_histogram(k, v, global_step) writer.add_histogram(k, v, global_step)
for k, v in images.items(): for k, v in images.items():
writer.add_image(k, v, global_step, dataformats='HWC') writer.add_image(k, v, global_step, dataformats="HWC")
for k, v in audios.items(): for k, v in audios.items():
writer.add_audio(k, v, global_step, audio_sampling_rate) writer.add_audio(k, v, global_step, audio_sampling_rate)
@@ -86,23 +110,23 @@ def plot_spectrogram_to_numpy(spectrogram):
global MATPLOTLIB_FLAG global MATPLOTLIB_FLAG
if not MATPLOTLIB_FLAG: if not MATPLOTLIB_FLAG:
import matplotlib import matplotlib
matplotlib.use("Agg") matplotlib.use("Agg")
MATPLOTLIB_FLAG = True MATPLOTLIB_FLAG = True
mpl_logger = logging.getLogger('matplotlib') mpl_logger = logging.getLogger("matplotlib")
mpl_logger.setLevel(logging.WARNING) mpl_logger.setLevel(logging.WARNING)
import matplotlib.pylab as plt import matplotlib.pylab as plt
import numpy as np import numpy as np
fig, ax = plt.subplots(figsize=(10, 2)) fig, ax = plt.subplots(figsize=(10, 2))
im = ax.imshow(spectrogram, aspect="auto", origin="lower", im = ax.imshow(spectrogram, aspect="auto", origin="lower", interpolation="none")
interpolation='none')
plt.colorbar(im, ax=ax) plt.colorbar(im, ax=ax)
plt.xlabel("Frames") plt.xlabel("Frames")
plt.ylabel("Channels") plt.ylabel("Channels")
plt.tight_layout() plt.tight_layout()
fig.canvas.draw() fig.canvas.draw()
data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='') data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep="")
data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,)) data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
plt.close() plt.close()
return data return data
@@ -112,26 +136,28 @@ def plot_alignment_to_numpy(alignment, info=None):
global MATPLOTLIB_FLAG global MATPLOTLIB_FLAG
if not MATPLOTLIB_FLAG: if not MATPLOTLIB_FLAG:
import matplotlib import matplotlib
matplotlib.use("Agg") matplotlib.use("Agg")
MATPLOTLIB_FLAG = True MATPLOTLIB_FLAG = True
mpl_logger = logging.getLogger('matplotlib') mpl_logger = logging.getLogger("matplotlib")
mpl_logger.setLevel(logging.WARNING) mpl_logger.setLevel(logging.WARNING)
import matplotlib.pylab as plt import matplotlib.pylab as plt
import numpy as np import numpy as np
fig, ax = plt.subplots(figsize=(6, 4)) fig, ax = plt.subplots(figsize=(6, 4))
im = ax.imshow(alignment.transpose(), aspect='auto', origin='lower', im = ax.imshow(
interpolation='none') alignment.transpose(), aspect="auto", origin="lower", interpolation="none"
)
fig.colorbar(im, ax=ax) fig.colorbar(im, ax=ax)
xlabel = 'Decoder timestep' xlabel = "Decoder timestep"
if info is not None: if info is not None:
xlabel += '\n\n' + info xlabel += "\n\n" + info
plt.xlabel(xlabel) plt.xlabel(xlabel)
plt.ylabel('Encoder timestep') plt.ylabel("Encoder timestep")
plt.tight_layout() plt.tight_layout()
fig.canvas.draw() fig.canvas.draw()
data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='') data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep="")
data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,)) data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
plt.close() plt.close()
return data return data
@@ -143,17 +169,21 @@ def load_wav_to_torch(full_path):
def load_filepaths_and_text(filename, split="|"): def load_filepaths_and_text(filename, split="|"):
with open(filename, encoding='utf-8') as f: with open(filename, encoding="utf-8") as f:
filepaths_and_text = [line.strip().split(split) for line in f] filepaths_and_text = [line.strip().split(split) for line in f]
return filepaths_and_text return filepaths_and_text
def get_hparams(init=True): def get_hparams(init=True):
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('-c', '--config', type=str, default="./configs/base.json", parser.add_argument(
help='JSON file for configuration') "-c",
parser.add_argument('-m', '--model', type=str, required=True, "--config",
help='Model name') type=str,
default="./configs/base.json",
help="JSON file for configuration",
)
parser.add_argument("-m", "--model", type=str, required=True, help="Model name")
args = parser.parse_args() args = parser.parse_args()
model_dir = os.path.join("./logs", args.model) model_dir = os.path.join("./logs", args.model)
@@ -178,7 +208,7 @@ def get_hparams(init=True):
return hparams return hparams
def clean_checkpoints(path_to_models='logs/44k/', n_ckpts_to_keep=2, sort_by_time=True): def clean_checkpoints(path_to_models="logs/44k/", n_ckpts_to_keep=2, sort_by_time=True):
"""Freeing up space by deleting saved ckpts """Freeing up space by deleting saved ckpts
Arguments: Arguments:
@@ -188,21 +218,31 @@ def clean_checkpoints(path_to_models='logs/44k/', n_ckpts_to_keep=2, sort_by_tim
False -> lexicographically delete ckpts False -> lexicographically delete ckpts
""" """
import re import re
ckpts_files = [f for f in os.listdir(path_to_models) if os.path.isfile(os.path.join(path_to_models, f))]
name_key = (lambda _f: int(re.compile('._(\d+)\.pth').match(_f).group(1))) ckpts_files = [
time_key = (lambda _f: os.path.getmtime(os.path.join(path_to_models, _f))) f
for f in os.listdir(path_to_models)
if os.path.isfile(os.path.join(path_to_models, f))
]
name_key = lambda _f: int(re.compile("._(\d+)\.pth").match(_f).group(1))
time_key = lambda _f: os.path.getmtime(os.path.join(path_to_models, _f))
sort_key = time_key if sort_by_time else name_key sort_key = time_key if sort_by_time else name_key
x_sorted = lambda _x: sorted([f for f in ckpts_files if f.startswith(_x) and not f.endswith('_0.pth')], x_sorted = lambda _x: sorted(
key=sort_key) [f for f in ckpts_files if f.startswith(_x) and not f.endswith("_0.pth")],
to_del = [os.path.join(path_to_models, fn) for fn in key=sort_key,
(x_sorted('G')[:-n_ckpts_to_keep] + x_sorted('D')[:-n_ckpts_to_keep])] )
to_del = [
os.path.join(path_to_models, fn)
for fn in (x_sorted("G")[:-n_ckpts_to_keep] + x_sorted("D")[:-n_ckpts_to_keep])
]
del_info = lambda fn: logger.info(f".. Free up space by deleting ckpt {fn}") del_info = lambda fn: logger.info(f".. Free up space by deleting ckpt {fn}")
del_routine = lambda x: [os.remove(x), del_info(x)] del_routine = lambda x: [os.remove(x), del_info(x)]
rs = [del_routine(fn) for fn in to_del] rs = [del_routine(fn) for fn in to_del]
def get_hparams_from_dir(model_dir): def get_hparams_from_dir(model_dir):
config_save_path = os.path.join(model_dir, "config.json") config_save_path = os.path.join(model_dir, "config.json")
with open(config_save_path, "r", encoding='utf-8') as f: with open(config_save_path, "r", encoding="utf-8") as f:
data = f.read() data = f.read()
config = json.loads(data) config = json.loads(data)
@@ -212,7 +252,7 @@ def get_hparams_from_dir(model_dir):
def get_hparams_from_file(config_path): def get_hparams_from_file(config_path):
with open(config_path, "r", encoding='utf-8') as f: with open(config_path, "r", encoding="utf-8") as f:
data = f.read() data = f.read()
config = json.loads(data) config = json.loads(data)
@@ -223,9 +263,11 @@ def get_hparams_from_file(config_path):
def check_git_hash(model_dir): def check_git_hash(model_dir):
source_dir = os.path.dirname(os.path.realpath(__file__)) source_dir = os.path.dirname(os.path.realpath(__file__))
if not os.path.exists(os.path.join(source_dir, ".git")): if not os.path.exists(os.path.join(source_dir, ".git")):
logger.warn("{} is not a git repository, therefore hash value comparison will be ignored.".format( logger.warn(
"{} is not a git repository, therefore hash value comparison will be ignored.".format(
source_dir source_dir
)) )
)
return return
cur_hash = subprocess.getoutput("git rev-parse HEAD") cur_hash = subprocess.getoutput("git rev-parse HEAD")
@@ -234,8 +276,11 @@ def check_git_hash(model_dir):
if os.path.exists(path): if os.path.exists(path):
saved_hash = open(path).read() saved_hash = open(path).read()
if saved_hash != cur_hash: if saved_hash != cur_hash:
logger.warn("git hash values are different. {}(saved) != {}(current)".format( logger.warn(
saved_hash[:8], cur_hash[:8])) "git hash values are different. {}(saved) != {}(current)".format(
saved_hash[:8], cur_hash[:8]
)
)
else: else:
open(path, "w").write(cur_hash) open(path, "w").write(cur_hash)
@@ -255,7 +300,7 @@ def get_logger(model_dir, filename="train.log"):
return logger return logger
class HParams(): class HParams:
def __init__(self, **kwargs): def __init__(self, **kwargs):
for k, v in kwargs.items(): for k, v in kwargs.items():
if type(v) == dict: if type(v) == dict:

112
webui.py
View File

@@ -10,7 +10,9 @@ logging.getLogger("markdown_it").setLevel(logging.WARNING)
logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("matplotlib").setLevel(logging.WARNING) logging.getLogger("matplotlib").setLevel(logging.WARNING)
logging.basicConfig(level=logging.INFO, format="| %(name)s | %(levelname)s | %(message)s") logging.basicConfig(
level=logging.INFO, format="| %(name)s | %(levelname)s | %(message)s"
)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -27,6 +29,7 @@ import webbrowser
net_g = None net_g = None
def get_text(text, language_str, hps): def get_text(text, language_str, hps):
norm_text, phone, tone, word2ph = clean_text(text, language_str) norm_text, phone, tone, word2ph = clean_text(text, language_str)
phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str) phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
@@ -42,7 +45,7 @@ def get_text(text, language_str, hps):
del word2ph del word2ph
assert bert.shape[-1] == len(phone), phone assert bert.shape[-1] == len(phone), phone
if language_str=='ZH': if language_str == "ZH":
bert = bert bert = bert
ja_bert = torch.zeros(768, len(phone)) ja_bert = torch.zeros(768, len(phone))
elif language_str == "JA": elif language_str == "JA":
@@ -52,12 +55,25 @@ def get_text(text, language_str, hps):
bert = torch.zeros(1024, len(phone)) bert = torch.zeros(1024, len(phone))
ja_bert = torch.zeros(768, len(phone)) ja_bert = torch.zeros(768, len(phone))
assert bert.shape[-1] == len(phone), ( assert bert.shape[-1] == len(phone), (
bert.shape, len(phone), sum(word2ph), p1, p2, t1, t2, pold, pold2, word2ph, text, w2pho) bert.shape,
len(phone),
sum(word2ph),
p1,
p2,
t1,
t2,
pold,
pold2,
word2ph,
text,
w2pho,
)
phone = torch.LongTensor(phone) phone = torch.LongTensor(phone)
tone = torch.LongTensor(tone) tone = torch.LongTensor(tone)
language = torch.LongTensor(language) language = torch.LongTensor(language)
return bert, ja_bert, phone, tone, language return bert, ja_bert, phone, tone, language
def infer(text, sdp_ratio, noise_scale, noise_scale_w, length_scale, sid, language): def infer(text, sdp_ratio, noise_scale, noise_scale_w, length_scale, sid, language):
global net_g global net_g
bert, ja_bert, phones, tones, lang_ids = get_text(text, language, hps) bert, ja_bert, phones, tones, lang_ids = get_text(text, language, hps)
@@ -70,24 +86,60 @@ def infer(text, sdp_ratio, noise_scale, noise_scale_w, length_scale, sid, langua
x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device) x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
del phones del phones
speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device) speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
audio = net_g.infer(x_tst, x_tst_lengths, speakers, tones, lang_ids, bert, ja_bert, sdp_ratio=sdp_ratio audio = (
, noise_scale=noise_scale, noise_scale_w=noise_scale_w, length_scale=length_scale)[0][0,0].data.cpu().float().numpy() net_g.infer(
x_tst,
x_tst_lengths,
speakers,
tones,
lang_ids,
bert,
ja_bert,
sdp_ratio=sdp_ratio,
noise_scale=noise_scale,
noise_scale_w=noise_scale_w,
length_scale=length_scale,
)[0][0, 0]
.data.cpu()
.float()
.numpy()
)
del x_tst, tones, lang_ids, bert, x_tst_lengths, speakers del x_tst, tones, lang_ids, bert, x_tst_lengths, speakers
return audio return audio
def tts_fn(text, speaker, sdp_ratio, noise_scale, noise_scale_w, length_scale, language):
def tts_fn(
text, speaker, sdp_ratio, noise_scale, noise_scale_w, length_scale, language
):
with torch.no_grad(): with torch.no_grad():
audio = infer(text, sdp_ratio=sdp_ratio, noise_scale=noise_scale, noise_scale_w=noise_scale_w, length_scale=length_scale, sid=speaker, language=language) audio = infer(
text,
sdp_ratio=sdp_ratio,
noise_scale=noise_scale,
noise_scale_w=noise_scale_w,
length_scale=length_scale,
sid=speaker,
language=language,
)
torch.cuda.empty_cache() torch.cuda.empty_cache()
return "Success", (hps.data.sampling_rate, audio) return "Success", (hps.data.sampling_rate, audio)
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument("-m", "--model", default="./logs/as/G_8000.pth", help="path of your model") parser.add_argument(
parser.add_argument("-c", "--config", default="./configs/config.json", help="path of your config file") "-m", "--model", default="./logs/as/G_8000.pth", help="path of your model"
)
parser.add_argument(
"-c",
"--config",
default="./configs/config.json",
help="path of your config file",
)
parser.add_argument("--share", default=False, help="make link public") parser.add_argument("--share", default=False, help="make link public")
parser.add_argument("-d", "--debug", action="store_true", help="enable DEBUG-LEVEL log") parser.add_argument(
"-d", "--debug", action="store_true", help="enable DEBUG-LEVEL log"
)
args = parser.parse_args() args = parser.parse_args()
if args.debug: if args.debug:
@@ -109,7 +161,8 @@ if __name__ == "__main__":
hps.data.filter_length // 2 + 1, hps.data.filter_length // 2 + 1,
hps.train.segment_size // hps.data.hop_length, hps.train.segment_size // hps.data.hop_length,
n_speakers=hps.data.n_speakers, n_speakers=hps.data.n_speakers,
**hps.model).to(device) **hps.model
).to(device)
_ = net_g.eval() _ = net_g.eval()
_ = utils.load_checkpoint(args.model, net_g, None, skip_optimizer=True) _ = utils.load_checkpoint(args.model, net_g, None, skip_optimizer=True)
@@ -120,22 +173,39 @@ if __name__ == "__main__":
with gr.Blocks() as app: with gr.Blocks() as app:
with gr.Row(): with gr.Row():
with gr.Column(): with gr.Column():
text = gr.TextArea(label="Text", placeholder="Input Text Here", text = gr.TextArea(
value="吃葡萄不吐葡萄皮，不吃葡萄倒吐葡萄皮。") label="Text",
speaker = gr.Dropdown(choices=speakers, value=speakers[0], label='Speaker') placeholder="Input Text Here",
sdp_ratio = gr.Slider(minimum=0, maximum=1, value=0.2, step=0.1, label='SDP Ratio') value="吃葡萄不吐葡萄皮，不吃葡萄倒吐葡萄皮。",
noise_scale = gr.Slider(minimum=0.1, maximum=2, value=0.6, step=0.1, label='Noise Scale') )
noise_scale_w = gr.Slider(minimum=0.1, maximum=2, value=0.8, step=0.1, label='Noise Scale W') speaker = gr.Dropdown(
length_scale = gr.Slider(minimum=0.1, maximum=2, value=1, step=0.1, label='Length Scale') choices=speakers, value=speakers[0], label="Speaker"
language = gr.Dropdown(choices=languages, value=languages[0], label='Language') )
sdp_ratio = gr.Slider(
minimum=0, maximum=1, value=0.2, step=0.1, label="SDP Ratio"
)
noise_scale = gr.Slider(
minimum=0.1, maximum=2, value=0.6, step=0.1, label="Noise Scale"
)
noise_scale_w = gr.Slider(
minimum=0.1, maximum=2, value=0.8, step=0.1, label="Noise Scale W"
)
length_scale = gr.Slider(
minimum=0.1, maximum=2, value=1, step=0.1, label="Length Scale"
)
language = gr.Dropdown(
choices=languages, value=languages[0], label="Language"
)
btn = gr.Button("Generate!", variant="primary") btn = gr.Button("Generate!", variant="primary")
with gr.Column(): with gr.Column():
text_output = gr.Textbox(label="Message") text_output = gr.Textbox(label="Message")
audio_output = gr.Audio(label="Output Audio") audio_output = gr.Audio(label="Output Audio")
btn.click(tts_fn, btn.click(
tts_fn,
inputs=[text, speaker, sdp_ratio, noise_scale, noise_scale_w, length_scale], inputs=[text, speaker, sdp_ratio, noise_scale, noise_scale_w, length_scale],
outputs=[text_output, audio_output]) outputs=[text_output, audio_output],
)
webbrowser.open("http://127.0.0.1:7860") webbrowser.open("http://127.0.0.1:7860")
app.launch(share=args.share) app.launch(share=args.share)