From db74ac71c2ae16f74f13c0f2d5c158e63a526372 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stardust=C2=B7=E5=87=8F?= Date: Sun, 3 Sep 2023 00:15:44 +0800 Subject: [PATCH 01/13] Update README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2d2c104..2083ee4 100644 --- a/README.md +++ b/README.md @@ -2,4 +2,6 @@ VITS2 Backbone with bert ## 成熟的旅行者/开拓者/舰长/博士/sensei/猎魔人/喵喵露/V应该参阅代码自己学习如何训练。 -### 严禁将此项目用于一切违反《中华人民共和国宪法》，《中华人民共和国刑法》，《中华人民共和国治安管理处罚法》和《中华人民共和国民法典》之用途。 \ No newline at end of file +### 严禁将此项目用于一切违反《中华人民共和国宪法》，《中华人民共和国刑法》，《中华人民共和国治安管理处罚法》和《中华人民共和国民法典》之用途。 +#### Video:https://www.bilibili.com/video/BV1hp4y1K78E +#### Demo:https://www.bilibili.com/video/BV1TF411k78w From d81876c456382857faceaf892de18f6b1d6bace0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stardust=C2=B7=E5=87=8F?= Date: Sun, 3 Sep 2023 00:42:26 +0800 Subject: [PATCH 02/13] =?UTF-8?q?=E9=80=82=E9=85=8Dwindows?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bert_gen.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/bert_gen.py b/bert_gen.py index d873223..1087095 100644 --- a/bert_gen.py +++ b/bert_gen.py @@ -5,7 +5,7 @@ import commons import utils from data_utils import TextAudioSpeakerLoader, TextAudioSpeakerCollate from tqdm import tqdm -from multiprocessing import Pool +import warnings from text import cleaned_text_to_sequence, get_bert @@ -40,14 +40,14 @@ def process_line(line): torch.save(bert, bert_path) -lines = [] -with open(hps.data.training_files) as f: - lines.extend(f.readlines()) +if __name__ == '__main__': + lines = [] + with open(hps.data.training_files) as f: + lines.extend(f.readlines()) -with open(hps.data.validation_files) as f: - lines.extend(f.readlines()) + with open(hps.data.validation_files) as f: + lines.extend(f.readlines()) - -with Pool(processes=12) as pool: #A100 suitable config,if coom,please decrease the processess number. - for _ in tqdm(pool.imap_unordered(process_line, lines)): - pass + with Pool(processes=12) as pool: #A100 suitable config,if coom,please decrease the processess number. + for _ in tqdm(pool.imap_unordered(process_line, lines)): + pass From 7dd803a6a89eeb5c1d3e10c5fcf2c5828fdbcc51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stardust=C2=B7=E5=87=8F?= Date: Sun, 3 Sep 2023 00:57:17 +0800 Subject: [PATCH 03/13] fix --- bert_gen.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bert_gen.py b/bert_gen.py index 1087095..193e768 100644 --- a/bert_gen.py +++ b/bert_gen.py @@ -42,12 +42,12 @@ def process_line(line): if __name__ == '__main__': lines = [] - with open(hps.data.training_files) as f: + with open(hps.data.training_files, encoding='utf-8' ) as f: lines.extend(f.readlines()) - with open(hps.data.validation_files) as f: + with open(hps.data.validation_files, encoding='utf-8' ) as f: lines.extend(f.readlines()) - with Pool(processes=12) as pool: #A100 suitable config,if coom,please decrease the processess number. + with Pool(processes=12) as pool: #A100 40GB suitable config,if coom,please decrease the processess number. for _ in tqdm(pool.imap_unordered(process_line, lines)): pass From 1835ed244269b263fff31ac51ec30e576673fbcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stardust=C2=B7=E5=87=8F?= Date: Sun, 3 Sep 2023 12:35:58 +0800 Subject: [PATCH 04/13] add drop_last --- train_ms.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/train_ms.py b/train_ms.py index 93846c2..4669f3b 100644 --- a/train_ms.py +++ b/train_ms.py @@ -78,7 +78,8 @@ def run(rank, n_gpus, hps): shuffle=True) collate_fn = TextAudioSpeakerCollate() train_loader = DataLoader(train_dataset, num_workers=24, shuffle=False, pin_memory=True, - collate_fn=collate_fn, batch_sampler=train_sampler, persistent_workers=True,prefetch_factor=4) + collate_fn=collate_fn, batch_sampler=train_sampler, persistent_workers=True, + prefetch_factor=4,drop_last=True) #256G Memory suitable loader. if rank == 0: eval_dataset = TextAudioSpeakerLoader(hps.data.validation_files, hps.data) eval_loader = DataLoader(eval_dataset, num_workers=0, shuffle=False, From 2ca8ee78530ef4e0b855fbeb499d69f456c86296 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stardust=C2=B7=E5=87=8F?= Date: Sun, 3 Sep 2023 12:36:35 +0800 Subject: [PATCH 05/13] try --- models.py | 40 +++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/models.py b/models.py index 9951bf6..87026d3 100644 --- a/models.py +++ b/models.py @@ -15,8 +15,6 @@ from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm from commons import init_weights, get_padding from text import symbols, num_tones, num_languages class DurationDiscriminator(nn.Module): #vits2 - # TODO : not using "spk conditioning" for now according to the paper. - # Can be a better discriminator if we use it. def __init__(self, in_channels, filter_channels, kernel_size, p_dropout, gin_channels=0): super().__init__() @@ -28,9 +26,9 @@ class DurationDiscriminator(nn.Module): #vits2 self.drop = nn.Dropout(p_dropout) self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size, padding=kernel_size//2) - #self.norm_1 = modules.LayerNorm(filter_channels) + self.norm_1 = modules.LayerNorm(filter_channels) self.conv_2 = nn.Conv1d(filter_channels, filter_channels, kernel_size, padding=kernel_size//2) - #self.norm_2 = modules.LayerNorm(filter_channels) + self.norm_2 = modules.LayerNorm(filter_channels) self.dur_proj = nn.Conv1d(1, filter_channels, 1) self.pre_out_conv_1 = nn.Conv1d(2*filter_channels, filter_channels, kernel_size, padding=kernel_size//2) @@ -38,8 +36,8 @@ class DurationDiscriminator(nn.Module): #vits2 self.pre_out_conv_2 = nn.Conv1d(filter_channels, filter_channels, kernel_size, padding=kernel_size//2) self.pre_out_norm_2 = modules.LayerNorm(filter_channels) - # if gin_channels != 0: - # self.cond = nn.Conv1d(gin_channels, in_channels, 1) + if gin_channels != 0: + self.cond = nn.Conv1d(gin_channels, in_channels, 1) self.output_layer = nn.Sequential( nn.Linear(filter_channels, 1), @@ -50,13 +48,13 @@ class DurationDiscriminator(nn.Module): #vits2 dur = self.dur_proj(dur) x = torch.cat([x, dur], dim=1) x = self.pre_out_conv_1(x * x_mask) - #x = torch.relu(x) - #x = self.pre_out_norm_1(x) - #x = self.drop(x) + x = torch.relu(x) + x = self.pre_out_norm_1(x) + x = self.drop(x) x = self.pre_out_conv_2(x * x_mask) - #x = torch.relu(x) - #x = self.pre_out_norm_2(x) - #x = self.drop(x) + x = torch.relu(x) + x = self.pre_out_norm_2(x) + x = self.drop(x) x = x * x_mask x = x.transpose(1, 2) output_prob = self.output_layer(x) @@ -64,17 +62,17 @@ class DurationDiscriminator(nn.Module): #vits2 def forward(self, x, x_mask, dur_r, dur_hat, g=None): x = torch.detach(x) - # if g is not None: - # g = torch.detach(g) - # x = x + self.cond(g) + if g is not None: + g = torch.detach(g) + x = x + self.cond(g) x = self.conv_1(x * x_mask) - #x = torch.relu(x) - #x = self.norm_1(x) - #x = self.drop(x) + x = torch.relu(x) + x = self.norm_1(x) + x = self.drop(x) x = self.conv_2(x * x_mask) - #x = torch.relu(x) - #x = self.norm_2(x) - #x = self.drop(x) + x = torch.relu(x) + x = self.norm_2(x) + x = self.drop(x) output_probs = [] for dur in [dur_r, dur_hat]: From 95cfacc76c13eb2fdd0cf491130516a63bbbacbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stardust=C2=B7=E5=87=8F?= Date: Sun, 3 Sep 2023 12:36:53 +0800 Subject: [PATCH 06/13] try --- train_ms.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/train_ms.py b/train_ms.py index 4669f3b..9afea31 100644 --- a/train_ms.py +++ b/train_ms.py @@ -147,10 +147,10 @@ def run(rank, n_gpus, hps): eps=hps.train.eps) else: optim_dur_disc = None - net_g = DDP(net_g, device_ids=[rank],find_unused_parameters=True) - net_d = DDP(net_d, device_ids=[rank],find_unused_parameters=True) + net_g = DDP(net_g, device_ids=[rank]) + net_d = DDP(net_d, device_ids=[rank]) if net_dur_disc is not None: - net_dur_disc = DDP(net_dur_disc, device_ids=[rank], find_unused_parameters=True) + net_dur_disc = DDP(net_dur_disc, device_ids=[rank]) pretrain_dir = None if pretrain_dir is None: From ad0870d8e1aa0f43c8480cf602b6e3106c79f96c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stardust=C2=B7=E5=87=8F?= Date: Sun, 3 Sep 2023 13:01:36 +0800 Subject: [PATCH 07/13] Update preprocess_text.py --- preprocess_text.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/preprocess_text.py b/preprocess_text.py index ca3a934..d517548 100644 --- a/preprocess_text.py +++ b/preprocess_text.py @@ -23,8 +23,8 @@ if 1 in stage: f.write('{}|{}|{}|{}|{}|{}|{}\n'.format(utt, spk, language, norm_text, ' '.join(phones), " ".join([str(i) for i in tones]), " ".join([str(i) for i in word2ph]))) - except: - print("err!", utt) + except Exception as error : + print("err!", utt, error) if 2 in stage: spk_utt_map = defaultdict(list) From d9a5696cf426cc696f162748df35b96d13a81d43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stardust=C2=B7=E5=87=8F?= Date: Sun, 3 Sep 2023 13:12:27 +0800 Subject: [PATCH 08/13] fix --- models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models.py b/models.py index 87026d3..a8e1e9f 100644 --- a/models.py +++ b/models.py @@ -62,7 +62,7 @@ class DurationDiscriminator(nn.Module): #vits2 def forward(self, x, x_mask, dur_r, dur_hat, g=None): x = torch.detach(x) - if g is not None: + if g is not None: g = torch.detach(g) x = x + self.cond(g) x = self.conv_1(x * x_mask) From c18837481d22b3aa3155c499bb19e503d048afa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stardust=C2=B7=E5=87=8F?= Date: Sun, 3 Sep 2023 13:14:55 +0800 Subject: [PATCH 09/13] fix --- train_ms.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/train_ms.py b/train_ms.py index 9afea31..4284a19 100644 --- a/train_ms.py +++ b/train_ms.py @@ -78,8 +78,8 @@ def run(rank, n_gpus, hps): shuffle=True) collate_fn = TextAudioSpeakerCollate() train_loader = DataLoader(train_dataset, num_workers=24, shuffle=False, pin_memory=True, - collate_fn=collate_fn, batch_sampler=train_sampler, persistent_workers=True, - prefetch_factor=4,drop_last=True) #256G Memory suitable loader. + collate_fn=collate_fn, batch_sampler=train_sampler, + persistent_workers=True,prefetch_factor=4) #256G Memory suitable loader. if rank == 0: eval_dataset = TextAudioSpeakerLoader(hps.data.validation_files, hps.data) eval_loader = DataLoader(eval_dataset, num_workers=0, shuffle=False, From ffad37c97519f4e73934e2755d755c05b1b23190 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stardust=C2=B7=E5=87=8F?= Date: Sun, 3 Sep 2023 13:24:13 +0800 Subject: [PATCH 10/13] fix --- bert_gen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bert_gen.py b/bert_gen.py index 193e768..4481471 100644 --- a/bert_gen.py +++ b/bert_gen.py @@ -1,6 +1,6 @@ import torch from torch.utils.data import DataLoader - +from multiprocessing import Pool import commons import utils from data_utils import TextAudioSpeakerLoader, TextAudioSpeakerCollate From 6ec8e78d4fcf9d5fd1e5142bb023bde1dbc7c771 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stardust=C2=B7=E5=87=8F?= Date: Sun, 3 Sep 2023 13:27:24 +0800 Subject: [PATCH 11/13] fix --- attentions.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/attentions.py b/attentions.py index ecbdbc8..b5f2fec 100644 --- a/attentions.py +++ b/attentions.py @@ -43,11 +43,11 @@ class Encoder(nn.Module): self.kernel_size = kernel_size self.p_dropout = p_dropout self.window_size = window_size - if isflow: - cond_layer = torch.nn.Conv1d(256, 2*hidden_channels*n_layers, 1) - self.cond_pre = torch.nn.Conv1d(hidden_channels, 2*hidden_channels, 1) - self.cond_layer = weight_norm(cond_layer, name='weight') - self.gin_channels = 256 + #if isflow: + # cond_layer = torch.nn.Conv1d(256, 2*hidden_channels*n_layers, 1) + # self.cond_pre = torch.nn.Conv1d(hidden_channels, 2*hidden_channels, 1) + # self.cond_layer = weight_norm(cond_layer, name='weight') + # self.gin_channels = 256 self.cond_layer_idx = self.n_layers if 'gin_channels' in kwargs: self.gin_channels = kwargs['gin_channels'] From d0015a0ca43c6afc288cee985f8ae740de0c56bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stardust=C2=B7=E5=87=8F?= Date: Sun, 3 Sep 2023 13:38:32 +0800 Subject: [PATCH 12/13] Update train_ms.py --- train_ms.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/train_ms.py b/train_ms.py index 4284a19..55f2bb5 100644 --- a/train_ms.py +++ b/train_ms.py @@ -147,10 +147,10 @@ def run(rank, n_gpus, hps): eps=hps.train.eps) else: optim_dur_disc = None - net_g = DDP(net_g, device_ids=[rank]) - net_d = DDP(net_d, device_ids=[rank]) + net_g = DDP(net_g, device_ids=[rank], find_unused_parameters=True) + net_d = DDP(net_d, device_ids=[rank], find_unused_parameters=True) if net_dur_disc is not None: - net_dur_disc = DDP(net_dur_disc, device_ids=[rank]) + net_dur_disc = DDP(net_dur_disc, device_ids=[rank], find_unused_parameters=True) pretrain_dir = None if pretrain_dir is None: From ca4b39cdcdddbdbc1be9d17a5ea1cb8de62c72ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stardust=C2=B7=E5=87=8F?= Date: Sun, 3 Sep 2023 14:23:04 +0800 Subject: [PATCH 13/13] fix dur --- models.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/models.py b/models.py index a8e1e9f..5248dcc 100644 --- a/models.py +++ b/models.py @@ -26,18 +26,18 @@ class DurationDiscriminator(nn.Module): #vits2 self.drop = nn.Dropout(p_dropout) self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size, padding=kernel_size//2) - self.norm_1 = modules.LayerNorm(filter_channels) + #self.norm_1 = modules.LayerNorm(filter_channels) self.conv_2 = nn.Conv1d(filter_channels, filter_channels, kernel_size, padding=kernel_size//2) - self.norm_2 = modules.LayerNorm(filter_channels) + #self.norm_2 = modules.LayerNorm(filter_channels) self.dur_proj = nn.Conv1d(1, filter_channels, 1) self.pre_out_conv_1 = nn.Conv1d(2*filter_channels, filter_channels, kernel_size, padding=kernel_size//2) - self.pre_out_norm_1 = modules.LayerNorm(filter_channels) + #self.pre_out_norm_1 = modules.LayerNorm(filter_channels) self.pre_out_conv_2 = nn.Conv1d(filter_channels, filter_channels, kernel_size, padding=kernel_size//2) - self.pre_out_norm_2 = modules.LayerNorm(filter_channels) + #self.pre_out_norm_2 = modules.LayerNorm(filter_channels) - if gin_channels != 0: - self.cond = nn.Conv1d(gin_channels, in_channels, 1) + #if gin_channels != 0: + # self.cond = nn.Conv1d(gin_channels, in_channels, 1) self.output_layer = nn.Sequential( nn.Linear(filter_channels, 1), @@ -48,13 +48,13 @@ class DurationDiscriminator(nn.Module): #vits2 dur = self.dur_proj(dur) x = torch.cat([x, dur], dim=1) x = self.pre_out_conv_1(x * x_mask) - x = torch.relu(x) - x = self.pre_out_norm_1(x) - x = self.drop(x) + #x = torch.relu(x) + #x = self.pre_out_norm_1(x) + #x = self.drop(x) x = self.pre_out_conv_2(x * x_mask) - x = torch.relu(x) - x = self.pre_out_norm_2(x) - x = self.drop(x) + #x = torch.relu(x) + #x = self.pre_out_norm_2(x) + #x = self.drop(x) x = x * x_mask x = x.transpose(1, 2) output_prob = self.output_layer(x) @@ -62,17 +62,17 @@ class DurationDiscriminator(nn.Module): #vits2 def forward(self, x, x_mask, dur_r, dur_hat, g=None): x = torch.detach(x) - if g is not None: - g = torch.detach(g) - x = x + self.cond(g) + #if g is not None: + # g = torch.detach(g) + # x = x + self.cond(g) x = self.conv_1(x * x_mask) - x = torch.relu(x) - x = self.norm_1(x) - x = self.drop(x) + #x = torch.relu(x) + #x = self.norm_1(x) + #x = self.drop(x) x = self.conv_2(x * x_mask) - x = torch.relu(x) - x = self.norm_2(x) - x = self.drop(x) + #x = torch.relu(x) + #x = self.norm_2(x) + #x = self.drop(x) output_probs = [] for dur in [dur_r, dur_hat]: