Format & polish training and webui

attentions.py

@@ -1,4 +1,3 @@
-import copy
 import math
 import torch
 from torch import nn
@@ -341,7 +340,7 @@ class MultiHeadAttention(nn.Module):
         return ret
 
     def _get_relative_embeddings(self, relative_embeddings, length):
-        max_relative_position = 2 * self.window_size + 1
+        2 * self.window_size + 1
         # Pad first before slice to avoid using cond ops.
         pad_length = max(length - (self.window_size + 1), 0)
         slice_start_position = max((self.window_size + 1) - length, 0)
@@ -385,7 +384,7 @@ class MultiHeadAttention(nn.Module):
         ret: [b, h, l, 2*l-1]
         """
         batch, heads, length, _ = x.size()
-        # padd along column
+        # pad along column
         x = F.pad(
             x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, length - 1]])
         )

commons.py

@@ -1,7 +1,5 @@
 import math
-import numpy as np
 import torch
-from torch import nn
 from torch.nn import functional as F
 
 
@@ -16,8 +14,8 @@ def get_padding(kernel_size, dilation=1):
 
 
 def convert_pad_shape(pad_shape):
-    l = pad_shape[::-1]
+    layer = pad_shape[::-1]
-    pad_shape = [item for sublist in l for item in sublist]
+    pad_shape = [item for sublist in layer for item in sublist]
     return pad_shape
 
 
@@ -110,8 +108,8 @@ def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
 
 
 def convert_pad_shape(pad_shape):
-    l = pad_shape[::-1]
+    layer = pad_shape[::-1]
-    pad_shape = [item for sublist in l for item in sublist]
+    pad_shape = [item for sublist in layer for item in sublist]
     return pad_shape
 
 
@@ -132,7 +130,6 @@ def generate_path(duration, mask):
     duration: [b, 1, t_x]
     mask: [b, 1, t_y, t_x]
     """
-    device = duration.device
 
     b, _, t_y, t_x = mask.shape
     cum_duration = torch.cumsum(duration, -1)

configs/config.json

@@ -17,7 +17,8 @@
     "init_lr_ratio": 1,
     "warmup_epochs": 0,
     "c_mel": 45,
-    "c_kl": 1.0
+    "c_kl": 1.0,
+    "skip_optimizer": true
   },
   "data": {
     "training_files": "filelists/train.list",

data_utils.py

@@ -1,13 +1,11 @@
-import time
 import os
 import random
-import numpy as np
 import torch
 import torch.utils.data
 from tqdm import tqdm
 from loguru import logger
 import commons
-from mel_processing import spectrogram_torch, mel_spectrogram_torch, spec_to_mel_torch
+from mel_processing import spectrogram_torch, mel_spectrogram_torch
 from utils import load_wav_to_torch, load_filepaths_and_text
 from text import cleaned_text_to_sequence, get_bert
 
@@ -100,7 +98,7 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
         if sampling_rate != self.sampling_rate:
             raise ValueError(
                 "{} {} SR doesn't match target {} SR".format(
-                    sampling_rate, self.sampling_rate
+                    filename, sampling_rate, self.sampling_rate
                 )
             )
         audio_norm = audio / self.max_wav_value

losses.py

@@ -1,7 +1,4 @@
 import torch
-from torch.nn import functional as F
-
-import commons
 
 
 def feature_loss(fmap_r, fmap_g):

mel_processing.py

@@ -1,16 +1,5 @@
-import math
-import os
-import random
 import torch
-from torch import nn
-import torch.nn.functional as F
 import torch.utils.data
-import numpy as np
-import librosa
-import librosa.util as librosa_util
-from librosa.util import normalize, pad_center, tiny
-from scipy.signal import get_window
-from scipy.io.wavfile import read
 from librosa.filters import mel as librosa_mel_fn
 
 MAX_WAV_VALUE = 32768.0

modules.py

@@ -1,12 +1,9 @@
-import copy
 import math
-import numpy as np
-import scipy
 import torch
 from torch import nn
 from torch.nn import functional as F
 
-from torch.nn import Conv1d, ConvTranspose1d, AvgPool1d, Conv2d
+from torch.nn import Conv1d
 from torch.nn.utils import weight_norm, remove_weight_norm
 
 import commons

resample.py

@@ -1,11 +1,9 @@
 import os
 import argparse
 import librosa
-import numpy as np
 from multiprocessing import Pool, cpu_count
 
 import soundfile
-from scipy.io import wavfile
 from tqdm import tqdm
 
 
@@ -29,9 +27,9 @@ if __name__ == "__main__":
         "--out_dir", type=str, default="./dataset", help="path to target dir"
     )
     args = parser.parse_args()
-    # processs = 8
+    # processes = 8
-    processs = cpu_count() - 2 if cpu_count() > 4 else 1
+    processes = cpu_count() - 2 if cpu_count() > 4 else 1
-    pool = Pool(processes=processs)
+    pool = Pool(processes=processes)
 
     for speaker in os.listdir(args.in_dir):
         spk_dir = os.path.join(args.in_dir, speaker)

server.py

@@ -40,20 +40,9 @@ def get_text(text, language_str, hps):
     else:
         bert = torch.zeros(1024, len(phone))
         ja_bert = torch.zeros(768, len(phone))
-    assert bert.shape[-1] == len(phone), (
+    assert bert.shape[-1] == len(
-        bert.shape,
+        phone
-        len(phone),
+    ), f"Bert seq len {bert.shape[-1]} != {len(phone)}"
-        sum(word2ph),
-        p1,
-        p2,
-        t1,
-        t2,
-        pold,
-        pold2,
-        word2ph,
-        text,
-        w2pho,
-    )
     phone = torch.LongTensor(phone)
     tone = torch.LongTensor(tone)
     language = torch.LongTensor(language)
@@ -126,7 +115,7 @@ net_g = SynthesizerTrn(
     hps.data.filter_length // 2 + 1,
     hps.train.segment_size // hps.data.hop_length,
     n_speakers=hps.data.n_speakers,
-    **hps.model
+    **hps.model,
 ).to(dev)
 _ = net_g.eval()
 

text/chinese.py

@@ -4,7 +4,6 @@ import re
 import cn2an
 from pypinyin import lazy_pinyin, Style
 
-from text import symbols
 from text.symbols import punctuation
 from text.tone_sandhi import ToneSandhi
 
@@ -96,7 +95,6 @@ def _g2p(segments):
     tones_list = []
     word2ph = []
     for seg in segments:
-        pinyins = []
         # Replace all English words in the sentence
         seg = re.sub("[a-zA-Z]+", "", seg)
         seg_cut = psg.lcut(seg)

text/english.py

@@ -2,7 +2,6 @@ import pickle
 import os
 import re
 from g2p_en import G2p
-from string import punctuation
 
 from text import symbols
 

text/japanese.py

@@ -1,6 +1,5 @@
 # Convert Japanese text to phonemes which is
 # compatible with Julius https://github.com/julius-speech/segmentation-kit
-import math
 import re
 import unicodedata
 

text/tone_sandhi.py

@@ -536,7 +536,7 @@ class ToneSandhi:
             [item.isnumeric() for item in word if item != "一"]
         ):
             return finals
-        # "一" between reduplication words shold be yi5, e.g. 看一看
+        # "一" between reduplication words should be yi5, e.g. 看一看
         elif len(word) == 3 and word[1] == "一" and word[0] == word[-1]:
             finals[1] = finals[1][:-1] + "5"
         # when "一" is ordinal word, it should be yi1

train_ms.py

@@ -1,14 +1,10 @@
+# flake8: noqa: E402
+
 import os
-import json
-import argparse
-import itertools
-import math
 import torch
-from torch import nn, optim
 from torch.nn import functional as F
 from torch.utils.data import DataLoader
 from torch.utils.tensorboard import SummaryWriter
-import torch.multiprocessing as mp
 import torch.distributed as dist
 from torch.nn.parallel import DistributedDataParallel as DDP
 from torch.cuda.amp import autocast, GradScaler
@@ -42,7 +38,7 @@ torch.backends.cuda.sdp_kernel("flash")
 torch.backends.cuda.enable_flash_sdp(True)
 torch.backends.cuda.enable_mem_efficient_sdp(
     True
-)  # Not avaliable if torch version is lower than 2.0
+)  # Not available if torch version is lower than 2.0
 torch.backends.cuda.enable_math_sdp(True)
 global_step = 0
 
@@ -97,23 +93,20 @@ def run():
         )
     if (
         "use_noise_scaled_mas" in hps.model.keys()
-        and hps.model.use_noise_scaled_mas == True
+        and hps.model.use_noise_scaled_mas is True
     ):
         print("Using noise scaled MAS for VITS2")
-        use_noise_scaled_mas = True
         mas_noise_scale_initial = 0.01
         noise_scale_delta = 2e-6
     else:
         print("Using normal MAS for VITS1")
-        use_noise_scaled_mas = False
         mas_noise_scale_initial = 0.0
         noise_scale_delta = 0.0
     if (
         "use_duration_discriminator" in hps.model.keys()
-        and hps.model.use_duration_discriminator == True
+        and hps.model.use_duration_discriminator is True
     ):
         print("Using duration discriminator for VITS2")
-        use_duration_discriminator = True
         net_dur_disc = DurationDiscriminator(
             hps.model.hidden_channels,
             hps.model.hidden_channels,
@@ -123,16 +116,14 @@ def run():
         ).cuda(rank)
     if (
         "use_spk_conditioned_encoder" in hps.model.keys()
-        and hps.model.use_spk_conditioned_encoder == True
+        and hps.model.use_spk_conditioned_encoder is True
     ):
         if hps.data.n_speakers == 0:
             raise ValueError(
                 "n_speakers must be > 0 when using spk conditioned encoder to train multi-speaker model"
             )
-        use_spk_conditioned_encoder = True
     else:
         print("Using normal encoder for VITS1")
-        use_spk_conditioned_encoder = False
 
     net_g = SynthesizerTrn(
         len(symbols),
@@ -176,19 +167,25 @@ def run():
                 utils.latest_checkpoint_path(hps.model_dir, "DUR_*.pth"),
                 net_dur_disc,
                 optim_dur_disc,
-                skip_optimizer=True,
+                skip_optimizer=hps.train.skip_optimizer
+                if "skip_optimizer" in hps.train
+                else True,
             )
             _, optim_g, g_resume_lr, epoch_str = utils.load_checkpoint(
                 utils.latest_checkpoint_path(hps.model_dir, "G_*.pth"),
                 net_g,
                 optim_g,
-                skip_optimizer=True,
+                skip_optimizer=hps.train.skip_optimizer
+                if "skip_optimizer" in hps.train
+                else True,
             )
             _, optim_d, d_resume_lr, epoch_str = utils.load_checkpoint(
                 utils.latest_checkpoint_path(hps.model_dir, "D_*.pth"),
                 net_d,
                 optim_d,
-                skip_optimizer=True,
+                skip_optimizer=hps.train.skip_optimizer
+                if "skip_optimizer" in hps.train
+                else True,
             )
             if not optim_g.param_groups[0].get("initial_lr"):
                 optim_g.param_groups[0]["initial_lr"] = g_resume_lr
@@ -371,9 +368,7 @@ def train_and_evaluate(
                 optim_dur_disc.zero_grad()
                 scaler.scale(loss_dur_disc_all).backward()
                 scaler.unscale_(optim_dur_disc)
-                grad_norm_dur_disc = commons.clip_grad_value_(
+                commons.clip_grad_value_(net_dur_disc.parameters(), None)
-                    net_dur_disc.parameters(), None
-                )
                 scaler.step(optim_dur_disc)
 
         optim_d.zero_grad()

utils.py

@@ -1,6 +1,5 @@
 import os
 import glob
-import sys
 import argparse
 import logging
 import json
@@ -32,11 +31,13 @@ def load_checkpoint(checkpoint_path, model, optimizer=None, skip_optimizer=False
         new_opt_dict["param_groups"] = checkpoint_dict["optimizer"]["param_groups"]
         new_opt_dict["param_groups"][0]["params"] = new_opt_dict_params
         optimizer.load_state_dict(new_opt_dict)
+
     saved_state_dict = checkpoint_dict["model"]
     if hasattr(model, "module"):
         state_dict = model.module.state_dict()
     else:
         state_dict = model.state_dict()
+
     new_state_dict = {}
     for k, v in state_dict.items():
         try:
@@ -47,15 +48,26 @@ def load_checkpoint(checkpoint_path, model, optimizer=None, skip_optimizer=False
                 v.shape,
             )
         except:
-            logger.error("%s is not in the checkpoint" % k)
+            # For upgrading from the old version
+            if "ja_bert_proj" in k:
+                v = torch.zeros_like(v)
+                logger.warn(
+                    f"Seems you are using the old version of the model, the {k} is automatically set to zero for backward compatibility"
+                )
+            else:
+                logger.error(f"{k} is not in the checkpoint")
+
             new_state_dict[k] = v
+
     if hasattr(model, "module"):
         model.module.load_state_dict(new_state_dict, strict=False)
     else:
         model.load_state_dict(new_state_dict, strict=False)
+
     logger.info(
         "Loaded checkpoint '{}' (iteration {})".format(checkpoint_path, iteration)
     )
+
     return model, optimizer, learning_rate, iteration
 
 
@@ -224,20 +236,33 @@ def clean_checkpoints(path_to_models="logs/44k/", n_ckpts_to_keep=2, sort_by_tim
         for f in os.listdir(path_to_models)
         if os.path.isfile(os.path.join(path_to_models, f))
     ]
-    name_key = lambda _f: int(re.compile("._(\d+)\.pth").match(_f).group(1))
+
-    time_key = lambda _f: os.path.getmtime(os.path.join(path_to_models, _f))
+    def name_key(_f):
+        return int(re.compile("._(\\d+)\\.pth").match(_f).group(1))
+
+    def time_key(_f):
+        return os.path.getmtime(os.path.join(path_to_models, _f))
+
     sort_key = time_key if sort_by_time else name_key
-    x_sorted = lambda _x: sorted(
+
-        [f for f in ckpts_files if f.startswith(_x) and not f.endswith("_0.pth")],
+    def x_sorted(_x):
-        key=sort_key,
+        return sorted(
-    )
+            [f for f in ckpts_files if f.startswith(_x) and not f.endswith("_0.pth")],
+            key=sort_key,
+        )
+
     to_del = [
         os.path.join(path_to_models, fn)
         for fn in (x_sorted("G")[:-n_ckpts_to_keep] + x_sorted("D")[:-n_ckpts_to_keep])
     ]
-    del_info = lambda fn: logger.info(f".. Free up space by deleting ckpt {fn}")
+
-    del_routine = lambda x: [os.remove(x), del_info(x)]
+    def del_info(fn):
-    rs = [del_routine(fn) for fn in to_del]
+        return logger.info(f".. Free up space by deleting ckpt {fn}")
+
+    def del_routine(x):
+        return [os.remove(x), del_info(x)]
+
+    [del_routine(fn) for fn in to_del]
 
 
 def get_hparams_from_dir(model_dir):

webui.py

@@ -1,3 +1,5 @@
+# flake8: noqa: E402
+
 import sys, os
 import logging
 
@@ -56,20 +58,11 @@ def get_text(text, language_str, hps):
     else:
         bert = torch.zeros(1024, len(phone))
         ja_bert = torch.zeros(768, len(phone))
-    assert bert.shape[-1] == len(phone), (
+
-        bert.shape,
+    assert bert.shape[-1] == len(
-        len(phone),
+        phone
-        sum(word2ph),
+    ), f"Bert seq len {bert.shape[-1]} != {len(phone)}"
-        p1,
+
-        p2,
-        t1,
-        t2,
-        pold,
-        pold2,
-        word2ph,
-        text,
-        w2pho,
-    )
     phone = torch.LongTensor(phone)
     tone = torch.LongTensor(tone)
     language = torch.LongTensor(language)
@@ -138,7 +131,9 @@ if __name__ == "__main__":
         default="./configs/config.json",
         help="path of your config file",
     )
-    parser.add_argument("--share", default=False, help="make link public")
+    parser.add_argument(
+        "--share", default=False, help="make link public", action="store_true"
+    )
     parser.add_argument(
         "-d", "--debug", action="store_true", help="enable DEBUG-LEVEL log"
     )
@@ -163,7 +158,7 @@ if __name__ == "__main__":
         hps.data.filter_length // 2 + 1,
         hps.train.segment_size // hps.data.hop_length,
         n_speakers=hps.data.n_speakers,
-        **hps.model
+        **hps.model,
     ).to(device)
     _ = net_g.eval()