update

2023-07-22 08:17:25 +08:00
parent abac0cb146
commit 8fc674fe2c
8 changed files with 50264 additions and 249 deletions
--- a/configs/config.json
+++ b/configs/config.json
@@ -34,7 +34,150 @@
    "n_speakers": 300,
    "cleaned_text": true,
    "spk2id": {
-      "0001_Angry": 0
+      "SSB0080": 0,
+      "SSB0012": 1,
+      "SSB0038": 2,
+      "SSB0382": 3,
+      "SSB0394": 4,
+      "SSB0395": 5,
+      "SSB0316": 6,
+      "SSB0200": 7,
+      "SSB1408": 8,
+      "SSB1392": 9,
+      "SSB0700": 10,
+      "SSB1138": 11,
+      "SSB1072": 12,
+      "SSB0751": 13,
+      "SSB0338": 14,
+      "SSB0435": 15,
+      "SSB0913": 16,
+      "SSB1806": 17,
+      "SSB1878": 18,
+      "SSB1385": 19,
+      "SSB0817": 20,
+      "SSB0599": 21,
+      "SSB0887": 22,
+      "SSB0720": 23,
+      "SSB1091": 24,
+      "SSB0786": 25,
+      "SSB0737": 26,
+      "SSB0666": 27,
+      "SSB0606": 28,
+      "SSB0535": 29,
+      "SSB0112": 30,
+      "SSB1161": 31,
+      "SSB1448": 32,
+      "SSB1684": 33,
+      "SSB1699": 34,
+      "SSB1341": 35,
+      "SSB0919": 36,
+      "SSB1056": 37,
+      "SSB1115": 38,
+      "SSB1563": 39,
+      "SSB0482": 40,
+      "SSB0502": 41,
+      "SSB0415": 42,
+      "SSB0145": 43,
+      "0001_Angry": 44,
+      "0001_Happy": 45,
+      "0001_Neutral": 46,
+      "0001_Sad": 47,
+      "0001_Surprise": 48,
+      "0002_Angry": 49,
+      "0002_Happy": 50,
+      "0002_Neutral": 51,
+      "0002_Sad": 52,
+      "0002_Surprise": 53,
+      "0003_Angry": 54,
+      "0003_Happy": 55,
+      "0003_Neutral": 56,
+      "0003_Sad": 57,
+      "0003_Surprise": 58,
+      "0004_Angry": 59,
+      "0004_Happy": 60,
+      "0004_Neutral": 61,
+      "0004_Sad": 62,
+      "0004_Surprise": 63,
+      "0005_Angry": 64,
+      "0005_Happy": 65,
+      "0005_Neutral": 66,
+      "0005_Sad": 67,
+      "0005_Surprise": 68,
+      "0006_Angry": 69,
+      "0006_Happy": 70,
+      "0006_Neutral": 71,
+      "0006_Sad": 72,
+      "0006_Surprise": 73,
+      "0007_Angry": 74,
+      "0007_Happy": 75,
+      "0007_Neutral": 76,
+      "0007_Sad": 77,
+      "0007_Surprise": 78,
+      "0008_Angry": 79,
+      "0008_Happy": 80,
+      "0008_Neutral": 81,
+      "0008_Sad": 82,
+      "0008_Surprise": 83,
+      "0009_Angry": 84,
+      "0009_Happy": 85,
+      "0009_Neutral": 86,
+      "0009_Sad": 87,
+      "0009_Surprise": 88,
+      "0010_Angry": 89,
+      "0010_Happy": 90,
+      "0010_Neutral": 91,
+      "0010_Sad": 92,
+      "0010_Surprise": 93,
+      "0011_Angry": 94,
+      "0011_Happy": 95,
+      "0011_Neutral": 96,
+      "0011_Sad": 97,
+      "0011_Surprise": 98,
+      "0012_Angry": 99,
+      "0012_Happy": 100,
+      "0012_Neutral": 101,
+      "0012_Sad": 102,
+      "0012_Surprise": 103,
+      "0013_Angry": 104,
+      "0013_Happy": 105,
+      "0013_Neutral": 106,
+      "0013_Sad": 107,
+      "0013_Surprise": 108,
+      "0014_Angry": 109,
+      "0014_Happy": 110,
+      "0014_Neutral": 111,
+      "0014_Sad": 112,
+      "0014_Surprise": 113,
+      "0015_Angry": 114,
+      "0015_Happy": 115,
+      "0015_Neutral": 116,
+      "0015_Sad": 117,
+      "0015_Surprise": 118,
+      "0016_Angry": 119,
+      "0016_Happy": 120,
+      "0016_Neutral": 121,
+      "0016_Sad": 122,
+      "0016_Surprise": 123,
+      "0017_Angry": 124,
+      "0017_Happy": 125,
+      "0017_Neutral": 126,
+      "0017_Sad": 127,
+      "0017_Surprise": 128,
+      "0018_Angry": 129,
+      "0018_Happy": 130,
+      "0018_Neutral": 131,
+      "0018_Sad": 132,
+      "0018_Surprise": 133,
+      "0019_Angry": 134,
+      "0019_Happy": 135,
+      "0019_Neutral": 136,
+      "0019_Sad": 137,
+      "0019_Surprise": 138,
+      "0020_Angry": 139,
+      "0020_Happy": 140,
+      "0020_Neutral": 141,
+      "0020_Sad": 142,
+      "0020_Surprise": 143
    }
  },
  "model": {
--- a/data_utils.py
+++ b/data_utils.py
@@ -8,7 +8,7 @@ import torch.utils.data
 import commons 
 from mel_processing import spectrogram_torch
 from utils import load_wav_to_torch, load_filepaths_and_text
-from text import cleaned_text_to_sequence, cleaned_text_to_sequence_bert, get_bert
+from text import cleaned_text_to_sequence, get_bert

 """Multi speaker version"""
 class TextAudioSpeakerLoader(torch.utils.data.Dataset):
@@ -66,7 +66,7 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
        # separate filename, speaker_id and text
        audiopath, sid, language, text, phones, tone, word2ph = audiopath_sid_text

-        bert, phones, tone, language = self.get_text(text, word2ph, phones, tone, language)
+        bert, phones, tone, language = self.get_text(text, word2ph, phones, tone, language,audiopath)

        spec, wav = self.get_audio(audiopath)
        sid = torch.LongTensor([int(self.spk_map[sid])])
@@ -90,15 +90,33 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
            torch.save(spec, spec_filename)
        return spec, audio_norm

-    def get_text(self,text, word2ph,phone, tone, language_str):
+    def get_text(self,text, word2ph,phone, tone, language_str, wav_path):
+        # print(text, word2ph,phone, tone, language_str)
+        pold = phone
+        w2pho = [i for i in word2ph]
+        word2ph = [i for i in word2ph]
        phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
+        pold2 = phone
+
        if self.add_blank:
+            p1 = len(phone)
            phone = commons.intersperse(phone, 0)
+            p2 = len(phone)
+            t1 = len(tone)
            tone = commons.intersperse(tone, 0)
+            t2 = len(tone)
            language = commons.intersperse(language, 0)
-            for i in range(1,len(word2ph)):
-                word2ph[i] += 1
+            for i in range(len(word2ph)):
+                word2ph[i] = word2ph[i] * 2
+            word2ph[0] += 1
+        bert_path = wav_path.replace(".wav", ".bert.pt")
+        try:
+            bert = torch.load(bert_path)
+            assert bert.shape[-1] == len(phone)
+        except:
            bert = get_bert(text, word2ph, language_str)
+        assert bert.shape[-1] == len(phone), (bert.shape, len(phone), sum(word2ph), p1, p2, t1, t2, pold, pold2,word2ph, text,w2pho)
+        torch.save(bert, bert_path)
        phone = torch.LongTensor(phone)
        tone = torch.LongTensor(tone)
        language = torch.LongTensor(language)
--- a/filelists/train.list
+++ b/filelists/train.list
--- a/filelists/val.list
+++ b/filelists/val.list
@@ -1,2 +1,8 @@
-0001_000529|0001_Angry|ZH|她和维克分手了,所以她申请转调.|_ t a h e w ei k e f en sh ou l e , s uo y i t a sh en q ing zh uan d iao . _|0 1 1 2 2 2 2 4 4 1 1 3 3 5 5 0 2 2 3 3 1 1 1 1 3 3 3 3 4 4 0 0|1 2 2 2 2 2 2 2 1 2 2 2 2 2 2 2 1 1
-0001_000443|0001_Angry|ZH|别不好意思.再多吃些鸡肉.|_ b ie b u h ao y i s i0 . z ai d uo ch ir x ie j i r ou . _|0 2 2 4 4 3 3 4 4 5 5 0 4 4 1 1 1 1 1 1 1 1 4 4 0 0|1 2 2 2 2 2 1 2 2 2 2 2 2 1 1
+SSB00800286|SSB0080|ZH|合肥的-城镇-有什么.|_ h e f ei d e - ch eng zh en - y ou sh en m e . _|0 2 2 2 2 5 5 0 2 2 4 4 0 3 3 2 2 5 5 0 0|1 2 2 2 1 2 2 1 2 2 2 1 1
+SSB00800245|SSB0080|ZH|跟-以前的-地方-差不多-一样.|_ g en - y i q ian d e - d i f ang - ch a b u d uo - y i y ang . _|0 1 1 0 3 3 2 2 5 5 0 4 4 5 5 0 4 4 5 5 1 1 0 2 2 4 4 0 0|1 2 1 2 2 2 1 2 2 1 2 2 2 1 2 2 1 1
+SSB00120433|SSB0012|ZH|江华的-连续剧-有什么.|_ j iang h ua d e - l ian x v j v - y ou sh en m e . _|0 1 1 2 2 5 5 0 2 2 4 4 4 4 0 3 3 2 2 5 5 0 0|1 2 2 2 1 2 2 2 1 2 2 2 1 1
+SSB00120132|SSB0012|ZH|还在你-老爸的-思达-柜台-干活吗.|_ h ai z ai n i - l ao b a d e - s i0 d a - g ui t ai - g an h uo m a . _|0 2 2 4 4 3 3 0 3 3 4 4 5 5 0 1 1 2 2 0 4 4 2 2 0 4 4 2 2 5 5 0 0|1 2 2 2 1 2 2 2 1 2 2 1 2 2 1 2 2 2 1 1
+SSB00380151|SSB0038|ZH|给我-放首歌-谭咏麟的歌.|_ g ei w o - f ang sh ou g e - t an y ong l in d e g e . _|0 2 2 3 3 0 4 4 3 3 1 1 0 2 2 3 3 2 2 5 5 1 1 0 0|1 2 2 1 2 2 2 1 2 2 2 2 2 1 1
+SSB00380331|SSB0038|ZH|推测-投放-时间为.十九日-下午-或-傍晚.|_ t ui c e - t ou f ang - sh ir j ian w ei . sh ir j iu r ir - x ia w u - h uo - b ang w an . _|0 1 1 4 4 0 2 2 4 4 0 2 2 1 1 4 4 0 2 2 3 3 4 4 0 4 4 3 3 0 4 4 0 4 4 3 3 0 0|1 2 2 1 2 2 1 2 2 2 1 2 2 2 1 2 2 1 2 1 2 2 1 1
+SSB03820389|SSB0382|ZH|停车场.|_ t ing ch e ch ang . _|0 2 2 1 1 3 3 0 0|1 2 2 2 1 1
+SSB03820048|SSB0382|ZH|宝华-图文.|_ b ao h ua - t u w en . _|0 3 3 2 2 0 2 2 2 2 0 0|1 2 2 1 2 2 1 1
--- a/models.py
+++ b/models.py
@@ -172,7 +172,7 @@ class TextEncoder(nn.Module):
        self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)

    def forward(self, x, x_lengths, tone, language, bert):
-        x = (self.emb(x)+ self.tone_emb(tone)+ self.language_emb(language) +self.bert_proj(bert)) * math.sqrt(self.hidden_channels)  # [b, t, h]
+        x = (self.emb(x)+ self.tone_emb(tone)+ self.language_emb(language)+self.bert_proj(bert).transpose(1,2)) * math.sqrt(self.hidden_channels)  # [b, t, h]
        x = torch.transpose(x, 1, -1)  # [b, h, t]
        x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(x.dtype)

--- a/preprocess_text.py
+++ b/preprocess_text.py
@@ -6,7 +6,7 @@ from text.cleaner import clean_text
 from collections import defaultdict
 stage = [1,2,3]

-transcription_path = 'filelists/esd.list'
+transcription_path = 'filelists/aishell.list'
 train_path = 'filelists/train.list'
 val_path = 'filelists/val.list'
 config_path = "configs/config.json"
--- a/spec_gen.py
+++ b/spec_gen.py
@@ -0,0 +1,14 @@
+import utils
+from data_utils import TextAudioSpeakerLoader
+from tqdm import tqdm
+
+config_path = 'configs/config.json'
+hps = utils.get_hparams_from_file(config_path)
+
+train_dataset = TextAudioSpeakerLoader("filelists/train.list", hps.data)
+eval_dataset = TextAudioSpeakerLoader("filelists/val.list", hps.data)
+
+for _ in tqdm(train_dataset):
+    pass
+for _ in tqdm(eval_dataset):
+    pass
--- a/text/init.py
+++ b/text/init.py
@@ -18,8 +18,8 @@ def cleaned_text_to_sequence(cleaned_text, tones, language):
  return phones, tones, lang_ids

 def get_bert(norm_text, word2ph, language):
-  from chinese_bert import get_bert_feature as zh_bert
-  from english_bert_mock import get_bert_feature as en_bert
+  from .chinese_bert import get_bert_feature as zh_bert
+  from .english_bert_mock import get_bert_feature as en_bert
  lang_bert_func_map = {
    'ZH': zh_bert,
    'EN': en_bert