tuna2134/sbv2-v2
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
rcell
2023-07-24 15:16:06 +08:00
parent 1fa3ce1d03
commit 9fd6e9a259
7 changed files with 348 additions and 50345 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
preprocess_text.py
View File

@@ -4,9 +4,9 @@ from random import shuffle
import tqdm
from text.cleaner import clean_text
from collections import defaultdict
stage = [1,2,3]
stage = [2,3]
transcription_path = 'filelists/aishell.list'
transcription_path = 'filelists/genshin.txt'
train_path = 'filelists/train.list'
val_path = 'filelists/val.list'
config_path = "configs/config.json"
@@ -16,11 +16,14 @@ max_val_total = 8
if 1 in stage:
with open( transcription_path+'.cleaned', 'w', encoding='utf-8') as f:
for line in tqdm.tqdm(open(transcription_path, encoding='utf-8').readlines()):
utt, spk, language, text = line.strip().split('|')
norm_text, phones, tones, word2ph = clean_text(text, language)
f.write('{}|{}|{}|{}|{}|{}|{}\n'.format(utt, spk, language, norm_text, ' '.join(phones),
" ".join([str(i) for i in tones]),
" ".join([str(i) for i in word2ph])))
try:
utt, spk, language, text = line.strip().split('|')
norm_text, phones, tones, word2ph = clean_text(text, language)
f.write('{}|{}|{}|{}|{}|{}|{}\n'.format(utt, spk, language, norm_text, ' '.join(phones),
" ".join([str(i) for i in tones]),
" ".join([str(i) for i in word2ph])))
except:
print("err!", text)
if 2 in stage:
spk_utt_map = defaultdict(list)
@@ -34,29 +37,29 @@ if 2 in stage:
if spk not in spk_id_map.keys():
spk_id_map[spk] = current_sid
current_sid += 1
train_list = []
val_list = []
for spk, utts in spk_utt_map.items():
shuffle(utts)
val_list+=utts[:val_per_spk]
train_list+=utts[val_per_spk:]
if len(val_list) > max_val_total:
train_list+=val_list[max_val_total:]
val_list = val_list[:max_val_total]
with open( train_path,"w", encoding='utf-8') as f:
for line in train_list:
f.write(line)
with open(val_path, "w", encoding='utf-8') as f:
for line in val_list:
f.write(line)
#
# train_list = []
# val_list = []
#
# for spk, utts in spk_utt_map.items():
# shuffle(utts)
# val_list+=utts[:val_per_spk]
# train_list+=utts[val_per_spk:]
# if len(val_list) > max_val_total:
# train_list+=val_list[max_val_total:]
# val_list = val_list[:max_val_total]
#
# with open( train_path,"w", encoding='utf-8') as f:
# for line in train_list:
# f.write(line)
#
# with open(val_path, "w", encoding='utf-8') as f:
# for line in val_list:
# f.write(line)
if 3 in stage:
assert 2 in stage
config = json.load(open(config_path))
config["data"]['spk2id'] = spk_id_map
with open(config_path, 'w', encoding='utf-8') as f:
json.dump(config, f, indent=2)
json.dump(config, f, indent=2, ensure_ascii=False)