diff --git a/preprocess_text.py b/preprocess_text.py index 726be95..223b706 100644 --- a/preprocess_text.py +++ b/preprocess_text.py @@ -1,4 +1,5 @@ import json +import os.path from collections import defaultdict from random import shuffle from typing import Optional @@ -68,11 +69,18 @@ def main( with open(transcription_path, encoding="utf-8") as f: audioPaths = set() + countSame = 0 + countNotFound = 0 for line in f.readlines(): utt, spk, language, text, phones, tones, word2ph = line.strip().split("|") if utt in audioPaths: # 过滤数据集错误:相同的音频匹配多个文本,导致后续bert出问题 print(f"重复音频文本:{line}") + countSame += 1 + continue + if not os.path.isfile(utt): + print(f"没有找到对应的音频:{utt}") + countNotFound += 1 continue audioPaths.add(utt) spk_utt_map[spk].append(line) @@ -80,6 +88,7 @@ def main( if spk not in spk_id_map.keys(): spk_id_map[spk] = current_sid current_sid += 1 + print(f"总重复音频数:{countSame},总未找到的音频数:{countNotFound}") train_list = [] val_list = []