From e3ce4d5a9490847115a8dcfe5cdbd1bfdfd2b536 Mon Sep 17 00:00:00 2001
From: Sora <atri@suzakuintsubaki.com>
Date: Thu, 12 Oct 2023 15:03:11 +0800
Subject: [PATCH] =?UTF-8?q?update=20preprocess=5Ftext.py=EF=BC=9A=E8=BF=87?=
 =?UTF-8?q?=E6=BB=A4=E9=9F=B3=E9=A2=91=E4=B8=8D=E5=AD=98=E5=9C=A8=E7=9A=84?=
 =?UTF-8?q?=E6=83=85=E5=86=B5=20(#58)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 preprocess_text.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/preprocess_text.py b/preprocess_text.py
index 726be95..223b706 100644
--- a/preprocess_text.py
+++ b/preprocess_text.py
@@ -1,4 +1,5 @@
 import json
+import os.path
 from collections import defaultdict
 from random import shuffle
 from typing import Optional
@@ -68,11 +69,18 @@ def main(
 
     with open(transcription_path, encoding="utf-8") as f:
         audioPaths = set()
+        countSame = 0
+        countNotFound = 0
         for line in f.readlines():
             utt, spk, language, text, phones, tones, word2ph = line.strip().split("|")
             if utt in audioPaths:
                 # 过滤数据集错误：相同的音频匹配多个文本，导致后续bert出问题
                 print(f"重复音频文本：{line}")
+                countSame += 1
+                continue
+            if not os.path.isfile(utt):
+                print(f"没有找到对应的音频：{utt}")
+                countNotFound += 1
                 continue
             audioPaths.add(utt)
             spk_utt_map[spk].append(line)
@@ -80,6 +88,7 @@ def main(
             if spk not in spk_id_map.keys():
                 spk_id_map[spk] = current_sid
                 current_sid += 1
+        print(f"总重复音频数：{countSame}，总未找到的音频数:{countNotFound}")
 
     train_list = []
     val_list = []