From 5dfe7c949340e62a1994856fc65db030055b370e Mon Sep 17 00:00:00 2001
From: Sora <atri@suzakuintsubaki.com>
Date: Thu, 12 Oct 2023 08:48:19 +0800
Subject: [PATCH] =?UTF-8?q?update=20preprocess=5Ftext.py=EF=BC=9A=E8=BF=87?=
 =?UTF-8?q?=E6=BB=A4=E4=B8=80=E4=B8=AA=E9=9F=B3=E9=A2=91=E5=8C=B9=E9=85=8D?=
 =?UTF-8?q?=E5=A4=9A=E4=B8=AA=E6=96=87=E6=9C=AC=E7=9A=84=E6=83=85=E5=86=B5?=
 =?UTF-8?q?=20(#57)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 preprocess_text.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/preprocess_text.py b/preprocess_text.py
index e827507..726be95 100644
--- a/preprocess_text.py
+++ b/preprocess_text.py
@@ -67,8 +67,14 @@ def main(
     current_sid = 0
 
     with open(transcription_path, encoding="utf-8") as f:
+        audioPaths = set()
         for line in f.readlines():
             utt, spk, language, text, phones, tones, word2ph = line.strip().split("|")
+            if utt in audioPaths:
+                # 过滤数据集错误：相同的音频匹配多个文本，导致后续bert出问题
+                print(f"重复音频文本：{line}")
+                continue
+            audioPaths.add(utt)
             spk_utt_map[spk].append(line)
 
             if spk not in spk_id_map.keys():