Feat: option for skipping invalid transcription

preprocess_text.py

@@ -15,6 +15,12 @@ from common.log import logger
 preprocess_text_config = config.preprocess_text_config
 
 
+# Count lines for tqdm
+def count_lines(file_path):
+    with file_path.open("r", encoding="utf-8") as file:
+        return sum(1 for _ in file)
+
+
 @click.command()
 @click.option(
     "--transcription-path",
@@ -34,6 +40,7 @@ preprocess_text_config = config.preprocess_text_config
 @click.option("--clean/--no-clean", default=preprocess_text_config.clean)
 @click.option("-y", "--yml_config")
 @click.option("--use_jp_extra", is_flag=True)
+@click.option("--skip_invalid", is_flag=True)
 def preprocess(
     transcription_path: str,
     cleaned_path: Optional[str],
@@ -45,14 +52,16 @@ def preprocess(
     clean: bool,
     yml_config: str,  # 这个不要删
     use_jp_extra: bool,
+    skip_invalid: bool = False,
 ):
     if cleaned_path == "" or cleaned_path is None:
         cleaned_path = transcription_path + ".cleaned"
 
     if clean:
+        total_lines = count_lines(transcription_path)
         with open(cleaned_path, "w", encoding="utf-8") as out_file:
             with open(transcription_path, "r", encoding="utf-8") as trans_file:
-                for line in tqdm(trans_file, file=SAFE_STDOUT):
+                for line in tqdm(trans_file, file=SAFE_STDOUT, total=total_lines):
                     try:
                         utt, spk, language, text = line.strip().split("|")
                         norm_text, phones, tones, word2ph = clean_text(
@@ -70,6 +79,14 @@ def preprocess(
                             )
                         )
                     except Exception as e:
+                        if skip_invalid:
+                            logger.warning(
+                                f"An error occurred while generating the training set and validation set, at line:\n{line}\nDetails:\n{e}"
+                            )
+                            logger.warning(
+                                "This line will be skipped from the training set and validation set."
+                            )
+                        else:
                             logger.error(
                                 f"An error occurred while generating the training set and validation set, at line:\n{line}\nDetails:\n{e}"
                             )

webui_train.py

@@ -150,7 +150,7 @@ def resample(model_name, normalize, trim, num_processes):
     return True, "Step 2, Success: 音声ファイルの前処理が完了しました"
 
 
-def preprocess_text(model_name, use_jp_extra, val_per_lang):
+def preprocess_text(model_name, use_jp_extra, val_per_lang, skip_invalid=False):
     logger.info("Step 3: start preprocessing text...")
     dataset_path, lbl_path, train_path, val_path, config_path = get_path(model_name)
     try:
@@ -180,6 +180,8 @@ def preprocess_text(model_name, use_jp_extra, val_per_lang):
     ]
     if use_jp_extra:
         cmd.append("--use_jp_extra")
+    if skip_invalid:
+        cmd.append("--skip_invalid")
     success, message = run_script_with_log(cmd)
     if not success:
         logger.error(f"Step 3: preprocessing text failed.")
@@ -265,6 +267,7 @@ def preprocess_all(
     use_jp_extra,
     val_per_lang,
     log_interval,
+    skip_invalid=False,
 ):
     if model_name == "":
         return False, "Error: モデル名を入力してください"
@@ -285,7 +288,9 @@ def preprocess_all(
     success, message = resample(model_name, normalize, trim, num_processes)
     if not success:
         return False, message
-    success, message = preprocess_text(model_name, use_jp_extra, val_per_lang)
+    success, message = preprocess_text(
+        model_name, use_jp_extra, val_per_lang, skip_invalid
+    )
     if not success:
         return False, message
     success, message = bert_gen(model_name)  # bert_genは重いのでプロセス数いじらない
@@ -490,6 +495,10 @@ if __name__ == "__main__":
                         maximum=1000,
                         step=10,
                     )
+                    skip_invalid = gr.Checkbox(
+                        label="日本語処理でエラーが出たファイルは学習に使わない",
+                        value=False,
+                    )
                     gr.Markdown("学習時に特定の部分を凍結させるかどうか")
                     freeze_EN_bert = gr.Checkbox(
                         label="英語bert部分を凍結",
@@ -599,6 +608,10 @@ if __name__ == "__main__":
                         maximum=100,
                         step=1,
                     )
+                    skip_invalid_manual = gr.Checkbox(
+                        label="日本語処理でエラーが出たファイルは学習に使わない",
+                        value=False,
+                    )
                 with gr.Column():
                     preprocess_text_btn = gr.Button(value="実行", variant="primary")
                     info_preprocess_text = gr.Textbox(label="状況")
@@ -661,6 +674,7 @@ if __name__ == "__main__":
                 use_jp_extra,
                 val_per_lang,
                 log_interval,
+                skip_invalid,
             ],
             outputs=[info_all],
         )
@@ -694,7 +708,12 @@ if __name__ == "__main__":
         )
         preprocess_text_btn.click(
             second_elem_of(preprocess_text),
-            inputs=[model_name, use_jp_extra_manual, val_per_lang_manual],
+            inputs=[
+                model_name,
+                use_jp_extra_manual,
+                val_per_lang_manual,
+                skip_invalid_manual,
+            ],
             outputs=[info_preprocess_text],
         )
         bert_gen_btn.click(