diff --git a/preprocess_text.py b/preprocess_text.py index 9fba5eb..4f49100 100644 --- a/preprocess_text.py +++ b/preprocess_text.py @@ -15,6 +15,12 @@ from common.log import logger preprocess_text_config = config.preprocess_text_config +# Count lines for tqdm +def count_lines(file_path): + with file_path.open("r", encoding="utf-8") as file: + return sum(1 for _ in file) + + @click.command() @click.option( "--transcription-path", @@ -34,6 +40,7 @@ preprocess_text_config = config.preprocess_text_config @click.option("--clean/--no-clean", default=preprocess_text_config.clean) @click.option("-y", "--yml_config") @click.option("--use_jp_extra", is_flag=True) +@click.option("--skip_invalid", is_flag=True) def preprocess( transcription_path: str, cleaned_path: Optional[str], @@ -45,14 +52,16 @@ def preprocess( clean: bool, yml_config: str, # 这个不要删 use_jp_extra: bool, + skip_invalid: bool = False, ): if cleaned_path == "" or cleaned_path is None: cleaned_path = transcription_path + ".cleaned" if clean: + total_lines = count_lines(transcription_path) with open(cleaned_path, "w", encoding="utf-8") as out_file: with open(transcription_path, "r", encoding="utf-8") as trans_file: - for line in tqdm(trans_file, file=SAFE_STDOUT): + for line in tqdm(trans_file, file=SAFE_STDOUT, total=total_lines): try: utt, spk, language, text = line.strip().split("|") norm_text, phones, tones, word2ph = clean_text( @@ -70,10 +79,18 @@ def preprocess( ) ) except Exception as e: - logger.error( - f"An error occurred while generating the training set and validation set, at line:\n{line}\nDetails:\n{e}" - ) - raise + if skip_invalid: + logger.warning( + f"An error occurred while generating the training set and validation set, at line:\n{line}\nDetails:\n{e}" + ) + logger.warning( + "This line will be skipped from the training set and validation set." + ) + else: + logger.error( + f"An error occurred while generating the training set and validation set, at line:\n{line}\nDetails:\n{e}" + ) + raise transcription_path = cleaned_path spk_utt_map = defaultdict(list) diff --git a/webui_train.py b/webui_train.py index ebd292e..6ca1216 100644 --- a/webui_train.py +++ b/webui_train.py @@ -150,7 +150,7 @@ def resample(model_name, normalize, trim, num_processes): return True, "Step 2, Success: 音声ファイルの前処理が完了しました" -def preprocess_text(model_name, use_jp_extra, val_per_lang): +def preprocess_text(model_name, use_jp_extra, val_per_lang, skip_invalid=False): logger.info("Step 3: start preprocessing text...") dataset_path, lbl_path, train_path, val_path, config_path = get_path(model_name) try: @@ -180,6 +180,8 @@ def preprocess_text(model_name, use_jp_extra, val_per_lang): ] if use_jp_extra: cmd.append("--use_jp_extra") + if skip_invalid: + cmd.append("--skip_invalid") success, message = run_script_with_log(cmd) if not success: logger.error(f"Step 3: preprocessing text failed.") @@ -265,6 +267,7 @@ def preprocess_all( use_jp_extra, val_per_lang, log_interval, + skip_invalid=False, ): if model_name == "": return False, "Error: モデル名を入力してください" @@ -285,7 +288,9 @@ def preprocess_all( success, message = resample(model_name, normalize, trim, num_processes) if not success: return False, message - success, message = preprocess_text(model_name, use_jp_extra, val_per_lang) + success, message = preprocess_text( + model_name, use_jp_extra, val_per_lang, skip_invalid + ) if not success: return False, message success, message = bert_gen(model_name) # bert_genは重いのでプロセス数いじらない @@ -490,6 +495,10 @@ if __name__ == "__main__": maximum=1000, step=10, ) + skip_invalid = gr.Checkbox( + label="日本語処理でエラーが出たファイルは学習に使わない", + value=False, + ) gr.Markdown("学習時に特定の部分を凍結させるかどうか") freeze_EN_bert = gr.Checkbox( label="英語bert部分を凍結", @@ -599,6 +608,10 @@ if __name__ == "__main__": maximum=100, step=1, ) + skip_invalid_manual = gr.Checkbox( + label="日本語処理でエラーが出たファイルは学習に使わない", + value=False, + ) with gr.Column(): preprocess_text_btn = gr.Button(value="実行", variant="primary") info_preprocess_text = gr.Textbox(label="状況") @@ -661,6 +674,7 @@ if __name__ == "__main__": use_jp_extra, val_per_lang, log_interval, + skip_invalid, ], outputs=[info_all], ) @@ -694,7 +708,12 @@ if __name__ == "__main__": ) preprocess_text_btn.click( second_elem_of(preprocess_text), - inputs=[model_name, use_jp_extra_manual, val_per_lang_manual], + inputs=[ + model_name, + use_jp_extra_manual, + val_per_lang_manual, + skip_invalid_manual, + ], outputs=[info_preprocess_text], ) bert_gen_btn.click(