tuna2134/sbv2-v2
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Improve: log error for preprocessing text

Browse Source
This commit is contained in:
litagin02
2024-02-19 18:35:22 +09:00
parent 036cea17f3
commit 4c8ea82959
2 changed files with 20 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
preprocess_all.py
View File

@@ -55,7 +55,7 @@ if __name__ == "__main__":
parser.add_argument( parser.add_argument(
"--use_jp_extra", "--use_jp_extra",
action="store_true", action="store_true",
help="Use JP-Extra pretrained model", help="Use JP-Extra model",
) )
parser.add_argument( parser.add_argument(
"--val_per_lang", "--val_per_lang",

37
preprocess_text.py
View File

@@ -40,7 +40,6 @@ def count_lines(file_path: str):
@click.option("--clean/--no-clean", default=preprocess_text_config.clean) @click.option("--clean/--no-clean", default=preprocess_text_config.clean)
@click.option("-y", "--yml_config") @click.option("-y", "--yml_config")
@click.option("--use_jp_extra", is_flag=True) @click.option("--use_jp_extra", is_flag=True)
@click.option("--skip_invalid", is_flag=True)
def preprocess( def preprocess(
transcription_path: str, transcription_path: str,
cleaned_path: Optional[str], cleaned_path: Optional[str],
@@ -52,11 +51,13 @@ def preprocess(
clean: bool, clean: bool,
yml_config: str, # 这个不要删 yml_config: str, # 这个不要删
use_jp_extra: bool, use_jp_extra: bool,
skip_invalid: bool = False,
): ):
if cleaned_path == "" or cleaned_path is None: if cleaned_path == "" or cleaned_path is None:
cleaned_path = transcription_path + ".cleaned" cleaned_path = transcription_path + ".cleaned"
error_log_path = os.path.join(os.path.dirname(cleaned_path), "error.log")
error_count = 0
if clean: if clean:
total_lines = count_lines(transcription_path) total_lines = count_lines(transcription_path)
with open(cleaned_path, "w", encoding="utf-8") as out_file: with open(cleaned_path, "w", encoding="utf-8") as out_file:
@@ -79,18 +80,10 @@ def preprocess(
) )
) )
except Exception as e: except Exception as e:
if skip_invalid: logger.error(f"An error occurred at line:\n{line.strip()}\n{e}")
logger.warning( with open(error_log_path, "a", encoding="utf-8") as error_log:
f"An error occurred while generating the training set and validation set, at line:\n{line}\nDetails:\n{e}" error_log.write(f"{line.strip()}\n{e}\n\n")
) error_count += 1
logger.warning(
"This line will be skipped from the training set and validation set."
)
else:
logger.error(
f"An error occurred while generating the training set and validation set, at line:\n{line}\nDetails:\n{e}"
)
raise
transcription_path = cleaned_path transcription_path = cleaned_path
spk_utt_map = defaultdict(list) spk_utt_map = defaultdict(list)
@@ -118,9 +111,10 @@ def preprocess(
if spk not in spk_id_map.keys(): if spk not in spk_id_map.keys():
spk_id_map[spk] = current_sid spk_id_map[spk] = current_sid
current_sid += 1 current_sid += 1
logger.info( if countSame > 0 or countNotFound > 0:
f"Total repeated audios: {countSame}, Total number of audio not found: {countNotFound}" logger.warning(
) f"Total repeated audios: {countSame}, Total number of audio not found: {countNotFound}"
)
train_list = [] train_list = []
val_list = [] val_list = []
@@ -156,7 +150,14 @@ def preprocess(
) )
with open(config_path, "w", encoding="utf-8") as f: with open(config_path, "w", encoding="utf-8") as f:
json.dump(json_config, f, indent=2, ensure_ascii=False) json.dump(json_config, f, indent=2, ensure_ascii=False)
logger.info("Training set and validation set generation from texts is complete!") if error_count > 0:
logger.warning(
f"An error occurred in {error_count} lines. Please check {error_log_path} for details. You can proceed with lines that do not have errors."
)
else:
logger.info(
"Training set and validation set generation from texts is complete!"
)
if __name__ == "__main__": if __name__ == "__main__":