tuna2134/sbv2-v2
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Remove read_lines() for large file

Browse Source
This commit is contained in:
litagin02
2024-01-14 07:25:51 +09:00
parent 64047195b5
commit 972ec0e627
1 changed files with 19 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
preprocess_text.py
View File

@@ -50,31 +50,26 @@ def preprocess(
if clean: if clean:
with open(cleaned_path, "w", encoding="utf-8") as out_file: with open(cleaned_path, "w", encoding="utf-8") as out_file:
with open(transcription_path, "r", encoding="utf-8") as trans_file: with open(transcription_path, "r", encoding="utf-8") as trans_file:
lines = trans_file.readlines() for line in tqdm(trans_file, file=SAFE_STDOUT):
# print(lines, ' ', len(lines)) try:
if len(lines) != 0: utt, spk, language, text = line.strip().split("|")
for line in tqdm(lines, file=SAFE_STDOUT): norm_text, phones, tones, word2ph = clean_text(text, language)
try: out_file.write(
utt, spk, language, text = line.strip().split("|") "{}|{}|{}|{}|{}|{}|{}\n".format(
norm_text, phones, tones, word2ph = clean_text( utt,
text, language spk,
language,
norm_text,
" ".join(phones),
" ".join([str(i) for i in tones]),
" ".join([str(i) for i in word2ph]),
) )
out_file.write( )
"{}|{}|{}|{}|{}|{}|{}\n".format( except Exception as e:
utt, logger.error(
spk, f"An error occurred while generating the training set and validation set, at line:\n{line}\nDetails:\n{e}"
language, )
norm_text, raise
" ".join(phones),
" ".join([str(i) for i in tones]),
" ".join([str(i) for i in word2ph]),
)
)
except Exception as e:
logger.error(
f"An error occurred while generating the training set and validation set, at line:\n{line}\nDetails:\n{e}"
)
raise
transcription_path = cleaned_path transcription_path = cleaned_path
spk_utt_map = defaultdict(list) spk_utt_map = defaultdict(list)