tuna2134/sbv2-v2
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Remove read_lines() for large file

Browse Source
This commit is contained in:
litagin02
2024-01-14 07:25:51 +09:00
parent 64047195b5
commit 972ec0e627
1 changed files with 19 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
preprocess_text.py
View File

@@ -50,15 +50,10 @@ def preprocess(
if clean: if clean:
with open(cleaned_path, "w", encoding="utf-8") as out_file: with open(cleaned_path, "w", encoding="utf-8") as out_file:
with open(transcription_path, "r", encoding="utf-8") as trans_file: with open(transcription_path, "r", encoding="utf-8") as trans_file:
lines = trans_file.readlines() for line in tqdm(trans_file, file=SAFE_STDOUT):
# print(lines, ' ', len(lines))
if len(lines) != 0:
for line in tqdm(lines, file=SAFE_STDOUT):
try: try:
utt, spk, language, text = line.strip().split("|") utt, spk, language, text = line.strip().split("|")
norm_text, phones, tones, word2ph = clean_text( norm_text, phones, tones, word2ph = clean_text(text, language)
text, language
)
out_file.write( out_file.write(
"{}|{}|{}|{}|{}|{}|{}\n".format( "{}|{}|{}|{}|{}|{}|{}\n".format(
utt, utt,