tuna2134/sbv2-v2
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
Stardust·减
2023-09-03 00:57:17 +08:00
committed by GitHub
parent d81876c456
commit 7dd803a6a8
1 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
bert_gen.py
View File

@@ -42,12 +42,12 @@ def process_line(line):
if __name__ == '__main__':
lines = []
with open(hps.data.training_files) as f:
with open(hps.data.training_files, encoding='utf-8' ) as f:
lines.extend(f.readlines())
with open(hps.data.validation_files) as f:
with open(hps.data.validation_files, encoding='utf-8' ) as f:
lines.extend(f.readlines())
with Pool(processes=12) as pool: #A100 suitable config,if coom,please decrease the processess number.
with Pool(processes=12) as pool: #A100 40GB suitable config,if coom,please decrease the processess number.
for _ in tqdm(pool.imap_unordered(process_line, lines)):
pass