From 7dd803a6a89eeb5c1d3e10c5fcf2c5828fdbcc51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stardust=C2=B7=E5=87=8F?= Date: Sun, 3 Sep 2023 00:57:17 +0800 Subject: [PATCH] fix --- bert_gen.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bert_gen.py b/bert_gen.py index 1087095..193e768 100644 --- a/bert_gen.py +++ b/bert_gen.py @@ -42,12 +42,12 @@ def process_line(line): if __name__ == '__main__': lines = [] - with open(hps.data.training_files) as f: + with open(hps.data.training_files, encoding='utf-8' ) as f: lines.extend(f.readlines()) - with open(hps.data.validation_files) as f: + with open(hps.data.validation_files, encoding='utf-8' ) as f: lines.extend(f.readlines()) - with Pool(processes=12) as pool: #A100 suitable config,if coom,please decrease the processess number. + with Pool(processes=12) as pool: #A100 40GB suitable config,if coom,please decrease the processess number. for _ in tqdm(pool.imap_unordered(process_line, lines)): pass