tuna2134/sbv2-v2
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Update bert_gen.py

Browse Source
This commit is contained in:
Stardust·减
2023-09-05 19:46:39 +08:00
committed by GitHub
parent 9d2ed39365
commit 70b6e3bb9a
1 changed files with 10 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
bert_gen.py
View File

@@ -5,10 +5,7 @@ import utils
from tqdm import tqdm from tqdm import tqdm
from loguru import logger from loguru import logger
from text import cleaned_text_to_sequence, get_bert from text import cleaned_text_to_sequence, get_bert
import argparse
config_path = "configs/config.json"
hps = utils.get_hparams_from_file(config_path)
total_gpus = torch.cuda.device_count()
def process_line(line): def process_line(line):
@@ -45,6 +42,9 @@ def process_line(line):
if __name__ == "__main__": if __name__ == "__main__":
parser.add_argument('-c', '--config', type=str, default=None)
config_path = args.config
hps = utils.get_hparams_from_file(config_path)
lines = [] lines = []
with open(hps.data.training_files, encoding="utf-8") as f: with open(hps.data.training_files, encoding="utf-8") as f:
lines.extend(f.readlines()) lines.extend(f.readlines())
@@ -52,7 +52,11 @@ if __name__ == "__main__":
with open(hps.data.validation_files, encoding="utf-8") as f: with open(hps.data.validation_files, encoding="utf-8") as f:
lines.extend(f.readlines()) lines.extend(f.readlines())
# RTX4090 24GB suitable config, if coom, please decrease the processes number. parser.add_argument(
with Pool(processes=6) as pool: '--num_processes', type=int, default=1, help='You are advised to set the number of processes to the same as the number of CPU cores'
)
num_processes = args.num_processes
with Pool(processes=num_processes) as pool:
for _ in tqdm(pool.imap_unordered(process_line, lines), total=len(lines)): for _ in tqdm(pool.imap_unordered(process_line, lines), total=len(lines)):
pass pass