diff --git a/.gitignore b/.gitignore index 7b7714e..4404cd5 100644 --- a/.gitignore +++ b/.gitignore @@ -166,7 +166,8 @@ cython_debug/ filelists/* !/filelists/esd.list data/* -/config.yml +/*.yml +!/default_config.yml /Web/ /emotional/*/*.bin /bert/*/*.bin @@ -176,5 +177,9 @@ data/* /bert/*/*.msgpack asr_transcript.py extract_list.py +dataset /Data +Model +raw/ +logs/ Data/* diff --git a/run_Mgpus.sh b/run_Mgpus.sh new file mode 100644 index 0000000..c43974c --- /dev/null +++ b/run_Mgpus.sh @@ -0,0 +1,7 @@ +torchrun \ + --nnodes=1:3\ + --nproc_per_node=2\ + --rdzv_id=1\ + --rdzv_backend=c10d\ + --rdzv_endpoint="ib1:8880"\ + train_ms.py diff --git a/train_ms.py b/train_ms.py index 8be0e8e..a99fb2d 100644 --- a/train_ms.py +++ b/train_ms.py @@ -62,6 +62,7 @@ def run(): init_method="env://", # If Windows,switch to gloo backend. ) # Use torchrun instead of mp.spawn rank = dist.get_rank() + local_rank = int(os.environ["LOCAL_RANK"]) n_gpus = dist.get_world_size() # 命令行/config.yml配置解析 @@ -99,7 +100,8 @@ def run(): f.write(data) torch.manual_seed(hps.train.seed) - torch.cuda.set_device(rank) + torch.cuda.set_device(local_rank) + global global_step if rank == 0: logger = utils.get_logger(hps.model_dir)