修复多机训练问题 (#143)

* 修复多机训练问题

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.gitignore

@@ -166,7 +166,8 @@ cython_debug/
 filelists/*
 !/filelists/esd.list
 data/*
-/config.yml
+/*.yml
+!/default_config.yml
 /Web/
 /emotional/*/*.bin
 /bert/*/*.bin
@@ -176,5 +177,9 @@ data/*
 /bert/*/*.msgpack
 asr_transcript.py
 extract_list.py
+dataset
 /Data
+Model
+raw/
+logs/
 Data/*

run_Mgpus.sh

@@ -0,0 +1,7 @@
+torchrun \
+    --nnodes=1:3\
+    --nproc_per_node=2\
+    --rdzv_id=1\
+    --rdzv_backend=c10d\
+    --rdzv_endpoint="ib1:8880"\
+    train_ms.py

train_ms.py

@@ -62,6 +62,7 @@ def run():
         init_method="env://",  # If Windows,switch to gloo backend.
     )  # Use torchrun instead of mp.spawn
     rank = dist.get_rank()
+    local_rank = int(os.environ["LOCAL_RANK"])
     n_gpus = dist.get_world_size()
 
     # 命令行/config.yml配置解析
@@ -99,7 +100,8 @@ def run():
             f.write(data)
 
     torch.manual_seed(hps.train.seed)
-    torch.cuda.set_device(rank)
+    torch.cuda.set_device(local_rank)
+
     global global_step
     if rank == 0:
         logger = utils.get_logger(hps.model_dir)