From 74467f5f73085e8cab7cb4d6abd7ec45c439ae4d Mon Sep 17 00:00:00 2001
From: Lvjinhong <96970081+Lvjinhong@users.noreply.github.com>
Date: Thu, 2 Nov 2023 10:38:45 +0800
Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=A4=9A=E6=9C=BA=E8=AE=AD?=
 =?UTF-8?q?=E7=BB=83=E9=97=AE=E9=A2=98=20(#143)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* 修复多机训练问题

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
---
 .gitignore   | 7 ++++++-
 run_Mgpus.sh | 7 +++++++
 train_ms.py  | 4 +++-
 3 files changed, 16 insertions(+), 2 deletions(-)
 create mode 100644 run_Mgpus.sh

diff --git a/.gitignore b/.gitignore
index 7b7714e..4404cd5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -166,7 +166,8 @@ cython_debug/
 filelists/*
 !/filelists/esd.list
 data/*
-/config.yml
+/*.yml
+!/default_config.yml
 /Web/
 /emotional/*/*.bin
 /bert/*/*.bin
@@ -176,5 +177,9 @@ data/*
 /bert/*/*.msgpack
 asr_transcript.py
 extract_list.py
+dataset
 /Data
+Model
+raw/
+logs/
 Data/*
diff --git a/run_Mgpus.sh b/run_Mgpus.sh
new file mode 100644
index 0000000..c43974c
--- /dev/null
+++ b/run_Mgpus.sh
@@ -0,0 +1,7 @@
+torchrun \
+    --nnodes=1:3\
+    --nproc_per_node=2\
+    --rdzv_id=1\
+    --rdzv_backend=c10d\
+    --rdzv_endpoint="ib1:8880"\
+    train_ms.py
diff --git a/train_ms.py b/train_ms.py
index 8be0e8e..a99fb2d 100644
--- a/train_ms.py
+++ b/train_ms.py
@@ -62,6 +62,7 @@ def run():
         init_method="env://",  # If Windows,switch to gloo backend.
     )  # Use torchrun instead of mp.spawn
     rank = dist.get_rank()
+    local_rank = int(os.environ["LOCAL_RANK"])
     n_gpus = dist.get_world_size()
 
     # 命令行/config.yml配置解析
@@ -99,7 +100,8 @@ def run():
             f.write(data)
 
     torch.manual_seed(hps.train.seed)
-    torch.cuda.set_device(rank)
+    torch.cuda.set_device(local_rank)
+
     global global_step
     if rank == 0:
         logger = utils.get_logger(hps.model_dir)