From 57b408e8df115f87e94249b4ef7cb6ee37f781d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stardust=C2=B7=E5=87=8F?= Date: Wed, 6 Sep 2023 23:00:33 +0800 Subject: [PATCH 1/2] rollback TF32 --- train_ms.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/train_ms.py b/train_ms.py index 7619237..4907c69 100644 --- a/train_ms.py +++ b/train_ms.py @@ -33,7 +33,9 @@ from mel_processing import mel_spectrogram_torch, spec_to_mel_torch from text.symbols import symbols os.environ['NCCL_P2P_DISABLE'] = 1 # Avoid training problem. - +torch.backends.cuda.matmul.allow_tf32 = True +torch.backends.cudnn.allow_tf32 = True # If encontered training problem,please try to disable TF32. +torch.set_float32_matmul_precision('medium') torch.backends.cudnn.benchmark = True torch.backends.cuda.sdp_kernel("flash") torch.backends.cuda.enable_flash_sdp(True) From c80621c7aec81a40c639b702cfa2ad69118f3482 Mon Sep 17 00:00:00 2001 From: Stardust-minus Date: Wed, 6 Sep 2023 15:00:48 +0000 Subject: [PATCH 2/2] Apply Code Formatter Change --- train_ms.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/train_ms.py b/train_ms.py index 4907c69..1277eae 100644 --- a/train_ms.py +++ b/train_ms.py @@ -32,10 +32,12 @@ from losses import generator_loss, discriminator_loss, feature_loss, kl_loss from mel_processing import mel_spectrogram_torch, spec_to_mel_torch from text.symbols import symbols -os.environ['NCCL_P2P_DISABLE'] = 1 # Avoid training problem. +os.environ["NCCL_P2P_DISABLE"] = 1 # Avoid training problem. torch.backends.cuda.matmul.allow_tf32 = True -torch.backends.cudnn.allow_tf32 = True # If encontered training problem,please try to disable TF32. -torch.set_float32_matmul_precision('medium') +torch.backends.cudnn.allow_tf32 = ( + True # If encontered training problem,please try to disable TF32. +) +torch.set_float32_matmul_precision("medium") torch.backends.cudnn.benchmark = True torch.backends.cuda.sdp_kernel("flash") torch.backends.cuda.enable_flash_sdp(True)