From 57b408e8df115f87e94249b4ef7cb6ee37f781d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stardust=C2=B7=E5=87=8F?= Date: Wed, 6 Sep 2023 23:00:33 +0800 Subject: [PATCH] rollback TF32 --- train_ms.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/train_ms.py b/train_ms.py index 7619237..4907c69 100644 --- a/train_ms.py +++ b/train_ms.py @@ -33,7 +33,9 @@ from mel_processing import mel_spectrogram_torch, spec_to_mel_torch from text.symbols import symbols os.environ['NCCL_P2P_DISABLE'] = 1 # Avoid training problem. - +torch.backends.cuda.matmul.allow_tf32 = True +torch.backends.cudnn.allow_tf32 = True # If encontered training problem,please try to disable TF32. +torch.set_float32_matmul_precision('medium') torch.backends.cudnn.benchmark = True torch.backends.cuda.sdp_kernel("flash") torch.backends.cuda.enable_flash_sdp(True)