From 432689e29fdca225ceb1254a6c867b8f2c05ebd1 Mon Sep 17 00:00:00 2001 From: litagin02 Date: Sat, 6 Jan 2024 13:07:57 +0900 Subject: [PATCH] Add preprocess log and adjust log files --- common/subprocess_utils.py | 6 +- train_ms.py | 114 ++++++++++++++++++++----------------- webui_train.py | 32 ++++++++--- 3 files changed, 88 insertions(+), 64 deletions(-) diff --git a/common/subprocess_utils.py b/common/subprocess_utils.py index cbb357b..9a0e323 100644 --- a/common/subprocess_utils.py +++ b/common/subprocess_utils.py @@ -16,12 +16,10 @@ def run_script_with_log(cmd: list[str], ignore_warning=False) -> tuple[bool, str text=True, ) if result.returncode != 0: - logger.error(f"Error: {' '.join(cmd)}") - print(result.stderr) + logger.error(f"Error: {' '.join(cmd)}\n{result.stderr}") return False, result.stderr elif result.stderr and not ignore_warning: - logger.warning(f"Warning: {' '.join(cmd)}") - print(result.stderr) + logger.warning(f"Warning: {' '.join(cmd)}\n{result.stderr}") return True, result.stderr logger.success(f"Success: {' '.join(cmd)}") return True, "" diff --git a/train_ms.py b/train_ms.py index cbf6d72..b3259e4 100644 --- a/train_ms.py +++ b/train_ms.py @@ -3,8 +3,6 @@ import datetime import gc import os import platform -import shutil -import sys import torch import torch.distributed as dist @@ -49,14 +47,53 @@ global_step = 0 def run(): - # 环境变量解析 + # Command line configuration is not recommended unless necessary, use config.yml + parser = argparse.ArgumentParser() + parser.add_argument( + "-c", + "--config", + type=str, + default=config.train_ms_config.config_path, + help="JSON file for configuration", + ) + parser.add_argument( + "-m", + "--model", + type=str, + help="数据集文件夹路径，请注意，数据不再默认放在/logs文件夹下。如果需要用命令行配置，请声明相对于根目录的路径", + default=config.dataset_path, + ) + parser.add_argument( + "--assets_root", + type=str, + help="Root directory of model assets needed for inference.", + default=config.assets_root, + ) + parser.add_argument( + "--skip_default_style", + action="store_true", + help="Skip saving default style config and mean vector.", + ) + parser.add_argument( + "--no_progress_bar", + action="store_true", + help="Do not show the progress bar while training.", + ) + args = parser.parse_args() + + # Set log file + model_dir = os.path.join(args.model, config.train_ms_config.model_dir) + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + logger.add(os.path.join(args.model, f"train_{timestamp}.log")) + + # Parsing environment variables envs = config.train_ms_config.env for env_name, env_value in envs.items(): if env_name not in os.environ.keys(): - logger.info("加载config中的配置{}".format(str(env_value))) + logger.info("Loading configuration from config {}".format(str(env_value))) os.environ[env_name] = str(env_value) logger.info( - "加载环境变量 \nMASTER_ADDR: {},\nMASTER_PORT: {},\nWORLD_SIZE: {},\nRANK: {},\nLOCAL_RANK: {}".format( + "Loading environment variables \nMASTER_ADDR: {},\nMASTER_PORT: {},\nWORLD_SIZE: {},\nRANK: {},\nLOCAL_RANK: {}".format( os.environ["MASTER_ADDR"], os.environ["MASTER_PORT"], os.environ["WORLD_SIZE"], @@ -77,47 +114,7 @@ def run(): local_rank = int(os.environ["LOCAL_RANK"]) n_gpus = dist.get_world_size() - # 命令行/config.yml配置解析 - # hps = utils.get_hparams() - parser = argparse.ArgumentParser() - # Command line configuration is not recommended unless necessary, use config.yml - parser.add_argument( - "-c", - "--config", - type=str, - default=config.train_ms_config.config_path, - help="JSON file for configuration", - ) - - parser.add_argument( - "-m", - "--model", - type=str, - help="数据集文件夹路径，请注意，数据不再默认放在/logs文件夹下。如果需要用命令行配置，请声明相对于根目录的路径", - default=config.dataset_path, - ) - parser.add_argument( - "--assets_root", - type=str, - help="Root directory of model assets needed for inference.", - default=config.assets_root, - ) - parser.add_argument( - "--skip_default_style", - action="store_true", - help="Skip saving default style config and mean vector.", - ) - parser.add_argument( - '--no_progress_bar', - action='store_true', - help='Do not show the progress bar while training.' - ) - args = parser.parse_args() - model_dir = os.path.join(args.model, config.train_ms_config.model_dir) - if not os.path.exists(model_dir): - os.makedirs(model_dir) hps = utils.get_hparams_from_file(args.config) - # This is needed because we have to pass `model_dir` to `train_and_evaluate()` hps.model_dir = model_dir @@ -167,7 +164,6 @@ def run(): if rank == 0: # logger = utils.get_logger(hps.model_dir) # logger.info(hps) - logger.add(os.path.join(model_dir, "train.log")) utils.check_git_hash(model_dir) writer = SummaryWriter(log_dir=model_dir) writer_eval = SummaryWriter(log_dir=os.path.join(model_dir, "eval")) @@ -371,10 +367,17 @@ def run(): scaler = GradScaler(enabled=hps.train.bf16_run) logger.info("Start training.") - diff = abs(epoch_str * len(train_loader) - (hps.train.epochs + 1) * len(train_loader)) + diff = abs( + epoch_str * len(train_loader) - (hps.train.epochs + 1) * len(train_loader) + ) pbar = None if not args.no_progress_bar: - pbar = tqdm(total=global_step + diff, initial=global_step, smoothing=0.05, file=SAFE_STDOUT) + pbar = tqdm( + total=global_step + diff, + initial=global_step, + smoothing=0.05, + file=SAFE_STDOUT, + ) initial_step = global_step for epoch in range(epoch_str, hps.train.epochs + 1): @@ -466,7 +469,7 @@ def train_and_evaluate( logger, writers, pbar: tqdm, - initial_step: int + initial_step: int, ): net_g, net_d, net_dur_disc = nets optim_g, optim_d, optim_dur_disc = optims @@ -684,7 +687,11 @@ def train_and_evaluate( scalars=scalar_dict, ) - if global_step % hps.train.eval_interval == 0 and global_step != 0 and initial_step != global_step: + if ( + global_step % hps.train.eval_interval == 0 + and global_step != 0 + and initial_step != global_step + ): evaluate(hps, net_g, eval_loader, writer_eval) utils.save_checkpoint( net_g, @@ -728,7 +735,11 @@ def train_and_evaluate( global_step += 1 if pbar is not None: - pbar.set_description("Epoch {}({:.0f}%)/{}".format(epoch, 100.0 * batch_idx / len(train_loader), hps.train.epochs)) + pbar.set_description( + "Epoch {}({:.0f}%)/{}".format( + epoch, 100.0 * batch_idx / len(train_loader), hps.train.epochs + ) + ) pbar.update() # 本家ではこれをスピードアップのために消すと書かれていたので、一応消してみる # gc.collect() @@ -736,6 +747,7 @@ def train_and_evaluate( if pbar is None and rank == 0: logger.info(f"====> Epoch: {epoch}, step: {global_step}") + def evaluate(hps, generator, eval_loader, writer_eval): generator.eval() image_dict = {} diff --git a/webui_train.py b/webui_train.py index 5fa450f..8a2c7c4 100644 --- a/webui_train.py +++ b/webui_train.py @@ -8,6 +8,10 @@ import yaml from common.log import logger from common.subprocess_utils import run_script_with_log, second_elem_of +from datetime import datetime + + +logger_handler = None # Get path settings with open(os.path.join("configs", "paths.yml"), "r", encoding="utf-8") as f: @@ -27,8 +31,19 @@ def get_path(model_name): def initialize(model_name, batch_size, epochs, save_every_steps, bf16_run): - logger.info("Step 1: start initialization...") + global logger_handler dataset_path, _, train_path, val_path, config_path = get_path(model_name) + + # 前処理のログをファイルに保存する + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + file_name = f"preprocess_{timestamp}.log" + if logger_handler is not None: + logger.remove(logger_handler) + logger_handler = logger.add(os.path.join(dataset_path, file_name)) + + logger.info( + f"Step 1: start initialization...\nmodel_name: {model_name}, batch_size: {batch_size}, epochs: {epochs}, save_every_steps: {save_every_steps}, bf16_run: {bf16_run}" + ) if os.path.isfile(config_path): with open(config_path, "r", encoding="utf-8") as f: config = json.load(f) @@ -230,13 +245,13 @@ def train(model_name): ["train_ms.py", "--config", config_path, "--model", dataset_path] ) if not success: - logger.error(f"Final Step: train failed.") - return False, f"Final Step, Error: 学習に失敗しました:\n{message}" + logger.error(f"Train failed.") + return False, f"Error: 学習に失敗しました:\n{message}" elif message: - logger.warning(f"Final Step: train finished with stderr.") - return True, f"Final Step, Success: 学習が完了しました:\n{message}" - logger.success("Final Step: train finished.") - return True, "Final Step, Success: 学習が完了しました" + logger.warning(f"Train finished with stderr.") + return True, f"Success: 学習が完了しました:\n{message}" + logger.success("Train finished.") + return True, "Success: 学習が完了しました" initial_md = """ @@ -299,7 +314,6 @@ if __name__ == "__main__": with gr.Column(): batch_size = gr.Slider( label="バッチサイズ", - info="VRAM 12GBで4くらい", value=4, minimum=1, maximum=64, @@ -335,7 +349,7 @@ if __name__ == "__main__": step=1, ) normalize = gr.Checkbox( - label="音声の音量を正規化する", + label="音声の音量を正規化する(音量の大小が揃っていない場合など)", value=False, ) trim = gr.Checkbox(