tuna2134/sbv2-v2
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Refactor: Use PathConfig and pathlib instead of paths.yml loading

Browse Source
This commit is contained in:
litagin02
2024-05-25 10:25:22 +09:00
parent 2771fbd209
commit 8976a3ed2f
22 changed files with 266 additions and 253 deletions
Download Patch File Download Diff File Expand all files Collapse all files

171
config.py
View File

@@ -2,9 +2,9 @@
@Desc: 全局配置文件读取
"""
import os
import shutil
from typing import Dict, List
from pathlib import Path
from typing import Any
import torch
import yaml
@@ -12,6 +12,12 @@ import yaml
from style_bert_vits2.logging import logger
class PathConfig:
def __init__(self, dataset_root: str, assets_root: str):
self.dataset_root = Path(dataset_root)
self.assets_root = Path(assets_root)
# If not cuda available, set possible devices to cpu
cuda_available = torch.cuda.is_available()
@@ -20,17 +26,17 @@ class Resample_config:
"""重采样配置"""
def __init__(self, in_dir: str, out_dir: str, sampling_rate: int = 44100):
self.sampling_rate: int = sampling_rate # 目标采样率
self.in_dir: str = in_dir # 待处理音频目录路径
self.out_dir: str = out_dir # 重采样输出路径
self.sampling_rate = sampling_rate # 目标采样率
self.in_dir = Path(in_dir) # 待处理音频目录路径
self.out_dir = Path(out_dir) # 重采样输出路径
@classmethod
def from_dict(cls, dataset_path: str, data: Dict[str, any]):
def from_dict(cls, dataset_path: Path, data: dict[str, Any]):
"""从字典中生成实例"""
# 不检查路径是否有效，此逻辑在resample.py中处理
data["in_dir"] = os.path.join(dataset_path, data["in_dir"])
data["out_dir"] = os.path.join(dataset_path, data["out_dir"])
data["in_dir"] = dataset_path / data["in_dir"]
data["out_dir"] = dataset_path / data["out_dir"]
return cls(**data)
@@ -49,39 +55,27 @@ class Preprocess_text_config:
max_val_total: int = 10000,
clean: bool = True,
):
self.transcription_path: str = (
transcription_path # 原始文本文件路径，文本格式应为{wav_path}|{speaker_name}|{language}|{text}。
)
self.cleaned_path: str = (
cleaned_path # 数据清洗后文本路径，可以不填。不填则将在原始文本目录生成
)
self.train_path: str = (
train_path # 训练集路径，可以不填。不填则将在原始文本目录生成
)
self.val_path: str = (
val_path # 验证集路径，可以不填。不填则将在原始文本目录生成
)
self.config_path: str = config_path # 配置文件路径
self.val_per_lang: int = val_per_lang # 每个speaker的验证集条数
self.max_val_total: int = (
max_val_total # 验证集最大条数，多于的会被截断并放到训练集中
)
self.clean: bool = clean # 是否进行数据清洗
self.transcription_path = Path(transcription_path)
self.cleaned_path = Path(cleaned_path)
self.train_path = Path(train_path)
self.val_path = Path(val_path)
self.config_path = Path(config_path)
self.val_per_lang = val_per_lang
self.max_val_total = max_val_total
self.clean = clean
@classmethod
def from_dict(cls, dataset_path: str, data: Dict[str, any]):
def from_dict(cls, dataset_path: Path, data: dict[str, Any]):
"""从字典中生成实例"""
data["transcription_path"] = os.path.join(
dataset_path, data["transcription_path"]
)
data["transcription_path"] = dataset_path / data["transcription_path"]
if data["cleaned_path"] == "" or data["cleaned_path"] is None:
data["cleaned_path"] = None
else:
data["cleaned_path"] = os.path.join(dataset_path, data["cleaned_path"])
data["train_path"] = os.path.join(dataset_path, data["train_path"])
data["val_path"] = os.path.join(dataset_path, data["val_path"])
data["config_path"] = os.path.join(dataset_path, data["config_path"])
data["cleaned_path"] = dataset_path / data["cleaned_path"]
data["train_path"] = dataset_path / data["train_path"]
data["val_path"] = dataset_path / data["val_path"]
data["config_path"] = dataset_path / data["config_path"]
return cls(**data)
@@ -96,7 +90,7 @@ class Bert_gen_config:
device: str = "cuda",
use_multi_device: bool = False,
):
self.config_path = config_path
self.config_path = Path(config_path)
self.num_processes = num_processes
if not cuda_available:
device = "cpu"
@@ -104,8 +98,8 @@ class Bert_gen_config:
self.use_multi_device = use_multi_device
@classmethod
def from_dict(cls, dataset_path: str, data: Dict[str, any]):
data["config_path"] = os.path.join(dataset_path, data["config_path"])
def from_dict(cls, dataset_path: Path, data: dict[str, Any]):
data["config_path"] = dataset_path / data["config_path"]
return cls(**data)
@@ -119,15 +113,15 @@ class Style_gen_config:
num_processes: int = 4,
device: str = "cuda",
):
self.config_path = config_path
self.config_path = Path(config_path)
self.num_processes = num_processes
if not cuda_available:
device = "cpu"
self.device = device
@classmethod
def from_dict(cls, dataset_path: str, data: Dict[str, any]):
data["config_path"] = os.path.join(dataset_path, data["config_path"])
def from_dict(cls, dataset_path: Path, data: dict[str, Any]):
data["config_path"] = dataset_path / data["config_path"]
return cls(**data)
@@ -138,7 +132,7 @@ class Train_ms_config:
def __init__(
self,
config_path: str,
env: Dict[str, any],
env: dict[str, Any],
# base: Dict[str, any],
model_dir: str,
num_workers: int,
@@ -147,16 +141,18 @@ class Train_ms_config:
):
self.env = env # 需要加载的环境变量
# self.base = base # 底模配置
self.model_dir = model_dir # 训练模型存储目录，该路径为相对于dataset_path的路径，而非项目根目录
self.config_path = config_path # 配置文件路径
self.model_dir = Path(
model_dir
) # 训练模型存储目录，该路径为相对于dataset_path的路径，而非项目根目录
self.config_path = Path(config_path) # 配置文件路径
self.num_workers = num_workers # worker数量
self.spec_cache = spec_cache # 是否启用spec缓存
self.keep_ckpts = keep_ckpts # ckpt数量
@classmethod
def from_dict(cls, dataset_path: str, data: Dict[str, any]):
def from_dict(cls, dataset_path: Path, data: dict[str, Any]):
# data["model"] = os.path.join(dataset_path, data["model"])
data["config_path"] = os.path.join(dataset_path, data["config_path"])
data["config_path"] = dataset_path / data["config_path"]
return cls(**data)
@@ -176,20 +172,18 @@ class Webui_config:
):
if not cuda_available:
device = "cpu"
self.device: str = device
self.model: str = model # 端口号
self.config_path: str = config_path # 是否公开部署，对外网开放
self.port: int = port # 是否开启debug模式
self.share: bool = share # 模型路径
self.debug: bool = debug # 配置文件路径
self.language_identification_library: str = (
language_identification_library # 语种识别库
)
self.device = device
self.model = Path(model)
self.config_path = Path(config_path)
self.port: int = port
self.share: bool = share
self.debug: bool = debug
self.language_identification_library: str = language_identification_library
@classmethod
def from_dict(cls, dataset_path: str, data: Dict[str, any]):
data["config_path"] = os.path.join(dataset_path, data["config_path"])
data["model"] = os.path.join(dataset_path, data["model"])
def from_dict(cls, dataset_path: Path, data: dict[str, Any]):
data["config_path"] = dataset_path / data["config_path"]
data["model"] = dataset_path / data["model"]
return cls(**data)
@@ -200,7 +194,7 @@ class Server_config:
device: str = "cuda",
limit: int = 100,
language: str = "JP",
origins: List[str] = None,
origins: list[str] = ["*"],
):
self.port: int = port
if not cuda_available:
@@ -208,10 +202,10 @@ class Server_config:
self.device: str = device
self.language: str = language
self.limit: int = limit
self.origins: List[str] = origins
self.origins: list[str] = origins
@classmethod
def from_dict(cls, data: Dict[str, any]):
def from_dict(cls, data: dict[str, Any]):
return cls(**data)
@@ -223,32 +217,32 @@ class Translate_config:
self.secret_key = secret_key
@classmethod
def from_dict(cls, data: Dict[str, any]):
def from_dict(cls, data: dict[str, Any]):
return cls(**data)
class Config:
def __init__(self, config_path: str, path_config: dict[str, str]):
if not os.path.isfile(config_path) and os.path.isfile("default_config.yml"):
def __init__(self, config_path: str, path_config: PathConfig):
if not Path(config_path).exists():
shutil.copy(src="default_config.yml", dst=config_path)
logger.info(
f"A configuration file {config_path} has been generated based on the default configuration file default_config.yml."
)
logger.info(
"If you have no special needs, please do not modify default_config.yml."
"Please do not modify default_config.yml. Instead, modify config.yml."
)
# sys.exit(0)
with open(config_path, "r", encoding="utf-8") as file:
yaml_config: Dict[str, any] = yaml.safe_load(file.read())
yaml_config: dict[str, Any] = yaml.safe_load(file.read())
model_name: str = yaml_config["model_name"]
self.model_name: str = model_name
if "dataset_path" in yaml_config:
dataset_path = yaml_config["dataset_path"]
dataset_path = Path(yaml_config["dataset_path"])
else:
dataset_path = os.path.join(path_config["dataset_root"], model_name)
self.dataset_path: str = dataset_path
self.assets_root: str = path_config["assets_root"]
self.out_dir = os.path.join(self.assets_root, model_name)
dataset_path = path_config.dataset_root / model_name
self.dataset_path = dataset_path
self.assets_root = path_config.assets_root
self.out_dir = self.assets_root / model_name
self.resample_config: Resample_config = Resample_config.from_dict(
dataset_path, yaml_config["resample"]
)
@@ -277,16 +271,31 @@ class Config:
# )
with open(os.path.join("configs", "paths.yml"), "r", encoding="utf-8") as f:
path_config: dict[str, str] = yaml.safe_load(f.read())
# Should contain the following keys:
# - dataset_root: the root directory of the dataset, default to "Data"
# - assets_root: the root directory of the assets, default to "model_assets"
# Load and initialize the configuration
try:
config = Config("config.yml", path_config)
except (TypeError, KeyError):
logger.warning("Old config.yml found. Replace it with default_config.yml.")
shutil.copy(src="default_config.yml", dst="config.yml")
config = Config("config.yml", path_config)
def get_path_config() -> PathConfig:
path_config_path = Path("configs/paths.yml")
if not path_config_path.exists():
shutil.copy(src="configs/default_paths.yml", dst=path_config_path)
logger.info(
f"A configuration file {path_config_path} has been generated based on the default configuration file default_paths.yml."
)
logger.info(
"Please do not modify configs/default_paths.yml. Instead, modify configs/paths.yml."
)
with open(path_config_path, "r", encoding="utf-8") as file:
path_config_dict: dict[str, str] = yaml.safe_load(file.read())
return PathConfig(**path_config_dict)
def get_config() -> Config:
path_config = get_path_config()
try:
config = Config("config.yml", path_config)
except (TypeError, KeyError):
logger.warning("Old config.yml found. Replace it with default_config.yml.")
shutil.copy(src="default_config.yml", dst="config.yml")
config = Config("config.yml", path_config)
return config