Refactor: moved text/cleaner.py to style_bert_vits2/text_processing/
This commit is contained in:
tsukumi
@@ -7,10 +7,10 @@ from typing import Optional
|
||||
import click
|
||||
from tqdm import tqdm
|
||||
|
||||
from style_bert_vits2.logging import logger
|
||||
from style_bert_vits2.utils.stdout_wrapper import SAFE_STDOUT
|
||||
from config import config
|
||||
from text.cleaner import clean_text
|
||||
from style_bert_vits2.logging import logger
|
||||
from style_bert_vits2.text_processing.cleaner import clean_text
|
||||
from style_bert_vits2.utils.stdout_wrapper import SAFE_STDOUT
|
||||
|
||||
preprocess_text_config = config.preprocess_text_config
|
||||
|
||||
@@ -72,7 +72,7 @@ def preprocess(
|
||||
utt, spk, language, text = line.strip().split("|")
|
||||
norm_text, phones, tones, word2ph = clean_text(
|
||||
text=text,
|
||||
language=language,
|
||||
language=language, # type: ignore
|
||||
use_jp_extra=use_jp_extra,
|
||||
raise_yomi_error=(yomi_error != "use"),
|
||||
)
|
||||
|
||||
@@ -1,314 +1,319 @@
|
||||
import torch
|
||||
|
||||
import utils
|
||||
from text import cleaned_text_to_sequence, get_bert
|
||||
from text.cleaner import clean_text
|
||||
from style_bert_vits2.logging import logger
|
||||
from style_bert_vits2.models import commons
|
||||
from style_bert_vits2.models.models import SynthesizerTrn
|
||||
from style_bert_vits2.models.models_jp_extra import SynthesizerTrn as SynthesizerTrnJPExtra
|
||||
from style_bert_vits2.text_processing.symbols import SYMBOLS
|
||||
|
||||
|
||||
class InvalidToneError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def get_net_g(model_path: str, version: str, device: str, hps):
|
||||
if version.endswith("JP-Extra"):
|
||||
logger.info("Using JP-Extra model")
|
||||
net_g = SynthesizerTrnJPExtra(
|
||||
len(SYMBOLS),
|
||||
hps.data.filter_length // 2 + 1,
|
||||
hps.train.segment_size // hps.data.hop_length,
|
||||
n_speakers=hps.data.n_speakers,
|
||||
**hps.model,
|
||||
).to(device)
|
||||
else:
|
||||
logger.info("Using normal model")
|
||||
net_g = SynthesizerTrn(
|
||||
len(SYMBOLS),
|
||||
hps.data.filter_length // 2 + 1,
|
||||
hps.train.segment_size // hps.data.hop_length,
|
||||
n_speakers=hps.data.n_speakers,
|
||||
**hps.model,
|
||||
).to(device)
|
||||
net_g.state_dict()
|
||||
_ = net_g.eval()
|
||||
if model_path.endswith(".pth") or model_path.endswith(".pt"):
|
||||
_ = utils.load_checkpoint(model_path, net_g, None, skip_optimizer=True)
|
||||
elif model_path.endswith(".safetensors"):
|
||||
_ = utils.load_safetensors(model_path, net_g, True)
|
||||
else:
|
||||
raise ValueError(f"Unknown model format: {model_path}")
|
||||
return net_g
|
||||
|
||||
|
||||
def get_text(
|
||||
text,
|
||||
language_str,
|
||||
hps,
|
||||
device,
|
||||
assist_text=None,
|
||||
assist_text_weight=0.7,
|
||||
given_tone=None,
|
||||
):
|
||||
use_jp_extra = hps.version.endswith("JP-Extra")
|
||||
# 推論のときにのみ呼び出されるので、raise_yomi_errorはFalseに設定
|
||||
norm_text, phone, tone, word2ph = clean_text(
|
||||
text, language_str, use_jp_extra, raise_yomi_error=False
|
||||
)
|
||||
if given_tone is not None:
|
||||
if len(given_tone) != len(phone):
|
||||
raise InvalidToneError(
|
||||
f"Length of given_tone ({len(given_tone)}) != length of phone ({len(phone)})"
|
||||
)
|
||||
tone = given_tone
|
||||
phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
|
||||
|
||||
if hps.data.add_blank:
|
||||
phone = commons.intersperse(phone, 0)
|
||||
tone = commons.intersperse(tone, 0)
|
||||
language = commons.intersperse(language, 0)
|
||||
for i in range(len(word2ph)):
|
||||
word2ph[i] = word2ph[i] * 2
|
||||
word2ph[0] += 1
|
||||
bert_ori = get_bert(
|
||||
norm_text,
|
||||
word2ph,
|
||||
language_str,
|
||||
device,
|
||||
assist_text,
|
||||
assist_text_weight,
|
||||
)
|
||||
del word2ph
|
||||
assert bert_ori.shape[-1] == len(phone), phone
|
||||
|
||||
if language_str == "ZH":
|
||||
bert = bert_ori
|
||||
ja_bert = torch.zeros(1024, len(phone))
|
||||
en_bert = torch.zeros(1024, len(phone))
|
||||
elif language_str == "JP":
|
||||
bert = torch.zeros(1024, len(phone))
|
||||
ja_bert = bert_ori
|
||||
en_bert = torch.zeros(1024, len(phone))
|
||||
elif language_str == "EN":
|
||||
bert = torch.zeros(1024, len(phone))
|
||||
ja_bert = torch.zeros(1024, len(phone))
|
||||
en_bert = bert_ori
|
||||
else:
|
||||
raise ValueError("language_str should be ZH, JP or EN")
|
||||
|
||||
assert bert.shape[-1] == len(
|
||||
phone
|
||||
), f"Bert seq len {bert.shape[-1]} != {len(phone)}"
|
||||
|
||||
phone = torch.LongTensor(phone)
|
||||
tone = torch.LongTensor(tone)
|
||||
language = torch.LongTensor(language)
|
||||
return bert, ja_bert, en_bert, phone, tone, language
|
||||
|
||||
|
||||
def infer(
|
||||
text,
|
||||
style_vec,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
sid: int, # In the original Bert-VITS2, its speaker_name: str, but here it's id
|
||||
language,
|
||||
hps,
|
||||
net_g,
|
||||
device,
|
||||
skip_start=False,
|
||||
skip_end=False,
|
||||
assist_text=None,
|
||||
assist_text_weight=0.7,
|
||||
given_tone=None,
|
||||
):
|
||||
is_jp_extra = hps.version.endswith("JP-Extra")
|
||||
bert, ja_bert, en_bert, phones, tones, lang_ids = get_text(
|
||||
text,
|
||||
language,
|
||||
hps,
|
||||
device,
|
||||
assist_text=assist_text,
|
||||
assist_text_weight=assist_text_weight,
|
||||
given_tone=given_tone,
|
||||
)
|
||||
if skip_start:
|
||||
phones = phones[3:]
|
||||
tones = tones[3:]
|
||||
lang_ids = lang_ids[3:]
|
||||
bert = bert[:, 3:]
|
||||
ja_bert = ja_bert[:, 3:]
|
||||
en_bert = en_bert[:, 3:]
|
||||
if skip_end:
|
||||
phones = phones[:-2]
|
||||
tones = tones[:-2]
|
||||
lang_ids = lang_ids[:-2]
|
||||
bert = bert[:, :-2]
|
||||
ja_bert = ja_bert[:, :-2]
|
||||
en_bert = en_bert[:, :-2]
|
||||
with torch.no_grad():
|
||||
x_tst = phones.to(device).unsqueeze(0)
|
||||
tones = tones.to(device).unsqueeze(0)
|
||||
lang_ids = lang_ids.to(device).unsqueeze(0)
|
||||
bert = bert.to(device).unsqueeze(0)
|
||||
ja_bert = ja_bert.to(device).unsqueeze(0)
|
||||
en_bert = en_bert.to(device).unsqueeze(0)
|
||||
x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
|
||||
style_vec = torch.from_numpy(style_vec).to(device).unsqueeze(0)
|
||||
del phones
|
||||
sid_tensor = torch.LongTensor([sid]).to(device)
|
||||
if is_jp_extra:
|
||||
output = net_g.infer(
|
||||
x_tst,
|
||||
x_tst_lengths,
|
||||
sid_tensor,
|
||||
tones,
|
||||
lang_ids,
|
||||
ja_bert,
|
||||
style_vec=style_vec,
|
||||
sdp_ratio=sdp_ratio,
|
||||
noise_scale=noise_scale,
|
||||
noise_scale_w=noise_scale_w,
|
||||
length_scale=length_scale,
|
||||
)
|
||||
else:
|
||||
output = net_g.infer(
|
||||
x_tst,
|
||||
x_tst_lengths,
|
||||
sid_tensor,
|
||||
tones,
|
||||
lang_ids,
|
||||
bert,
|
||||
ja_bert,
|
||||
en_bert,
|
||||
style_vec=style_vec,
|
||||
sdp_ratio=sdp_ratio,
|
||||
noise_scale=noise_scale,
|
||||
noise_scale_w=noise_scale_w,
|
||||
length_scale=length_scale,
|
||||
)
|
||||
audio = output[0][0, 0].data.cpu().float().numpy()
|
||||
del (
|
||||
x_tst,
|
||||
tones,
|
||||
lang_ids,
|
||||
bert,
|
||||
x_tst_lengths,
|
||||
sid_tensor,
|
||||
ja_bert,
|
||||
en_bert,
|
||||
style_vec,
|
||||
) # , emo
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
return audio
|
||||
|
||||
|
||||
def infer_multilang(
|
||||
text,
|
||||
style_vec,
|
||||
sdp_ratio,
|
||||
noise_scale,
|
||||
noise_scale_w,
|
||||
length_scale,
|
||||
sid,
|
||||
language,
|
||||
hps,
|
||||
net_g,
|
||||
device,
|
||||
skip_start=False,
|
||||
skip_end=False,
|
||||
):
|
||||
bert, ja_bert, en_bert, phones, tones, lang_ids = [], [], [], [], [], []
|
||||
# emo = get_emo_(reference_audio, emotion, sid)
|
||||
# if isinstance(reference_audio, np.ndarray):
|
||||
# emo = get_clap_audio_feature(reference_audio, device)
|
||||
# else:
|
||||
# emo = get_clap_text_feature(emotion, device)
|
||||
# emo = torch.squeeze(emo, dim=1)
|
||||
for idx, (txt, lang) in enumerate(zip(text, language)):
|
||||
_skip_start = (idx != 0) or (skip_start and idx == 0)
|
||||
_skip_end = (idx != len(language) - 1) or skip_end
|
||||
(
|
||||
temp_bert,
|
||||
temp_ja_bert,
|
||||
temp_en_bert,
|
||||
temp_phones,
|
||||
temp_tones,
|
||||
temp_lang_ids,
|
||||
) = get_text(txt, lang, hps, device)
|
||||
if _skip_start:
|
||||
temp_bert = temp_bert[:, 3:]
|
||||
temp_ja_bert = temp_ja_bert[:, 3:]
|
||||
temp_en_bert = temp_en_bert[:, 3:]
|
||||
temp_phones = temp_phones[3:]
|
||||
temp_tones = temp_tones[3:]
|
||||
temp_lang_ids = temp_lang_ids[3:]
|
||||
if _skip_end:
|
||||
temp_bert = temp_bert[:, :-2]
|
||||
temp_ja_bert = temp_ja_bert[:, :-2]
|
||||
temp_en_bert = temp_en_bert[:, :-2]
|
||||
temp_phones = temp_phones[:-2]
|
||||
temp_tones = temp_tones[:-2]
|
||||
temp_lang_ids = temp_lang_ids[:-2]
|
||||
bert.append(temp_bert)
|
||||
ja_bert.append(temp_ja_bert)
|
||||
en_bert.append(temp_en_bert)
|
||||
phones.append(temp_phones)
|
||||
tones.append(temp_tones)
|
||||
lang_ids.append(temp_lang_ids)
|
||||
bert = torch.concatenate(bert, dim=1)
|
||||
ja_bert = torch.concatenate(ja_bert, dim=1)
|
||||
en_bert = torch.concatenate(en_bert, dim=1)
|
||||
phones = torch.concatenate(phones, dim=0)
|
||||
tones = torch.concatenate(tones, dim=0)
|
||||
lang_ids = torch.concatenate(lang_ids, dim=0)
|
||||
with torch.no_grad():
|
||||
x_tst = phones.to(device).unsqueeze(0)
|
||||
tones = tones.to(device).unsqueeze(0)
|
||||
lang_ids = lang_ids.to(device).unsqueeze(0)
|
||||
bert = bert.to(device).unsqueeze(0)
|
||||
ja_bert = ja_bert.to(device).unsqueeze(0)
|
||||
en_bert = en_bert.to(device).unsqueeze(0)
|
||||
# emo = emo.to(device).unsqueeze(0)
|
||||
x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
|
||||
del phones
|
||||
speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
|
||||
audio = (
|
||||
net_g.infer(
|
||||
x_tst,
|
||||
x_tst_lengths,
|
||||
speakers,
|
||||
tones,
|
||||
lang_ids,
|
||||
bert,
|
||||
ja_bert,
|
||||
en_bert,
|
||||
style_vec=style_vec,
|
||||
sdp_ratio=sdp_ratio,
|
||||
noise_scale=noise_scale,
|
||||
noise_scale_w=noise_scale_w,
|
||||
length_scale=length_scale,
|
||||
)[0][0, 0]
|
||||
.data.cpu()
|
||||
.float()
|
||||
.numpy()
|
||||
)
|
||||
del (
|
||||
x_tst,
|
||||
tones,
|
||||
lang_ids,
|
||||
bert,
|
||||
x_tst_lengths,
|
||||
speakers,
|
||||
ja_bert,
|
||||
en_bert,
|
||||
) # , emo
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
return audio
|
||||
from typing import Literal
|
||||
|
||||
import torch
|
||||
|
||||
import utils
|
||||
from text import cleaned_text_to_sequence, get_bert
|
||||
from style_bert_vits2.logging import logger
|
||||
from style_bert_vits2.models import commons
|
||||
from style_bert_vits2.models.models import SynthesizerTrn
|
||||
from style_bert_vits2.models.models_jp_extra import SynthesizerTrn as SynthesizerTrnJPExtra
|
||||
from style_bert_vits2.text_processing.cleaner import clean_text
|
||||
from style_bert_vits2.text_processing.symbols import SYMBOLS
|
||||
|
||||
|
||||
class InvalidToneError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def get_net_g(model_path: str, version: str, device: str, hps):
|
||||
if version.endswith("JP-Extra"):
|
||||
logger.info("Using JP-Extra model")
|
||||
net_g = SynthesizerTrnJPExtra(
|
||||
len(SYMBOLS),
|
||||
hps.data.filter_length // 2 + 1,
|
||||
hps.train.segment_size // hps.data.hop_length,
|
||||
n_speakers=hps.data.n_speakers,
|
||||
**hps.model,
|
||||
).to(device)
|
||||
else:
|
||||
logger.info("Using normal model")
|
||||
net_g = SynthesizerTrn(
|
||||
len(SYMBOLS),
|
||||
hps.data.filter_length // 2 + 1,
|
||||
hps.train.segment_size // hps.data.hop_length,
|
||||
n_speakers=hps.data.n_speakers,
|
||||
**hps.model,
|
||||
).to(device)
|
||||
net_g.state_dict()
|
||||
_ = net_g.eval()
|
||||
if model_path.endswith(".pth") or model_path.endswith(".pt"):
|
||||
_ = utils.load_checkpoint(model_path, net_g, None, skip_optimizer=True)
|
||||
elif model_path.endswith(".safetensors"):
|
||||
_ = utils.load_safetensors(model_path, net_g, True)
|
||||
else:
|
||||
raise ValueError(f"Unknown model format: {model_path}")
|
||||
return net_g
|
||||
|
||||
|
||||
def get_text(
|
||||
text: str,
|
||||
language_str: Literal["JP", "EN", "ZH"],
|
||||
hps,
|
||||
device: str,
|
||||
assist_text: str | None = None,
|
||||
assist_text_weight: float = 0.7,
|
||||
given_tone: list[int] | None = None,
|
||||
):
|
||||
use_jp_extra = hps.version.endswith("JP-Extra")
|
||||
# 推論時のみ呼び出されるので、raise_yomi_error は False に設定
|
||||
norm_text, phone, tone, word2ph = clean_text(
|
||||
text,
|
||||
language_str,
|
||||
use_jp_extra = use_jp_extra,
|
||||
raise_yomi_error = False,
|
||||
)
|
||||
if given_tone is not None:
|
||||
if len(given_tone) != len(phone):
|
||||
raise InvalidToneError(
|
||||
f"Length of given_tone ({len(given_tone)}) != length of phone ({len(phone)})"
|
||||
)
|
||||
tone = given_tone
|
||||
phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
|
||||
|
||||
if hps.data.add_blank:
|
||||
phone = commons.intersperse(phone, 0)
|
||||
tone = commons.intersperse(tone, 0)
|
||||
language = commons.intersperse(language, 0)
|
||||
for i in range(len(word2ph)):
|
||||
word2ph[i] = word2ph[i] * 2
|
||||
word2ph[0] += 1
|
||||
bert_ori = get_bert(
|
||||
norm_text,
|
||||
word2ph,
|
||||
language_str,
|
||||
device,
|
||||
assist_text,
|
||||
assist_text_weight,
|
||||
)
|
||||
del word2ph
|
||||
assert bert_ori.shape[-1] == len(phone), phone
|
||||
|
||||
if language_str == "ZH":
|
||||
bert = bert_ori
|
||||
ja_bert = torch.zeros(1024, len(phone))
|
||||
en_bert = torch.zeros(1024, len(phone))
|
||||
elif language_str == "JP":
|
||||
bert = torch.zeros(1024, len(phone))
|
||||
ja_bert = bert_ori
|
||||
en_bert = torch.zeros(1024, len(phone))
|
||||
elif language_str == "EN":
|
||||
bert = torch.zeros(1024, len(phone))
|
||||
ja_bert = torch.zeros(1024, len(phone))
|
||||
en_bert = bert_ori
|
||||
else:
|
||||
raise ValueError("language_str should be ZH, JP or EN")
|
||||
|
||||
assert bert.shape[-1] == len(
|
||||
phone
|
||||
), f"Bert seq len {bert.shape[-1]} != {len(phone)}"
|
||||
|
||||
phone = torch.LongTensor(phone)
|
||||
tone = torch.LongTensor(tone)
|
||||
language = torch.LongTensor(language)
|
||||
return bert, ja_bert, en_bert, phone, tone, language
|
||||
|
||||
|
||||
def infer(
|
||||
text: str,
|
||||
style_vec,
|
||||
sdp_ratio: float,
|
||||
noise_scale: float,
|
||||
noise_scale_w: float,
|
||||
length_scale: float,
|
||||
sid: int, # In the original Bert-VITS2, its speaker_name: str, but here it's id
|
||||
language: Literal["JP", "EN", "ZH"],
|
||||
hps,
|
||||
net_g,
|
||||
device: str,
|
||||
skip_start: bool = False,
|
||||
skip_end: bool = False,
|
||||
assist_text: str | None = None,
|
||||
assist_text_weight: float = 0.7,
|
||||
given_tone: list[int] | None = None,
|
||||
):
|
||||
is_jp_extra = hps.version.endswith("JP-Extra")
|
||||
bert, ja_bert, en_bert, phones, tones, lang_ids = get_text(
|
||||
text,
|
||||
language,
|
||||
hps,
|
||||
device,
|
||||
assist_text=assist_text,
|
||||
assist_text_weight=assist_text_weight,
|
||||
given_tone=given_tone,
|
||||
)
|
||||
if skip_start:
|
||||
phones = phones[3:]
|
||||
tones = tones[3:]
|
||||
lang_ids = lang_ids[3:]
|
||||
bert = bert[:, 3:]
|
||||
ja_bert = ja_bert[:, 3:]
|
||||
en_bert = en_bert[:, 3:]
|
||||
if skip_end:
|
||||
phones = phones[:-2]
|
||||
tones = tones[:-2]
|
||||
lang_ids = lang_ids[:-2]
|
||||
bert = bert[:, :-2]
|
||||
ja_bert = ja_bert[:, :-2]
|
||||
en_bert = en_bert[:, :-2]
|
||||
with torch.no_grad():
|
||||
x_tst = phones.to(device).unsqueeze(0)
|
||||
tones = tones.to(device).unsqueeze(0)
|
||||
lang_ids = lang_ids.to(device).unsqueeze(0)
|
||||
bert = bert.to(device).unsqueeze(0)
|
||||
ja_bert = ja_bert.to(device).unsqueeze(0)
|
||||
en_bert = en_bert.to(device).unsqueeze(0)
|
||||
x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
|
||||
style_vec = torch.from_numpy(style_vec).to(device).unsqueeze(0)
|
||||
del phones
|
||||
sid_tensor = torch.LongTensor([sid]).to(device)
|
||||
if is_jp_extra:
|
||||
output = net_g.infer(
|
||||
x_tst,
|
||||
x_tst_lengths,
|
||||
sid_tensor,
|
||||
tones,
|
||||
lang_ids,
|
||||
ja_bert,
|
||||
style_vec=style_vec,
|
||||
sdp_ratio=sdp_ratio,
|
||||
noise_scale=noise_scale,
|
||||
noise_scale_w=noise_scale_w,
|
||||
length_scale=length_scale,
|
||||
)
|
||||
else:
|
||||
output = net_g.infer(
|
||||
x_tst,
|
||||
x_tst_lengths,
|
||||
sid_tensor,
|
||||
tones,
|
||||
lang_ids,
|
||||
bert,
|
||||
ja_bert,
|
||||
en_bert,
|
||||
style_vec=style_vec,
|
||||
sdp_ratio=sdp_ratio,
|
||||
noise_scale=noise_scale,
|
||||
noise_scale_w=noise_scale_w,
|
||||
length_scale=length_scale,
|
||||
)
|
||||
audio = output[0][0, 0].data.cpu().float().numpy()
|
||||
del (
|
||||
x_tst,
|
||||
tones,
|
||||
lang_ids,
|
||||
bert,
|
||||
x_tst_lengths,
|
||||
sid_tensor,
|
||||
ja_bert,
|
||||
en_bert,
|
||||
style_vec,
|
||||
) # , emo
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
return audio
|
||||
|
||||
|
||||
def infer_multilang(
|
||||
text: str,
|
||||
style_vec,
|
||||
sdp_ratio: float,
|
||||
noise_scale: float,
|
||||
noise_scale_w: float,
|
||||
length_scale: float,
|
||||
sid: int,
|
||||
language: Literal["JP", "EN", "ZH"],
|
||||
hps,
|
||||
net_g,
|
||||
device: str,
|
||||
skip_start: bool = False,
|
||||
skip_end: bool = False,
|
||||
):
|
||||
bert, ja_bert, en_bert, phones, tones, lang_ids = [], [], [], [], [], []
|
||||
# emo = get_emo_(reference_audio, emotion, sid)
|
||||
# if isinstance(reference_audio, np.ndarray):
|
||||
# emo = get_clap_audio_feature(reference_audio, device)
|
||||
# else:
|
||||
# emo = get_clap_text_feature(emotion, device)
|
||||
# emo = torch.squeeze(emo, dim=1)
|
||||
for idx, (txt, lang) in enumerate(zip(text, language)):
|
||||
_skip_start = (idx != 0) or (skip_start and idx == 0)
|
||||
_skip_end = (idx != len(language) - 1) or skip_end
|
||||
(
|
||||
temp_bert,
|
||||
temp_ja_bert,
|
||||
temp_en_bert,
|
||||
temp_phones,
|
||||
temp_tones,
|
||||
temp_lang_ids,
|
||||
) = get_text(txt, lang, hps, device) # type: ignore
|
||||
if _skip_start:
|
||||
temp_bert = temp_bert[:, 3:]
|
||||
temp_ja_bert = temp_ja_bert[:, 3:]
|
||||
temp_en_bert = temp_en_bert[:, 3:]
|
||||
temp_phones = temp_phones[3:]
|
||||
temp_tones = temp_tones[3:]
|
||||
temp_lang_ids = temp_lang_ids[3:]
|
||||
if _skip_end:
|
||||
temp_bert = temp_bert[:, :-2]
|
||||
temp_ja_bert = temp_ja_bert[:, :-2]
|
||||
temp_en_bert = temp_en_bert[:, :-2]
|
||||
temp_phones = temp_phones[:-2]
|
||||
temp_tones = temp_tones[:-2]
|
||||
temp_lang_ids = temp_lang_ids[:-2]
|
||||
bert.append(temp_bert)
|
||||
ja_bert.append(temp_ja_bert)
|
||||
en_bert.append(temp_en_bert)
|
||||
phones.append(temp_phones)
|
||||
tones.append(temp_tones)
|
||||
lang_ids.append(temp_lang_ids)
|
||||
bert = torch.concatenate(bert, dim=1)
|
||||
ja_bert = torch.concatenate(ja_bert, dim=1)
|
||||
en_bert = torch.concatenate(en_bert, dim=1)
|
||||
phones = torch.concatenate(phones, dim=0)
|
||||
tones = torch.concatenate(tones, dim=0)
|
||||
lang_ids = torch.concatenate(lang_ids, dim=0)
|
||||
with torch.no_grad():
|
||||
x_tst = phones.to(device).unsqueeze(0)
|
||||
tones = tones.to(device).unsqueeze(0)
|
||||
lang_ids = lang_ids.to(device).unsqueeze(0)
|
||||
bert = bert.to(device).unsqueeze(0)
|
||||
ja_bert = ja_bert.to(device).unsqueeze(0)
|
||||
en_bert = en_bert.to(device).unsqueeze(0)
|
||||
# emo = emo.to(device).unsqueeze(0)
|
||||
x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
|
||||
del phones
|
||||
speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
|
||||
audio = (
|
||||
net_g.infer(
|
||||
x_tst,
|
||||
x_tst_lengths,
|
||||
speakers,
|
||||
tones,
|
||||
lang_ids,
|
||||
bert,
|
||||
ja_bert,
|
||||
en_bert,
|
||||
style_vec=style_vec,
|
||||
sdp_ratio=sdp_ratio,
|
||||
noise_scale=noise_scale,
|
||||
noise_scale_w=noise_scale_w,
|
||||
length_scale=length_scale,
|
||||
)[0][0, 0]
|
||||
.data.cpu()
|
||||
.float()
|
||||
.numpy()
|
||||
)
|
||||
del (
|
||||
x_tst,
|
||||
tones,
|
||||
lang_ids,
|
||||
bert,
|
||||
x_tst_lengths,
|
||||
speakers,
|
||||
ja_bert,
|
||||
en_bert,
|
||||
) # , emo
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
return audio
|
||||
|
||||
46
style_bert_vits2/text_processing/cleaner.py
Normal file
46
style_bert_vits2/text_processing/cleaner.py
Normal file
@@ -0,0 +1,46 @@
|
||||
from typing import Literal
|
||||
|
||||
|
||||
def clean_text(
|
||||
text: str,
|
||||
language: Literal["JP", "EN", "ZH"],
|
||||
use_jp_extra: bool = True,
|
||||
raise_yomi_error: bool = False,
|
||||
) -> tuple[str, list[str], list[int], list[int]]:
|
||||
"""
|
||||
テキストをクリーニングし、音素に変換する
|
||||
|
||||
Args:
|
||||
text (str): クリーニングするテキスト
|
||||
language (Literal["JP", "EN", "ZH"]): テキストの言語
|
||||
use_jp_extra (bool, optional): テキストが日本語の場合に JP-Extra モデルを利用するかどうか。Defaults to True.
|
||||
raise_yomi_error (bool, optional): False の場合、読めない文字が消えたような扱いとして処理される。Defaults to False.
|
||||
|
||||
Returns:
|
||||
tuple[str, list[str], list[int], list[int]]: クリーニングされたテキストと、音素・アクセント・元のテキストの各文字に音素が何個割り当てられるかのリスト
|
||||
"""
|
||||
|
||||
# Changed to import inside if condition to avoid unnecessary import
|
||||
if language == "JP":
|
||||
from transformers import AutoTokenizer
|
||||
from style_bert_vits2.text_processing.japanese.g2p import g2p
|
||||
from style_bert_vits2.text_processing.japanese.normalizer import normalize_text
|
||||
norm_text = normalize_text(text)
|
||||
phones, tones, word2ph = g2p(
|
||||
norm_text,
|
||||
tokenizer = AutoTokenizer.from_pretrained("./bert/deberta-v2-large-japanese-char-wwm"), # 暫定的にここで指定
|
||||
use_jp_extra = use_jp_extra,
|
||||
raise_yomi_error = raise_yomi_error,
|
||||
)
|
||||
elif language == "EN":
|
||||
from ...text import english as language_module
|
||||
norm_text = language_module.normalize_text(text)
|
||||
phones, tones, word2ph = language_module.g2p(norm_text)
|
||||
elif language == "ZH":
|
||||
from ...text import chinese as language_module
|
||||
norm_text = language_module.normalize_text(text)
|
||||
phones, tones, word2ph = language_module.g2p(norm_text)
|
||||
else:
|
||||
raise ValueError(f"Language {language} not supported")
|
||||
|
||||
return norm_text, phones, tones, word2ph
|
||||
@@ -168,7 +168,7 @@ def _g2p(segments):
|
||||
return phones_list, tones_list, word2ph
|
||||
|
||||
|
||||
def text_normalize(text):
|
||||
def normalize_text(text):
|
||||
numbers = re.findall(r"\d+(?:\.?\d+)?", text)
|
||||
for number in numbers:
|
||||
text = text.replace(number, cn2an.an2cn(number), 1)
|
||||
@@ -186,7 +186,7 @@ if __name__ == "__main__":
|
||||
from text.chinese_bert import get_bert_feature
|
||||
|
||||
text = "啊！但是《原神》是由,米哈\游自主， [研发]的一款全.新开放世界.冒险游戏"
|
||||
text = text_normalize(text)
|
||||
text = normalize_text(text)
|
||||
print(text)
|
||||
phones, tones, word2ph = g2p(text)
|
||||
bert = get_bert_feature(text, word2ph)
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
def clean_text(text, language, use_jp_extra=True, raise_yomi_error=False):
|
||||
# Changed to import inside if condition to avoid unnecessary import
|
||||
if language == "ZH":
|
||||
from . import chinese as language_module
|
||||
|
||||
norm_text = language_module.text_normalize(text)
|
||||
phones, tones, word2ph = language_module.g2p(norm_text)
|
||||
elif language == "EN":
|
||||
from . import english as language_module
|
||||
|
||||
norm_text = language_module.text_normalize(text)
|
||||
phones, tones, word2ph = language_module.g2p(norm_text)
|
||||
elif language == "JP":
|
||||
from . import japanese as language_module
|
||||
|
||||
norm_text = language_module.text_normalize(text)
|
||||
phones, tones, word2ph = language_module.g2p(
|
||||
norm_text, use_jp_extra, raise_yomi_error=raise_yomi_error
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Language {language} not supported")
|
||||
return norm_text, phones, tones, word2ph
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pass
|
||||
@@ -369,7 +369,7 @@ def normalize_numbers(text):
|
||||
return text
|
||||
|
||||
|
||||
def text_normalize(text):
|
||||
def normalize_text(text):
|
||||
text = normalize_numbers(text)
|
||||
text = replace_punctuation(text)
|
||||
text = re.sub(r"([,;.\?\!])([\w])", r"\1 \2", text)
|
||||
|
||||
@@ -96,7 +96,7 @@ rep_map = {
|
||||
}
|
||||
|
||||
|
||||
def text_normalize(text):
|
||||
def normalize_text(text):
|
||||
"""
|
||||
日本語のテキストを正規化する。
|
||||
結果は、ちょうど次の文字のみからなる：
|
||||
@@ -177,7 +177,7 @@ def g2p(
|
||||
norm_text: str, use_jp_extra: bool = True, raise_yomi_error: bool = False
|
||||
) -> tuple[list[str], list[int], list[int]]:
|
||||
"""
|
||||
他で使われるメインの関数。`text_normalize()`で正規化された`norm_text`を受け取り、
|
||||
他で使われるメインの関数。`normalize_text()`で正規化された`norm_text`を受け取り、
|
||||
- phones: 音素のリスト（ただし`!`や`,`や`.`等punctuationが含まれうる）
|
||||
- tones: アクセントのリスト、0（低）と1（高）からなり、phonesと同じ長さ
|
||||
- word2ph: 元のテキストの各文字に音素が何個割り当てられるかを表すリスト
|
||||
@@ -350,7 +350,7 @@ def text2sep_kata(
|
||||
norm_text: str, raise_yomi_error: bool = False
|
||||
) -> tuple[list[str], list[str]]:
|
||||
"""
|
||||
`text_normalize`で正規化済みの`norm_text`を受け取り、それを単語分割し、
|
||||
`normalize_text()`で正規化済みの`norm_text`を受け取り、それを単語分割し、
|
||||
分割された単語リストとその読み（カタカナor記号1文字）のリストのタプルを返す。
|
||||
単語分割結果は、`g2p()`の`word2ph`で1文字あたりに割り振る音素記号の数を決めるために使う。
|
||||
例:
|
||||
@@ -634,7 +634,7 @@ if __name__ == "__main__":
|
||||
text = "こんにちは、世界。"
|
||||
from text.japanese_bert import get_bert_feature
|
||||
|
||||
text = text_normalize(text)
|
||||
text = normalize_text(text)
|
||||
|
||||
phones, tones, word2ph = g2p(text)
|
||||
bert = get_bert_feature(text, word2ph)
|
||||
|
||||
Reference in New Issue
Block a user