98f5917ac9
* Fix: support infer 2.2 models * Fix imports * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
228 lines
7.2 KiB
Python
228 lines
7.2 KiB
Python
"""
|
|
@Desc: 2.2版本兼容 对应版本 v2.2 Clap-Enhanced prompt audio generation
|
|
"""
|
|
import numpy as np
|
|
import torch
|
|
import commons
|
|
from .text import cleaned_text_to_sequence, get_bert
|
|
from .text.cleaner import clean_text
|
|
from .clap_wrapper import get_clap_audio_feature, get_clap_text_feature
|
|
|
|
|
|
def get_text(text, language_str, hps, device, style_text=None, style_weight=0.7):
|
|
# 在此处实现当前版本的get_text
|
|
norm_text, phone, tone, word2ph = clean_text(text, language_str)
|
|
phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
|
|
|
|
if hps.data.add_blank:
|
|
phone = commons.intersperse(phone, 0)
|
|
tone = commons.intersperse(tone, 0)
|
|
language = commons.intersperse(language, 0)
|
|
for i in range(len(word2ph)):
|
|
word2ph[i] = word2ph[i] * 2
|
|
word2ph[0] += 1
|
|
bert_ori = get_bert(
|
|
norm_text, word2ph, language_str, device, style_text=None, style_weight=0.7
|
|
)
|
|
del word2ph
|
|
assert bert_ori.shape[-1] == len(phone), phone
|
|
|
|
if language_str == "ZH":
|
|
bert = bert_ori
|
|
ja_bert = torch.rand(1024, len(phone))
|
|
en_bert = torch.rand(1024, len(phone))
|
|
elif language_str == "JP":
|
|
bert = torch.rand(1024, len(phone))
|
|
ja_bert = bert_ori
|
|
en_bert = torch.rand(1024, len(phone))
|
|
elif language_str == "EN":
|
|
bert = torch.rand(1024, len(phone))
|
|
ja_bert = torch.rand(1024, len(phone))
|
|
en_bert = bert_ori
|
|
else:
|
|
raise ValueError("language_str should be ZH, JP or EN")
|
|
|
|
assert bert.shape[-1] == len(
|
|
phone
|
|
), f"Bert seq len {bert.shape[-1]} != {len(phone)}"
|
|
|
|
phone = torch.LongTensor(phone)
|
|
tone = torch.LongTensor(tone)
|
|
language = torch.LongTensor(language)
|
|
return bert, ja_bert, en_bert, phone, tone, language
|
|
|
|
|
|
def infer(
|
|
text,
|
|
emotion,
|
|
sdp_ratio,
|
|
noise_scale,
|
|
noise_scale_w,
|
|
length_scale,
|
|
sid,
|
|
language,
|
|
hps,
|
|
net_g,
|
|
device,
|
|
reference_audio=None,
|
|
skip_start=False,
|
|
skip_end=False,
|
|
style_text=None,
|
|
style_weight=0.7,
|
|
):
|
|
if isinstance(reference_audio, np.ndarray):
|
|
emo = get_clap_audio_feature(reference_audio, device)
|
|
else:
|
|
emo = get_clap_text_feature(emotion, device)
|
|
emo = torch.squeeze(emo, dim=1)
|
|
|
|
bert, ja_bert, en_bert, phones, tones, lang_ids = get_text(
|
|
text, language, hps, device
|
|
)
|
|
if skip_start:
|
|
phones = phones[3:]
|
|
tones = tones[3:]
|
|
lang_ids = lang_ids[3:]
|
|
bert = bert[:, 3:]
|
|
ja_bert = ja_bert[:, 3:]
|
|
en_bert = en_bert[:, 3:]
|
|
if skip_end:
|
|
phones = phones[:-2]
|
|
tones = tones[:-2]
|
|
lang_ids = lang_ids[:-2]
|
|
bert = bert[:, :-2]
|
|
ja_bert = ja_bert[:, :-2]
|
|
en_bert = en_bert[:, :-2]
|
|
with torch.no_grad():
|
|
x_tst = phones.to(device).unsqueeze(0)
|
|
tones = tones.to(device).unsqueeze(0)
|
|
lang_ids = lang_ids.to(device).unsqueeze(0)
|
|
bert = bert.to(device).unsqueeze(0)
|
|
ja_bert = ja_bert.to(device).unsqueeze(0)
|
|
en_bert = en_bert.to(device).unsqueeze(0)
|
|
x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
|
|
emo = emo.to(device).unsqueeze(0)
|
|
del phones
|
|
speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
|
|
audio = (
|
|
net_g.infer(
|
|
x_tst,
|
|
x_tst_lengths,
|
|
speakers,
|
|
tones,
|
|
lang_ids,
|
|
bert,
|
|
ja_bert,
|
|
en_bert,
|
|
emo,
|
|
sdp_ratio=sdp_ratio,
|
|
noise_scale=noise_scale,
|
|
noise_scale_w=noise_scale_w,
|
|
length_scale=length_scale,
|
|
)[0][0, 0]
|
|
.data.cpu()
|
|
.float()
|
|
.numpy()
|
|
)
|
|
del x_tst, tones, lang_ids, bert, x_tst_lengths, speakers, ja_bert, en_bert, emo
|
|
if torch.cuda.is_available():
|
|
torch.cuda.empty_cache()
|
|
return audio
|
|
|
|
|
|
def infer_multilang(
|
|
text,
|
|
sdp_ratio,
|
|
noise_scale,
|
|
noise_scale_w,
|
|
length_scale,
|
|
sid,
|
|
language,
|
|
hps,
|
|
net_g,
|
|
device,
|
|
reference_audio=None,
|
|
emotion=None,
|
|
skip_start=False,
|
|
skip_end=False,
|
|
):
|
|
bert, ja_bert, en_bert, phones, tones, lang_ids = [], [], [], [], [], []
|
|
# emo = get_emo_(reference_audio, emotion, sid)
|
|
if isinstance(reference_audio, np.ndarray):
|
|
emo = get_clap_audio_feature(reference_audio, device)
|
|
else:
|
|
emo = get_clap_text_feature(emotion, device)
|
|
emo = torch.squeeze(emo, dim=1)
|
|
for idx, (txt, lang) in enumerate(zip(text, language)):
|
|
skip_start = (idx != 0) or (skip_start and idx == 0)
|
|
skip_end = (idx != len(text) - 1) or (skip_end and idx == len(text) - 1)
|
|
(
|
|
temp_bert,
|
|
temp_ja_bert,
|
|
temp_en_bert,
|
|
temp_phones,
|
|
temp_tones,
|
|
temp_lang_ids,
|
|
) = get_text(txt, lang, hps, device)
|
|
if skip_start:
|
|
temp_bert = temp_bert[:, 3:]
|
|
temp_ja_bert = temp_ja_bert[:, 3:]
|
|
temp_en_bert = temp_en_bert[:, 3:]
|
|
temp_phones = temp_phones[3:]
|
|
temp_tones = temp_tones[3:]
|
|
temp_lang_ids = temp_lang_ids[3:]
|
|
if skip_end:
|
|
temp_bert = temp_bert[:, :-2]
|
|
temp_ja_bert = temp_ja_bert[:, :-2]
|
|
temp_en_bert = temp_en_bert[:, :-2]
|
|
temp_phones = temp_phones[:-2]
|
|
temp_tones = temp_tones[:-2]
|
|
temp_lang_ids = temp_lang_ids[:-2]
|
|
bert.append(temp_bert)
|
|
ja_bert.append(temp_ja_bert)
|
|
en_bert.append(temp_en_bert)
|
|
phones.append(temp_phones)
|
|
tones.append(temp_tones)
|
|
lang_ids.append(temp_lang_ids)
|
|
bert = torch.concatenate(bert, dim=1)
|
|
ja_bert = torch.concatenate(ja_bert, dim=1)
|
|
en_bert = torch.concatenate(en_bert, dim=1)
|
|
phones = torch.concatenate(phones, dim=0)
|
|
tones = torch.concatenate(tones, dim=0)
|
|
lang_ids = torch.concatenate(lang_ids, dim=0)
|
|
with torch.no_grad():
|
|
x_tst = phones.to(device).unsqueeze(0)
|
|
tones = tones.to(device).unsqueeze(0)
|
|
lang_ids = lang_ids.to(device).unsqueeze(0)
|
|
bert = bert.to(device).unsqueeze(0)
|
|
ja_bert = ja_bert.to(device).unsqueeze(0)
|
|
en_bert = en_bert.to(device).unsqueeze(0)
|
|
emo = emo.to(device).unsqueeze(0)
|
|
x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
|
|
del phones
|
|
speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
|
|
audio = (
|
|
net_g.infer(
|
|
x_tst,
|
|
x_tst_lengths,
|
|
speakers,
|
|
tones,
|
|
lang_ids,
|
|
bert,
|
|
ja_bert,
|
|
en_bert,
|
|
emo,
|
|
sdp_ratio=sdp_ratio,
|
|
noise_scale=noise_scale,
|
|
noise_scale_w=noise_scale_w,
|
|
length_scale=length_scale,
|
|
)[0][0, 0]
|
|
.data.cpu()
|
|
.float()
|
|
.numpy()
|
|
)
|
|
del x_tst, tones, lang_ids, bert, x_tst_lengths, speakers, ja_bert, en_bert, emo
|
|
if torch.cuda.is_available():
|
|
torch.cuda.empty_cache()
|
|
return audio
|