Use clap to achieve prompt controlled generation (#223)
* 快速分类音频并把yml格式结果存在训练根目录里 (#190) * Add files via upload * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Update models.py * Update webui.py * Update infer.py * Create compress_model.py * 重新提交，更新Gradio推理UI (#193) * Update webui.py * Update webui.py * 更新 train_ms.py * 更新 models.py * 更新 models.py * 更新 models.py * 更新 train_ms.py * 更新 train_ms.py * 更新 models.py * Update preprocess_text.py * Update config.json * Update train_ms.py * Update webui.py (#206) * Add files via upload (#209) * Update train_ms.py * Update train_ms.py * Update preprocess_text.py * Update train_ms.py * fix (#211) * Update emotion_clustering.py * Add files via upload * Update emotion_clustering.py * add cluster center save * Add files via upload * Update config.py * Update default_config.yml * Update config.py * Update config.py * Update emotion_clustering.py * Update emotion_clustering.py * Update config.py * Update emotion_clustering.py * Update emotion_clustering.py * Update webui.py * Update emotion_clustering.py * Update commons.py * Update emotion_clustering.py * Update webui.py * Update webui.py * Add files via upload * Update train_ms.py * Update train_ms.py * Update train_ms.py * Update train_ms.py * Update train_ms.py * Update webui.py * Update emotion_clustering.py * Update emotion_clustering.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix default_config.yml. * Update infer.py * feat: support infer 2.1 models * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix: support infer 2.1 models 兼容bug修复 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update train_ms.py * Add CLAP * Fix data loader * Fix infer.py * Fix webui.py * Add prompt template * Update clap_gen.py * Fix wrong environ value * Add g for dur disc * Update clap_gen.py * Fix multilang generation * Update config.json * Prompt mode * Improve slice segments performance * Add preprocess webui * Update webui_preprocess.py * Update webui_preprocess.py * Update config.py * Update default_config.yml * Update config.py * Update clap_gen.py * Delete emo_gen.py * Delete get_emo.py * Delete emotional/wav2vec2-large-robust-12-ft-emotion-msp-dim directory * Update README.md * Update README * Split val per lang * Delete emotion_clustering.py * Update default_config.yml * Update default_config.yml * Update config.py * Update preprocess_text.py * Update webui_preprocess.py * Update defalut_config.yml * Update webui_preprocess.py * Update preprocess_text.py * Random augmentation for CLAP * Update data_utils.py * Update preprocess_text.py * Add vq for CLAP features to avoid overfitting * Random dummy inputs * Update webui.py * Update models.py * Update infer.py * Apply Code Formatter Change * Update config.json * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: YYuX-1145 <138500330+YYuX-1145@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Sora <654163754@qq.com> Co-authored-by: Sihan Wang <wangsihan1995@gmail.com> Co-authored-by: Stardust-minus <Stardust-minus@users.noreply.github.com>
This commit is contained in:
Stardust·减
@@ -44,6 +44,10 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
|
||||
self.min_text_len = getattr(hparams, "min_text_len", 1)
|
||||
self.max_text_len = getattr(hparams, "max_text_len", 384)
|
||||
|
||||
self.empty_emo = torch.squeeze(
|
||||
torch.load("empty_emo.npy", map_location="cpu"), dim=1
|
||||
)
|
||||
|
||||
random.seed(1234)
|
||||
random.shuffle(self.audiopaths_sid_text)
|
||||
self._filter()
|
||||
@@ -93,7 +97,14 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
|
||||
|
||||
spec, wav = self.get_audio(audiopath)
|
||||
sid = torch.LongTensor([int(self.spk_map[sid])])
|
||||
emo = torch.FloatTensor(np.load(audiopath.replace(".wav", ".emo.npy")))
|
||||
|
||||
if np.random.rand() > 0.1:
|
||||
emo = torch.squeeze(
|
||||
torch.load(audiopath.replace(".wav", ".emo.npy"), map_location="cpu"),
|
||||
dim=1,
|
||||
)
|
||||
else:
|
||||
emo = self.empty_emo
|
||||
return (phones, spec, wav, sid, tone, language, bert, ja_bert, en_bert, emo)
|
||||
|
||||
def get_audio(self, filename):
|
||||
@@ -157,15 +168,15 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
|
||||
|
||||
if language_str == "ZH":
|
||||
bert = bert_ori
|
||||
ja_bert = torch.zeros(1024, len(phone))
|
||||
en_bert = torch.zeros(1024, len(phone))
|
||||
ja_bert = torch.rand(1024, len(phone))
|
||||
en_bert = torch.rand(1024, len(phone))
|
||||
elif language_str == "JP":
|
||||
bert = torch.zeros(1024, len(phone))
|
||||
bert = torch.rand(1024, len(phone))
|
||||
ja_bert = bert_ori
|
||||
en_bert = torch.zeros(1024, len(phone))
|
||||
en_bert = torch.rand(1024, len(phone))
|
||||
elif language_str == "EN":
|
||||
bert = torch.zeros(1024, len(phone))
|
||||
ja_bert = torch.zeros(1024, len(phone))
|
||||
bert = torch.rand(1024, len(phone))
|
||||
ja_bert = torch.rand(1024, len(phone))
|
||||
en_bert = bert_ori
|
||||
phone = torch.LongTensor(phone)
|
||||
tone = torch.LongTensor(tone)
|
||||
@@ -215,7 +226,7 @@ class TextAudioSpeakerCollate:
|
||||
bert_padded = torch.FloatTensor(len(batch), 1024, max_text_len)
|
||||
ja_bert_padded = torch.FloatTensor(len(batch), 1024, max_text_len)
|
||||
en_bert_padded = torch.FloatTensor(len(batch), 1024, max_text_len)
|
||||
emo = torch.FloatTensor(len(batch), 1024)
|
||||
emo = torch.FloatTensor(len(batch), 512)
|
||||
|
||||
spec_padded = torch.FloatTensor(len(batch), batch[0][1].size(0), max_spec_len)
|
||||
wav_padded = torch.FloatTensor(len(batch), 1, max_wav_len)
|
||||
|
||||
Reference in New Issue
Block a user