Use clap to achieve prompt controlled generation (#223)
* 快速分类音频并把yml格式结果存在训练根目录里 (#190) * Add files via upload * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Update models.py * Update webui.py * Update infer.py * Create compress_model.py * 重新提交，更新Gradio推理UI (#193) * Update webui.py * Update webui.py * 更新 train_ms.py * 更新 models.py * 更新 models.py * 更新 models.py * 更新 train_ms.py * 更新 train_ms.py * 更新 models.py * Update preprocess_text.py * Update config.json * Update train_ms.py * Update webui.py (#206) * Add files via upload (#209) * Update train_ms.py * Update train_ms.py * Update preprocess_text.py * Update train_ms.py * fix (#211) * Update emotion_clustering.py * Add files via upload * Update emotion_clustering.py * add cluster center save * Add files via upload * Update config.py * Update default_config.yml * Update config.py * Update config.py * Update emotion_clustering.py * Update emotion_clustering.py * Update config.py * Update emotion_clustering.py * Update emotion_clustering.py * Update webui.py * Update emotion_clustering.py * Update commons.py * Update emotion_clustering.py * Update webui.py * Update webui.py * Add files via upload * Update train_ms.py * Update train_ms.py * Update train_ms.py * Update train_ms.py * Update train_ms.py * Update webui.py * Update emotion_clustering.py * Update emotion_clustering.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix default_config.yml. * Update infer.py * feat: support infer 2.1 models * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix: support infer 2.1 models 兼容bug修复 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update train_ms.py * Add CLAP * Fix data loader * Fix infer.py * Fix webui.py * Add prompt template * Update clap_gen.py * Fix wrong environ value * Add g for dur disc * Update clap_gen.py * Fix multilang generation * Update config.json * Prompt mode * Improve slice segments performance * Add preprocess webui * Update webui_preprocess.py * Update webui_preprocess.py * Update config.py * Update default_config.yml * Update config.py * Update clap_gen.py * Delete emo_gen.py * Delete get_emo.py * Delete emotional/wav2vec2-large-robust-12-ft-emotion-msp-dim directory * Update README.md * Update README * Split val per lang * Delete emotion_clustering.py * Update default_config.yml * Update default_config.yml * Update config.py * Update preprocess_text.py * Update webui_preprocess.py * Update defalut_config.yml * Update webui_preprocess.py * Update preprocess_text.py * Random augmentation for CLAP * Update data_utils.py * Update preprocess_text.py * Add vq for CLAP features to avoid overfitting * Random dummy inputs * Update webui.py * Update models.py * Update infer.py * Apply Code Formatter Change * Update config.json * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: YYuX-1145 <138500330+YYuX-1145@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Sora <654163754@qq.com> Co-authored-by: Sihan Wang <wangsihan1995@gmail.com> Co-authored-by: Stardust-minus <Stardust-minus@users.noreply.github.com>
This commit is contained in:
Stardust·减
@@ -1,127 +0,0 @@
|
||||
---
|
||||
language: en
|
||||
datasets:
|
||||
- msp-podcast
|
||||
inference: true
|
||||
tags:
|
||||
- speech
|
||||
- audio
|
||||
- wav2vec2
|
||||
- audio-classification
|
||||
- emotion-recognition
|
||||
license: cc-by-nc-sa-4.0
|
||||
pipeline_tag: audio-classification
|
||||
---
|
||||
|
||||
# Model for Dimensional Speech Emotion Recognition based on Wav2vec 2.0
|
||||
|
||||
The model expects a raw audio signal as input and outputs predictions for arousal, dominance and valence in a range of approximately 0...1. In addition, it also provides the pooled states of the last transformer layer. The model was created by fine-tuning [
|
||||
Wav2Vec2-Large-Robust](https://huggingface.co/facebook/wav2vec2-large-robust) on [MSP-Podcast](https://ecs.utdallas.edu/research/researchlabs/msp-lab/MSP-Podcast.html) (v1.7). The model was pruned from 24 to 12 transformer layers before fine-tuning. An [ONNX](https://onnx.ai/") export of the model is available from [doi:10.5281/zenodo.6221127](https://zenodo.org/record/6221127). Further details are given in the associated [paper](https://arxiv.org/abs/2203.07378) and [tutorial](https://github.com/audeering/w2v2-how-to).
|
||||
|
||||
# Usage
|
||||
|
||||
```python
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import Wav2Vec2Processor
|
||||
from transformers.models.wav2vec2.modeling_wav2vec2 import (
|
||||
Wav2Vec2Model,
|
||||
Wav2Vec2PreTrainedModel,
|
||||
)
|
||||
|
||||
|
||||
class RegressionHead(nn.Module):
|
||||
r"""Classification head."""
|
||||
|
||||
def __init__(self, config):
|
||||
|
||||
super().__init__()
|
||||
|
||||
self.dense = nn.Linear(config.hidden_size, config.hidden_size)
|
||||
self.dropout = nn.Dropout(config.final_dropout)
|
||||
self.out_proj = nn.Linear(config.hidden_size, config.num_labels)
|
||||
|
||||
def forward(self, features, **kwargs):
|
||||
|
||||
x = features
|
||||
x = self.dropout(x)
|
||||
x = self.dense(x)
|
||||
x = torch.tanh(x)
|
||||
x = self.dropout(x)
|
||||
x = self.out_proj(x)
|
||||
|
||||
return x
|
||||
|
||||
|
||||
class EmotionModel(Wav2Vec2PreTrainedModel):
|
||||
r"""Speech emotion classifier."""
|
||||
|
||||
def __init__(self, config):
|
||||
|
||||
super().__init__(config)
|
||||
|
||||
self.config = config
|
||||
self.wav2vec2 = Wav2Vec2Model(config)
|
||||
self.classifier = RegressionHead(config)
|
||||
self.init_weights()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_values,
|
||||
):
|
||||
|
||||
outputs = self.wav2vec2(input_values)
|
||||
hidden_states = outputs[0]
|
||||
hidden_states = torch.mean(hidden_states, dim=1)
|
||||
logits = self.classifier(hidden_states)
|
||||
|
||||
return hidden_states, logits
|
||||
|
||||
|
||||
|
||||
# load model from hub
|
||||
device = 'cpu'
|
||||
model_name = 'audeering/wav2vec2-large-robust-12-ft-emotion-msp-dim'
|
||||
processor = Wav2Vec2Processor.from_pretrained(model_name)
|
||||
model = EmotionModel.from_pretrained(model_name)
|
||||
|
||||
# dummy signal
|
||||
sampling_rate = 16000
|
||||
signal = np.zeros((1, sampling_rate), dtype=np.float32)
|
||||
|
||||
|
||||
def process_func(
|
||||
x: np.ndarray,
|
||||
sampling_rate: int,
|
||||
embeddings: bool = False,
|
||||
) -> np.ndarray:
|
||||
r"""Predict emotions or extract embeddings from raw audio signal."""
|
||||
|
||||
# run through processor to normalize signal
|
||||
# always returns a batch, so we just get the first entry
|
||||
# then we put it on the device
|
||||
y = processor(x, sampling_rate=sampling_rate)
|
||||
y = y['input_values'][0]
|
||||
y = y.reshape(1, -1)
|
||||
y = torch.from_numpy(y).to(device)
|
||||
|
||||
# run through model
|
||||
with torch.no_grad():
|
||||
y = model(y)[0 if embeddings else 1]
|
||||
|
||||
# convert to numpy
|
||||
y = y.detach().cpu().numpy()
|
||||
|
||||
return y
|
||||
|
||||
|
||||
print(process_func(signal, sampling_rate))
|
||||
# Arousal dominance valence
|
||||
# [[0.5460754 0.6062266 0.40431657]]
|
||||
|
||||
print(process_func(signal, sampling_rate, embeddings=True))
|
||||
# Pooled hidden states of last transformer layer
|
||||
# [[-0.00752167 0.0065819 -0.00746342 ... 0.00663632 0.00848748
|
||||
# 0.00599211]]
|
||||
```
|
||||
Reference in New Issue
Block a user