Spaces:
Running
on
Zero
Running
on
Zero
import numpy as np | |
import torch | |
from typing import Union, List, Optional | |
from transformers.feature_extraction_sequence_utils import SequenceFeatureExtractor | |
from transformers.feature_extraction_utils import BatchFeature | |
from transformers.utils import TensorType, logging | |
from transformers.utils.import_utils import is_torch_available | |
from transformers.audio_utils import mel_filter_bank, spectrogram, window_function | |
class MelFeatureExtractor(SequenceFeatureExtractor): | |
model_input_names = ["input_features"] | |
def __init__( | |
self, | |
feature_size=80, | |
sampling_rate=16000, | |
hop_length=160, | |
chunk_length=30, | |
n_fft=400, | |
padding_value=0.0, | |
dither=0.0, | |
return_attention_mask=False, | |
max_frequency=None, | |
**kwargs, | |
): | |
super().__init__( | |
feature_size=feature_size, | |
sampling_rate=sampling_rate, | |
padding_value=padding_value, | |
return_attention_mask=return_attention_mask, | |
**kwargs, | |
) | |
self.n_fft = n_fft | |
self.hop_length = hop_length | |
self.chunk_length = chunk_length | |
self.n_samples = chunk_length * sampling_rate | |
self.nb_max_frames = self.n_samples // hop_length | |
self.sampling_rate = sampling_rate | |
self.dither = dither | |
self.max_frequency = max_frequency if max_frequency is not None else sampling_rate / 2 | |
self.mel_filters = mel_filter_bank( | |
num_frequency_bins=1 + n_fft // 2, | |
num_mel_filters=feature_size, | |
min_frequency=0.0, | |
max_frequency=self.max_frequency, | |
sampling_rate=sampling_rate, | |
norm="slaney", | |
mel_scale="slaney", | |
) | |
def _np_extract_fbank_features(self, waveform_batch: np.array, device: str) -> np.ndarray: | |
if device != "cpu": | |
raise ValueError( | |
f"Got device `{device}` for feature extraction, but feature extraction on CUDA accelerator " | |
"devices requires torch, which is not installed. Either set `device='cpu'`, or " | |
"install torch according to the official instructions: https://pytorch.org/get-started/locally/" | |
) | |
log_spec_batch = [] | |
for waveform in waveform_batch: | |
log_spec = spectrogram( | |
waveform, | |
window_function(self.n_fft, "hann"), | |
frame_length=self.n_fft, | |
hop_length=self.hop_length, | |
power=2.0, | |
dither=self.dither, | |
mel_filters=self.mel_filters, | |
log_mel="log10", | |
) | |
log_spec = log_spec[:, :-1] | |
log_spec = np.maximum(log_spec, log_spec.max() - 8.0) | |
log_spec = (log_spec + 4.0) / 4.0 | |
log_spec_batch.append(log_spec) | |
log_spec_batch = np.array(log_spec_batch) | |
return log_spec_batch | |
def _torch_extract_fbank_features(self, waveform: np.array, device: str = "cpu") -> np.ndarray: | |
""" | |
Compute the log-mel spectrogram of the audio using PyTorch's GPU-accelerated STFT implementation with batching, | |
yielding results similar to cpu computing with 1e-5 tolerance. | |
""" | |
waveform = torch.from_numpy(waveform).to(device, torch.float32) | |
window = torch.hann_window(self.n_fft, device=device) | |
if self.dither != 0.0: | |
waveform += self.dither * torch.randn(waveform.shape, dtype=waveform.dtype, device=waveform.device) | |
stft = torch.stft(waveform, self.n_fft, self.hop_length, window=window, return_complex=True) | |
magnitudes = stft[..., :-1].abs() ** 2 | |
mel_filters = torch.from_numpy(self.mel_filters).to(device, torch.float32) | |
mel_spec = mel_filters.T @ magnitudes | |
log_spec = torch.clamp(mel_spec, min=1e-10).log10() | |
if waveform.dim() == 2: | |
max_val = log_spec.max(dim=2, keepdim=True)[0].max(dim=1, keepdim=True)[0] | |
log_spec = torch.maximum(log_spec, max_val - 8.0) | |
else: | |
log_spec = torch.maximum(log_spec, log_spec.max() - 8.0) | |
log_spec = (log_spec + 4.0) / 4.0 | |
if device != "cpu": | |
log_spec = log_spec.detach().cpu() | |
return log_spec.numpy() | |
def zero_mean_unit_var_norm( | |
input_values: List[np.ndarray], attention_mask: List[np.ndarray], padding_value: float = 0.0 | |
) -> List[np.ndarray]: | |
""" | |
Every array in the list is normalized to have zero mean and unit variance | |
""" | |
if attention_mask is not None: | |
attention_mask = np.array(attention_mask, np.int32) | |
normed_input_values = [] | |
for vector, length in zip(input_values, attention_mask.sum(-1)): | |
normed_slice = (vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1e-7) | |
if length < normed_slice.shape[0]: | |
normed_slice[length:] = padding_value | |
normed_input_values.append(normed_slice) | |
else: | |
normed_input_values = [(x - x.mean()) / np.sqrt(x.var() + 1e-7) for x in input_values] | |
return normed_input_values | |
def __call__( | |
self, | |
raw_speech: Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]], | |
truncation: bool = True, | |
pad_to_multiple_of: Optional[int] = None, | |
return_tensors: Optional[Union[str, TensorType]] = None, | |
return_attention_mask: Optional[bool] = None, | |
padding: Optional[str] = "max_length", | |
max_length: Optional[int] = None, | |
sampling_rate: Optional[int] = None, | |
do_normalize: Optional[bool] = None, | |
device: Optional[str] = "cpu", | |
return_token_timestamps: Optional[bool] = None, | |
**kwargs, | |
) -> BatchFeature: | |
""" | |
Main method to featurize and prepare for the model one or several sequence(s). Implementation uses PyTorch for | |
the STFT computation if available, otherwise a slower NumPy based one. | |
Args: | |
raw_speech (`np.ndarray`, `List[float]`, `List[np.ndarray]`, `List[List[float]]`): | |
The sequence or batch of sequences to be padded. Each sequence can be a numpy array, a list of float | |
values, a list of numpy arrays or a list of list of float values. Must be mono channel audio, not | |
stereo, i.e. single float per timestep. | |
truncation (`bool`, *optional*, default to `True`): | |
Activates truncation to cut input sequences longer than *max_length* to *max_length*. | |
pad_to_multiple_of (`int`, *optional*, defaults to None): | |
If set will pad the sequence to a multiple of the provided value. | |
return_tensors (`str` or [`~utils.TensorType`], *optional*): | |
If set, will return tensors instead of list of python integers. Acceptable values are: | |
- `'tf'`: Return TensorFlow `tf.constant` objects. | |
- `'pt'`: Return PyTorch `torch.Tensor` objects. | |
- `'np'`: Return Numpy `np.ndarray` objects. | |
sampling_rate (`int`, *optional*): | |
The sampling rate at which the `raw_speech` input was sampled. If provided, it is checked against | |
the extractor's sampling rate. | |
padding_value (`float`, *optional*, defaults to 0.0): | |
The value that is used to fill the padding values / vectors. | |
do_normalize (`bool`, *optional*, defaults to `False`): | |
Whether or not to zero-mean unit-variance normalize the input. | |
device (`str`, *optional*, defaults to `'cpu'`): | |
Specifies the device for computation of the log-mel spectrogram. | |
return_token_timestamps (`bool`, *optional*, defaults to `None`): | |
Whether or not to return the number of frames of the input raw_speech. | |
""" | |
if sampling_rate is not None and sampling_rate != self.sampling_rate: | |
logger.warning( | |
f"The provided `raw_speech` input was sampled at {sampling_rate}Hz, but the feature extractor " | |
f"is configured for {self.sampling_rate}Hz. You should resample the audio to match the " | |
f"extractor's sampling rate to ensure correct feature extraction." | |
) | |
is_batched_numpy = isinstance(raw_speech, np.ndarray) and len(raw_speech.shape) > 1 | |
if is_batched_numpy and len(raw_speech.shape) > 2: | |
raise ValueError(f"Only mono-channel audio is supported for input to {self}") | |
is_batched = is_batched_numpy or ( | |
isinstance(raw_speech, (list, tuple)) and (isinstance(raw_speech[0], (np.ndarray, tuple, list))) | |
) | |
if is_batched: | |
raw_speech = [np.asarray([speech], dtype=np.float32).T for speech in raw_speech] | |
elif not is_batched and not isinstance(raw_speech, np.ndarray): | |
raw_speech = np.asarray(raw_speech, dtype=np.float32) | |
elif isinstance(raw_speech, np.ndarray) and raw_speech.dtype is np.dtype(np.float64): | |
raw_speech = raw_speech.astype(np.float32) | |
if not is_batched: | |
raw_speech = [np.asarray([raw_speech]).T] | |
batched_speech = BatchFeature({"input_features": raw_speech}) | |
padded_inputs = self.pad( | |
batched_speech, | |
padding=padding, | |
max_length=max_length if max_length else self.n_samples, | |
truncation=truncation, | |
pad_to_multiple_of=pad_to_multiple_of, | |
return_attention_mask=return_attention_mask or do_normalize, | |
) | |
if do_normalize: | |
padded_inputs["input_features"] = self.zero_mean_unit_var_norm( | |
padded_inputs["input_features"], | |
attention_mask=padded_inputs["attention_mask"], | |
padding_value=self.padding_value, | |
) | |
padded_inputs["input_features"] = np.stack(padded_inputs["input_features"], axis=0) | |
input_features = padded_inputs.get("input_features").transpose(2, 0, 1) | |
extract_fbank_features = ( | |
self._torch_extract_fbank_features if is_torch_available() else self._np_extract_fbank_features | |
) | |
input_features = extract_fbank_features(input_features[0], device) | |
if isinstance(input_features[0], List): | |
padded_inputs["input_features"] = [np.asarray(feature, dtype=np.float32) for feature in input_features] | |
else: | |
padded_inputs["input_features"] = input_features | |
if return_attention_mask: | |
padded_inputs["attention_mask"] = padded_inputs["attention_mask"][:, :: self.hop_length] | |
if return_token_timestamps is not None: | |
padded_inputs["num_frames"] = [len(raw_speech_i) // self.hop_length for raw_speech_i in raw_speech] | |
if return_tensors is not None: | |
padded_inputs = padded_inputs.convert_to_tensors(return_tensors) | |
return padded_inputs |