#!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. # MIT License (https://opensource.org/licenses/MIT) import os import json import time import math import torch from torch import nn from enum import Enum from dataclasses import dataclass from funasr_detach.register import tables from typing import List, Tuple, Dict, Any, Optional from funasr_detach.utils.datadir_writer import DatadirWriter from funasr_detach.utils.load_utils import load_audio_text_image_video, extract_fbank class VadStateMachine(Enum): kVadInStateStartPointNotDetected = 1 kVadInStateInSpeechSegment = 2 kVadInStateEndPointDetected = 3 class FrameState(Enum): kFrameStateInvalid = -1 kFrameStateSpeech = 1 kFrameStateSil = 0 # final voice/unvoice state per frame class AudioChangeState(Enum): kChangeStateSpeech2Speech = 0 kChangeStateSpeech2Sil = 1 kChangeStateSil2Sil = 2 kChangeStateSil2Speech = 3 kChangeStateNoBegin = 4 kChangeStateInvalid = 5 class VadDetectMode(Enum): kVadSingleUtteranceDetectMode = 0 kVadMutipleUtteranceDetectMode = 1 class VADXOptions: """ Author: Speech Lab of DAMO Academy, Alibaba Group Deep-FSMN for Large Vocabulary Continuous Speech Recognition https://arxiv.org/abs/1803.05030 """ def __init__( self, sample_rate: int = 16000, detect_mode: int = VadDetectMode.kVadMutipleUtteranceDetectMode.value, snr_mode: int = 0, max_end_silence_time: int = 800, max_start_silence_time: int = 3000, do_start_point_detection: bool = True, do_end_point_detection: bool = True, window_size_ms: int = 200, sil_to_speech_time_thres: int = 150, speech_to_sil_time_thres: int = 150, speech_2_noise_ratio: float = 1.0, do_extend: int = 1, lookback_time_start_point: int = 200, lookahead_time_end_point: int = 100, max_single_segment_time: int = 60000, nn_eval_block_size: int = 8, dcd_block_size: int = 4, snr_thres: int = -100.0, noise_frame_num_used_for_snr: int = 100, decibel_thres: int = -100.0, speech_noise_thres: float = 0.6, fe_prior_thres: float = 1e-4, silence_pdf_num: int = 1, sil_pdf_ids: List[int] = [0], speech_noise_thresh_low: float = -0.1, speech_noise_thresh_high: float = 0.3, output_frame_probs: bool = False, frame_in_ms: int = 10, frame_length_ms: int = 25, **kwargs, ): self.sample_rate = sample_rate self.detect_mode = detect_mode self.snr_mode = snr_mode self.max_end_silence_time = max_end_silence_time self.max_start_silence_time = max_start_silence_time self.do_start_point_detection = do_start_point_detection self.do_end_point_detection = do_end_point_detection self.window_size_ms = window_size_ms self.sil_to_speech_time_thres = sil_to_speech_time_thres self.speech_to_sil_time_thres = speech_to_sil_time_thres self.speech_2_noise_ratio = speech_2_noise_ratio self.do_extend = do_extend self.lookback_time_start_point = lookback_time_start_point self.lookahead_time_end_point = lookahead_time_end_point self.max_single_segment_time = max_single_segment_time self.nn_eval_block_size = nn_eval_block_size self.dcd_block_size = dcd_block_size self.snr_thres = snr_thres self.noise_frame_num_used_for_snr = noise_frame_num_used_for_snr self.decibel_thres = decibel_thres self.speech_noise_thres = speech_noise_thres self.fe_prior_thres = fe_prior_thres self.silence_pdf_num = silence_pdf_num self.sil_pdf_ids = sil_pdf_ids self.speech_noise_thresh_low = speech_noise_thresh_low self.speech_noise_thresh_high = speech_noise_thresh_high self.output_frame_probs = output_frame_probs self.frame_in_ms = frame_in_ms self.frame_length_ms = frame_length_ms class E2EVadSpeechBufWithDoa(object): """ Author: Speech Lab of DAMO Academy, Alibaba Group Deep-FSMN for Large Vocabulary Continuous Speech Recognition https://arxiv.org/abs/1803.05030 """ def __init__(self): self.start_ms = 0 self.end_ms = 0 self.buffer = [] self.contain_seg_start_point = False self.contain_seg_end_point = False self.doa = 0 def Reset(self): self.start_ms = 0 self.end_ms = 0 self.buffer = [] self.contain_seg_start_point = False self.contain_seg_end_point = False self.doa = 0 class E2EVadFrameProb(object): """ Author: Speech Lab of DAMO Academy, Alibaba Group Deep-FSMN for Large Vocabulary Continuous Speech Recognition https://arxiv.org/abs/1803.05030 """ def __init__(self): self.noise_prob = 0.0 self.speech_prob = 0.0 self.score = 0.0 self.frame_id = 0 self.frm_state = 0 class WindowDetector(object): """ Author: Speech Lab of DAMO Academy, Alibaba Group Deep-FSMN for Large Vocabulary Continuous Speech Recognition https://arxiv.org/abs/1803.05030 """ def __init__( self, window_size_ms: int, sil_to_speech_time: int, speech_to_sil_time: int, frame_size_ms: int, ): self.window_size_ms = window_size_ms self.sil_to_speech_time = sil_to_speech_time self.speech_to_sil_time = speech_to_sil_time self.frame_size_ms = frame_size_ms self.win_size_frame = int(window_size_ms / frame_size_ms) self.win_sum = 0 self.win_state = [0] * self.win_size_frame # 初始化窗 self.cur_win_pos = 0 self.pre_frame_state = FrameState.kFrameStateSil self.cur_frame_state = FrameState.kFrameStateSil self.sil_to_speech_frmcnt_thres = int(sil_to_speech_time / frame_size_ms) self.speech_to_sil_frmcnt_thres = int(speech_to_sil_time / frame_size_ms) self.voice_last_frame_count = 0 self.noise_last_frame_count = 0 self.hydre_frame_count = 0 def Reset(self) -> None: self.cur_win_pos = 0 self.win_sum = 0 self.win_state = [0] * self.win_size_frame self.pre_frame_state = FrameState.kFrameStateSil self.cur_frame_state = FrameState.kFrameStateSil self.voice_last_frame_count = 0 self.noise_last_frame_count = 0 self.hydre_frame_count = 0 def GetWinSize(self) -> int: return int(self.win_size_frame) def DetectOneFrame( self, frameState: FrameState, frame_count: int, cache: dict = {} ) -> AudioChangeState: cur_frame_state = FrameState.kFrameStateSil if frameState == FrameState.kFrameStateSpeech: cur_frame_state = 1 elif frameState == FrameState.kFrameStateSil: cur_frame_state = 0 else: return AudioChangeState.kChangeStateInvalid self.win_sum -= self.win_state[self.cur_win_pos] self.win_sum += cur_frame_state self.win_state[self.cur_win_pos] = cur_frame_state self.cur_win_pos = (self.cur_win_pos + 1) % self.win_size_frame if ( self.pre_frame_state == FrameState.kFrameStateSil and self.win_sum >= self.sil_to_speech_frmcnt_thres ): self.pre_frame_state = FrameState.kFrameStateSpeech return AudioChangeState.kChangeStateSil2Speech if ( self.pre_frame_state == FrameState.kFrameStateSpeech and self.win_sum <= self.speech_to_sil_frmcnt_thres ): self.pre_frame_state = FrameState.kFrameStateSil return AudioChangeState.kChangeStateSpeech2Sil if self.pre_frame_state == FrameState.kFrameStateSil: return AudioChangeState.kChangeStateSil2Sil if self.pre_frame_state == FrameState.kFrameStateSpeech: return AudioChangeState.kChangeStateSpeech2Speech return AudioChangeState.kChangeStateInvalid def FrameSizeMs(self) -> int: return int(self.frame_size_ms) class Stats(object): def __init__( self, sil_pdf_ids, max_end_sil_frame_cnt_thresh, speech_noise_thres, ): self.data_buf_start_frame = 0 self.frm_cnt = 0 self.latest_confirmed_speech_frame = 0 self.lastest_confirmed_silence_frame = -1 self.continous_silence_frame_count = 0 self.vad_state_machine = VadStateMachine.kVadInStateStartPointNotDetected self.confirmed_start_frame = -1 self.confirmed_end_frame = -1 self.number_end_time_detected = 0 self.sil_frame = 0 self.sil_pdf_ids = sil_pdf_ids self.noise_average_decibel = -100.0 self.pre_end_silence_detected = False self.next_seg = True self.output_data_buf = [] self.output_data_buf_offset = 0 self.frame_probs = [] self.max_end_sil_frame_cnt_thresh = max_end_sil_frame_cnt_thresh self.speech_noise_thres = speech_noise_thres self.scores = None self.max_time_out = False self.decibel = [] self.data_buf = None self.data_buf_all = None self.waveform = None self.last_drop_frames = 0 @tables.register("model_classes", "FsmnVADStreaming") class FsmnVADStreaming(nn.Module): """ Author: Speech Lab of DAMO Academy, Alibaba Group Deep-FSMN for Large Vocabulary Continuous Speech Recognition https://arxiv.org/abs/1803.05030 """ def __init__( self, encoder: str = None, encoder_conf: Optional[Dict] = None, vad_post_args: Dict[str, Any] = None, **kwargs, ): super().__init__() self.vad_opts = VADXOptions(**kwargs) encoder_class = tables.encoder_classes.get(encoder) encoder = encoder_class(**encoder_conf) self.encoder = encoder def ResetDetection(self, cache: dict = {}): cache["stats"].continous_silence_frame_count = 0 cache["stats"].latest_confirmed_speech_frame = 0 cache["stats"].lastest_confirmed_silence_frame = -1 cache["stats"].confirmed_start_frame = -1 cache["stats"].confirmed_end_frame = -1 cache["stats"].vad_state_machine = ( VadStateMachine.kVadInStateStartPointNotDetected ) cache["windows_detector"].Reset() cache["stats"].sil_frame = 0 cache["stats"].frame_probs = [] if cache["stats"].output_data_buf: assert cache["stats"].output_data_buf[-1].contain_seg_end_point == True drop_frames = int( cache["stats"].output_data_buf[-1].end_ms / self.vad_opts.frame_in_ms ) real_drop_frames = drop_frames - cache["stats"].last_drop_frames cache["stats"].last_drop_frames = drop_frames cache["stats"].data_buf_all = cache["stats"].data_buf_all[ real_drop_frames * int(self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000) : ] cache["stats"].decibel = cache["stats"].decibel[real_drop_frames:] cache["stats"].scores = cache["stats"].scores[:, real_drop_frames:, :] def ComputeDecibel(self, cache: dict = {}) -> None: frame_sample_length = int( self.vad_opts.frame_length_ms * self.vad_opts.sample_rate / 1000 ) frame_shift_length = int( self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000 ) if cache["stats"].data_buf_all is None: cache["stats"].data_buf_all = cache["stats"].waveform[ 0 ] # cache["stats"].data_buf is pointed to cache["stats"].waveform[0] cache["stats"].data_buf = cache["stats"].data_buf_all else: cache["stats"].data_buf_all = torch.cat( (cache["stats"].data_buf_all, cache["stats"].waveform[0]) ) for offset in range( 0, cache["stats"].waveform.shape[1] - frame_sample_length + 1, frame_shift_length, ): cache["stats"].decibel.append( 10 * math.log10( (cache["stats"].waveform[0][offset : offset + frame_sample_length]) .square() .sum() + 0.000001 ) ) def ComputeScores(self, feats: torch.Tensor, cache: dict = {}) -> None: scores = self.encoder(feats, cache=cache["encoder"]).to( "cpu" ) # return B * T * D assert ( scores.shape[1] == feats.shape[1] ), "The shape between feats and scores does not match" self.vad_opts.nn_eval_block_size = scores.shape[1] cache["stats"].frm_cnt += scores.shape[1] # count total frames if cache["stats"].scores is None: cache["stats"].scores = scores # the first calculation else: cache["stats"].scores = torch.cat((cache["stats"].scores, scores), dim=1) def PopDataBufTillFrame( self, frame_idx: int, cache: dict = {} ) -> None: # need check again while cache["stats"].data_buf_start_frame < frame_idx: if len(cache["stats"].data_buf) >= int( self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000 ): cache["stats"].data_buf_start_frame += 1 cache["stats"].data_buf = cache["stats"].data_buf_all[ ( cache["stats"].data_buf_start_frame - cache["stats"].last_drop_frames ) * int( self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000 ) : ] def PopDataToOutputBuf( self, start_frm: int, frm_cnt: int, first_frm_is_start_point: bool, last_frm_is_end_point: bool, end_point_is_sent_end: bool, cache: dict = {}, ) -> None: self.PopDataBufTillFrame(start_frm, cache=cache) expected_sample_number = int( frm_cnt * self.vad_opts.sample_rate * self.vad_opts.frame_in_ms / 1000 ) if last_frm_is_end_point: extra_sample = max( 0, int( self.vad_opts.frame_length_ms * self.vad_opts.sample_rate / 1000 - self.vad_opts.sample_rate * self.vad_opts.frame_in_ms / 1000 ), ) expected_sample_number += int(extra_sample) if end_point_is_sent_end: expected_sample_number = max( expected_sample_number, len(cache["stats"].data_buf) ) if len(cache["stats"].data_buf) < expected_sample_number: print("error in calling pop data_buf\n") if len(cache["stats"].output_data_buf) == 0 or first_frm_is_start_point: cache["stats"].output_data_buf.append(E2EVadSpeechBufWithDoa()) cache["stats"].output_data_buf[-1].Reset() cache["stats"].output_data_buf[-1].start_ms = ( start_frm * self.vad_opts.frame_in_ms ) cache["stats"].output_data_buf[-1].end_ms = ( cache["stats"].output_data_buf[-1].start_ms ) cache["stats"].output_data_buf[-1].doa = 0 cur_seg = cache["stats"].output_data_buf[-1] if cur_seg.end_ms != start_frm * self.vad_opts.frame_in_ms: print("warning\n") out_pos = len(cur_seg.buffer) # cur_seg.buff现在没做任何操作 data_to_pop = 0 if end_point_is_sent_end: data_to_pop = expected_sample_number else: data_to_pop = int( frm_cnt * self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000 ) if data_to_pop > len(cache["stats"].data_buf): print('VAD data_to_pop is bigger than cache["stats"].data_buf.size()!!!\n') data_to_pop = len(cache["stats"].data_buf) expected_sample_number = len(cache["stats"].data_buf) cur_seg.doa = 0 for sample_cpy_out in range(0, data_to_pop): # cur_seg.buffer[out_pos ++] = data_buf_.back(); out_pos += 1 for sample_cpy_out in range(data_to_pop, expected_sample_number): # cur_seg.buffer[out_pos++] = data_buf_.back() out_pos += 1 if cur_seg.end_ms != start_frm * self.vad_opts.frame_in_ms: print("Something wrong with the VAD algorithm\n") cache["stats"].data_buf_start_frame += frm_cnt cur_seg.end_ms = (start_frm + frm_cnt) * self.vad_opts.frame_in_ms if first_frm_is_start_point: cur_seg.contain_seg_start_point = True if last_frm_is_end_point: cur_seg.contain_seg_end_point = True def OnSilenceDetected(self, valid_frame: int, cache: dict = {}): cache["stats"].lastest_confirmed_silence_frame = valid_frame if ( cache["stats"].vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected ): self.PopDataBufTillFrame(valid_frame, cache=cache) # silence_detected_callback_ # pass def OnVoiceDetected(self, valid_frame: int, cache: dict = {}) -> None: cache["stats"].latest_confirmed_speech_frame = valid_frame self.PopDataToOutputBuf(valid_frame, 1, False, False, False, cache=cache) def OnVoiceStart( self, start_frame: int, fake_result: bool = False, cache: dict = {} ) -> None: if self.vad_opts.do_start_point_detection: pass if cache["stats"].confirmed_start_frame != -1: print("not reset vad properly\n") else: cache["stats"].confirmed_start_frame = start_frame if ( not fake_result and cache["stats"].vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected ): self.PopDataToOutputBuf( cache["stats"].confirmed_start_frame, 1, True, False, False, cache=cache ) def OnVoiceEnd( self, end_frame: int, fake_result: bool, is_last_frame: bool, cache: dict = {} ) -> None: for t in range(cache["stats"].latest_confirmed_speech_frame + 1, end_frame): self.OnVoiceDetected(t, cache=cache) if self.vad_opts.do_end_point_detection: pass if cache["stats"].confirmed_end_frame != -1: print("not reset vad properly\n") else: cache["stats"].confirmed_end_frame = end_frame if not fake_result: cache["stats"].sil_frame = 0 self.PopDataToOutputBuf( cache["stats"].confirmed_end_frame, 1, False, True, is_last_frame, cache=cache, ) cache["stats"].number_end_time_detected += 1 def MaybeOnVoiceEndIfLastFrame( self, is_final_frame: bool, cur_frm_idx: int, cache: dict = {} ) -> None: if is_final_frame: self.OnVoiceEnd(cur_frm_idx, False, True, cache=cache) cache["stats"].vad_state_machine = ( VadStateMachine.kVadInStateEndPointDetected ) def GetLatency(self, cache: dict = {}) -> int: return int( self.LatencyFrmNumAtStartPoint(cache=cache) * self.vad_opts.frame_in_ms ) def LatencyFrmNumAtStartPoint(self, cache: dict = {}) -> int: vad_latency = cache["windows_detector"].GetWinSize() if self.vad_opts.do_extend: vad_latency += int( self.vad_opts.lookback_time_start_point / self.vad_opts.frame_in_ms ) return vad_latency def GetFrameState(self, t: int, cache: dict = {}): frame_state = FrameState.kFrameStateInvalid cur_decibel = cache["stats"].decibel[t] cur_snr = cur_decibel - cache["stats"].noise_average_decibel # for each frame, calc log posterior probability of each state if cur_decibel < self.vad_opts.decibel_thres: frame_state = FrameState.kFrameStateSil self.DetectOneFrame(frame_state, t, False, cache=cache) return frame_state sum_score = 0.0 noise_prob = 0.0 assert len(cache["stats"].sil_pdf_ids) == self.vad_opts.silence_pdf_num if len(cache["stats"].sil_pdf_ids) > 0: assert len(cache["stats"].scores) == 1 # 只支持batch_size = 1的测试 sil_pdf_scores = [ cache["stats"].scores[0][t][sil_pdf_id] for sil_pdf_id in cache["stats"].sil_pdf_ids ] sum_score = sum(sil_pdf_scores) noise_prob = math.log(sum_score) * self.vad_opts.speech_2_noise_ratio total_score = 1.0 sum_score = total_score - sum_score speech_prob = math.log(sum_score) if self.vad_opts.output_frame_probs: frame_prob = E2EVadFrameProb() frame_prob.noise_prob = noise_prob frame_prob.speech_prob = speech_prob frame_prob.score = sum_score frame_prob.frame_id = t cache["stats"].frame_probs.append(frame_prob) if ( math.exp(speech_prob) >= math.exp(noise_prob) + cache["stats"].speech_noise_thres ): if ( cur_snr >= self.vad_opts.snr_thres and cur_decibel >= self.vad_opts.decibel_thres ): frame_state = FrameState.kFrameStateSpeech else: frame_state = FrameState.kFrameStateSil else: frame_state = FrameState.kFrameStateSil if cache["stats"].noise_average_decibel < -99.9: cache["stats"].noise_average_decibel = cur_decibel else: cache["stats"].noise_average_decibel = ( cur_decibel + cache["stats"].noise_average_decibel * (self.vad_opts.noise_frame_num_used_for_snr - 1) ) / self.vad_opts.noise_frame_num_used_for_snr return frame_state def forward( self, feats: torch.Tensor, waveform: torch.tensor, cache: dict = {}, is_final: bool = False, **kwargs, ): # if len(cache) == 0: # self.AllResetDetection() # self.waveform = waveform # compute decibel for each frame cache["stats"].waveform = waveform is_streaming_input = kwargs.get("is_streaming_input", True) self.ComputeDecibel(cache=cache) self.ComputeScores(feats, cache=cache) if not is_final: self.DetectCommonFrames(cache=cache) else: self.DetectLastFrames(cache=cache) segments = [] for batch_num in range(0, feats.shape[0]): # only support batch_size = 1 now segment_batch = [] if len(cache["stats"].output_data_buf) > 0: for i in range( cache["stats"].output_data_buf_offset, len(cache["stats"].output_data_buf), ): if ( is_streaming_input ): # in this case, return [beg, -1], [], [-1, end], [beg, end] if ( not cache["stats"] .output_data_buf[i] .contain_seg_start_point ): continue if ( not cache["stats"].next_seg and not cache["stats"] .output_data_buf[i] .contain_seg_end_point ): continue start_ms = ( cache["stats"].output_data_buf[i].start_ms if cache["stats"].next_seg else -1 ) if cache["stats"].output_data_buf[i].contain_seg_end_point: end_ms = cache["stats"].output_data_buf[i].end_ms cache["stats"].next_seg = True cache["stats"].output_data_buf_offset += 1 else: end_ms = -1 cache["stats"].next_seg = False segment = [start_ms, end_ms] else: # in this case, return [beg, end] if not is_final and ( not cache["stats"] .output_data_buf[i] .contain_seg_start_point or not cache["stats"] .output_data_buf[i] .contain_seg_end_point ): continue segment = [ cache["stats"].output_data_buf[i].start_ms, cache["stats"].output_data_buf[i].end_ms, ] cache[ "stats" ].output_data_buf_offset += 1 # need update this parameter segment_batch.append(segment) if segment_batch: segments.append(segment_batch) # if is_final: # # reset class variables and clear the dict for the next query # self.AllResetDetection() return segments def init_cache(self, cache: dict = {}, **kwargs): cache["frontend"] = {} cache["prev_samples"] = torch.empty(0) cache["encoder"] = {} windows_detector = WindowDetector( self.vad_opts.window_size_ms, self.vad_opts.sil_to_speech_time_thres, self.vad_opts.speech_to_sil_time_thres, self.vad_opts.frame_in_ms, ) windows_detector.Reset() stats = Stats( sil_pdf_ids=self.vad_opts.sil_pdf_ids, max_end_sil_frame_cnt_thresh=self.vad_opts.max_end_silence_time - self.vad_opts.speech_to_sil_time_thres, speech_noise_thres=self.vad_opts.speech_noise_thres, ) cache["windows_detector"] = windows_detector cache["stats"] = stats return cache def inference( self, data_in, data_lengths=None, key: list = None, tokenizer=None, frontend=None, cache: dict = {}, **kwargs, ): if len(cache) == 0: self.init_cache(cache, **kwargs) meta_data = {} chunk_size = kwargs.get("chunk_size", 60000) # 50ms chunk_stride_samples = int(chunk_size * frontend.fs / 1000) time1 = time.perf_counter() is_streaming_input = ( kwargs.get("is_streaming_input", False) if chunk_size >= 15000 else kwargs.get("is_streaming_input", True) ) is_final = ( kwargs.get("is_final", False) if is_streaming_input else kwargs.get("is_final", True) ) cfg = {"is_final": is_final, "is_streaming_input": is_streaming_input} audio_sample_list = load_audio_text_image_video( data_in, fs=frontend.fs, audio_fs=kwargs.get("fs", 16000), data_type=kwargs.get("data_type", "sound"), tokenizer=tokenizer, cache=cfg, ) _is_final = cfg["is_final"] # if data_in is a file or url, set is_final=True is_streaming_input = cfg["is_streaming_input"] time2 = time.perf_counter() meta_data["load_data"] = f"{time2 - time1:0.3f}" assert len(audio_sample_list) == 1, "batch_size must be set 1" audio_sample = torch.cat((cache["prev_samples"], audio_sample_list[0])) n = int(len(audio_sample) // chunk_stride_samples + int(_is_final)) m = int(len(audio_sample) % chunk_stride_samples * (1 - int(_is_final))) segments = [] for i in range(n): kwargs["is_final"] = _is_final and i == n - 1 audio_sample_i = audio_sample[ i * chunk_stride_samples : (i + 1) * chunk_stride_samples ] # extract fbank feats speech, speech_lengths = extract_fbank( [audio_sample_i], data_type=kwargs.get("data_type", "sound"), frontend=frontend, cache=cache["frontend"], is_final=kwargs["is_final"], ) time3 = time.perf_counter() meta_data["extract_feat"] = f"{time3 - time2:0.3f}" meta_data["batch_data_time"] = ( speech_lengths.sum().item() * frontend.frame_shift * frontend.lfr_n / 1000 ) speech = speech.to(device=kwargs["device"]) speech_lengths = speech_lengths.to(device=kwargs["device"]) batch = { "feats": speech, "waveform": cache["frontend"]["waveforms"], "is_final": kwargs["is_final"], "cache": cache, "is_streaming_input": is_streaming_input, } segments_i = self.forward(**batch) if len(segments_i) > 0: segments.extend(*segments_i) cache["prev_samples"] = audio_sample[:-m] if _is_final: self.init_cache(cache) ibest_writer = None if kwargs.get("output_dir") is not None: if not hasattr(self, "writer"): self.writer = DatadirWriter(kwargs.get("output_dir")) ibest_writer = self.writer[f"{1}best_recog"] results = [] result_i = {"key": key[0], "value": segments} if ( "MODELSCOPE_ENVIRONMENT" in os.environ and os.environ["MODELSCOPE_ENVIRONMENT"] == "eas" ): result_i = json.dumps(result_i) results.append(result_i) if ibest_writer is not None: ibest_writer["text"][key[0]] = segments return results, meta_data def DetectCommonFrames(self, cache: dict = {}) -> int: if ( cache["stats"].vad_state_machine == VadStateMachine.kVadInStateEndPointDetected ): return 0 for i in range(self.vad_opts.nn_eval_block_size - 1, -1, -1): frame_state = FrameState.kFrameStateInvalid frame_state = self.GetFrameState( cache["stats"].frm_cnt - 1 - i - cache["stats"].last_drop_frames, cache=cache, ) self.DetectOneFrame( frame_state, cache["stats"].frm_cnt - 1 - i, False, cache=cache ) return 0 def DetectLastFrames(self, cache: dict = {}) -> int: if ( cache["stats"].vad_state_machine == VadStateMachine.kVadInStateEndPointDetected ): return 0 for i in range(self.vad_opts.nn_eval_block_size - 1, -1, -1): frame_state = FrameState.kFrameStateInvalid frame_state = self.GetFrameState( cache["stats"].frm_cnt - 1 - i - cache["stats"].last_drop_frames, cache=cache, ) if i != 0: self.DetectOneFrame( frame_state, cache["stats"].frm_cnt - 1 - i, False, cache=cache ) else: self.DetectOneFrame( frame_state, cache["stats"].frm_cnt - 1, True, cache=cache ) return 0 def DetectOneFrame( self, cur_frm_state: FrameState, cur_frm_idx: int, is_final_frame: bool, cache: dict = {}, ) -> None: tmp_cur_frm_state = FrameState.kFrameStateInvalid if cur_frm_state == FrameState.kFrameStateSpeech: if math.fabs(1.0) > self.vad_opts.fe_prior_thres: tmp_cur_frm_state = FrameState.kFrameStateSpeech else: tmp_cur_frm_state = FrameState.kFrameStateSil elif cur_frm_state == FrameState.kFrameStateSil: tmp_cur_frm_state = FrameState.kFrameStateSil state_change = cache["windows_detector"].DetectOneFrame( tmp_cur_frm_state, cur_frm_idx, cache=cache ) frm_shift_in_ms = self.vad_opts.frame_in_ms if AudioChangeState.kChangeStateSil2Speech == state_change: silence_frame_count = cache["stats"].continous_silence_frame_count cache["stats"].continous_silence_frame_count = 0 cache["stats"].pre_end_silence_detected = False start_frame = 0 if ( cache["stats"].vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected ): start_frame = max( cache["stats"].data_buf_start_frame, cur_frm_idx - self.LatencyFrmNumAtStartPoint(cache=cache), ) self.OnVoiceStart(start_frame, cache=cache) cache["stats"].vad_state_machine = ( VadStateMachine.kVadInStateInSpeechSegment ) for t in range(start_frame + 1, cur_frm_idx + 1): self.OnVoiceDetected(t, cache=cache) elif ( cache["stats"].vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment ): for t in range( cache["stats"].latest_confirmed_speech_frame + 1, cur_frm_idx ): self.OnVoiceDetected(t, cache=cache) if ( cur_frm_idx - cache["stats"].confirmed_start_frame + 1 > self.vad_opts.max_single_segment_time / frm_shift_in_ms ): self.OnVoiceEnd(cur_frm_idx, False, False, cache=cache) cache["stats"].vad_state_machine = ( VadStateMachine.kVadInStateEndPointDetected ) elif not is_final_frame: self.OnVoiceDetected(cur_frm_idx, cache=cache) else: self.MaybeOnVoiceEndIfLastFrame( is_final_frame, cur_frm_idx, cache=cache ) else: pass elif AudioChangeState.kChangeStateSpeech2Sil == state_change: cache["stats"].continous_silence_frame_count = 0 if ( cache["stats"].vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected ): pass elif ( cache["stats"].vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment ): if ( cur_frm_idx - cache["stats"].confirmed_start_frame + 1 > self.vad_opts.max_single_segment_time / frm_shift_in_ms ): self.OnVoiceEnd(cur_frm_idx, False, False, cache=cache) cache["stats"].vad_state_machine = ( VadStateMachine.kVadInStateEndPointDetected ) elif not is_final_frame: self.OnVoiceDetected(cur_frm_idx, cache=cache) else: self.MaybeOnVoiceEndIfLastFrame( is_final_frame, cur_frm_idx, cache=cache ) else: pass elif AudioChangeState.kChangeStateSpeech2Speech == state_change: cache["stats"].continous_silence_frame_count = 0 if ( cache["stats"].vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment ): if ( cur_frm_idx - cache["stats"].confirmed_start_frame + 1 > self.vad_opts.max_single_segment_time / frm_shift_in_ms ): cache["stats"].max_time_out = True self.OnVoiceEnd(cur_frm_idx, False, False, cache=cache) cache["stats"].vad_state_machine = ( VadStateMachine.kVadInStateEndPointDetected ) elif not is_final_frame: self.OnVoiceDetected(cur_frm_idx, cache=cache) else: self.MaybeOnVoiceEndIfLastFrame( is_final_frame, cur_frm_idx, cache=cache ) else: pass elif AudioChangeState.kChangeStateSil2Sil == state_change: cache["stats"].continous_silence_frame_count += 1 if ( cache["stats"].vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected ): # silence timeout, return zero length decision if ( ( self.vad_opts.detect_mode == VadDetectMode.kVadSingleUtteranceDetectMode.value ) and ( cache["stats"].continous_silence_frame_count * frm_shift_in_ms > self.vad_opts.max_start_silence_time ) ) or (is_final_frame and cache["stats"].number_end_time_detected == 0): for t in range( cache["stats"].lastest_confirmed_silence_frame + 1, cur_frm_idx ): self.OnSilenceDetected(t, cache=cache) self.OnVoiceStart(0, True, cache=cache) self.OnVoiceEnd(0, True, False, cache=cache) cache["stats"].vad_state_machine = ( VadStateMachine.kVadInStateEndPointDetected ) else: if cur_frm_idx >= self.LatencyFrmNumAtStartPoint(cache=cache): self.OnSilenceDetected( cur_frm_idx - self.LatencyFrmNumAtStartPoint(cache=cache), cache=cache, ) elif ( cache["stats"].vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment ): if ( cache["stats"].continous_silence_frame_count * frm_shift_in_ms >= cache["stats"].max_end_sil_frame_cnt_thresh ): lookback_frame = int( cache["stats"].max_end_sil_frame_cnt_thresh / frm_shift_in_ms ) if self.vad_opts.do_extend: lookback_frame -= int( self.vad_opts.lookahead_time_end_point / frm_shift_in_ms ) lookback_frame -= 1 lookback_frame = max(0, lookback_frame) self.OnVoiceEnd( cur_frm_idx - lookback_frame, False, False, cache=cache ) cache["stats"].vad_state_machine = ( VadStateMachine.kVadInStateEndPointDetected ) elif ( cur_frm_idx - cache["stats"].confirmed_start_frame + 1 > self.vad_opts.max_single_segment_time / frm_shift_in_ms ): self.OnVoiceEnd(cur_frm_idx, False, False, cache=cache) cache["stats"].vad_state_machine = ( VadStateMachine.kVadInStateEndPointDetected ) elif self.vad_opts.do_extend and not is_final_frame: if cache["stats"].continous_silence_frame_count <= int( self.vad_opts.lookahead_time_end_point / frm_shift_in_ms ): self.OnVoiceDetected(cur_frm_idx, cache=cache) else: self.MaybeOnVoiceEndIfLastFrame( is_final_frame, cur_frm_idx, cache=cache ) else: pass if ( cache["stats"].vad_state_machine == VadStateMachine.kVadInStateEndPointDetected and self.vad_opts.detect_mode == VadDetectMode.kVadMutipleUtteranceDetectMode.value ): self.ResetDetection(cache=cache)