update strategy

main.py

@@ -65,7 +65,7 @@ async def translate(websocket: WebSocket):
     )
 
     if from_lang and to_lang:
-        client.set_lang(from_lang, to_lang)
+        client.set_language(from_lang, to_lang)
         logger.info(f"Source lange: {from_lang}  -> Dst lange: {to_lang}")
         await websocket.accept()
     try:

transcribe/strategy.py

@@ -18,6 +18,9 @@ class TranscriptSegment:
     t0: float  # 开始时间（百分之一秒）
     t1: float  # 结束时间（百分之一秒）
 
+def join_segment_text(segments: List[TranscriptSegment], separator: str = "") -> str:
+    """连接多个片段的文本"""
+    return separator.join(seg.text for seg in segments)
 
 class TextStabilityBuffer:
     """
@@ -77,12 +80,26 @@ class TranscriptionManager:
         self._committed_segments: List[str] = []  # 确认的完整段落
         self._committed_sentences: List[str] = []  # 确认的短句
         self._temp_string: str = ""  # 临时字符串缓冲
+    
+    def check_line_break(self, min_length: int = 20) -> bool:
+        """检查当前短句长度是否达到换行标准"""
+        return self.sentence_length >= min_length
+
+    def force_line_break(self) -> None:
+        """强制换行，保留当前内容但创建新段落"""
+        if self.current_sentence:
+            self._committed_segments.append(self.current_sentence)
+            self._committed_sentences = []
         
     @property
     def current_sentence(self) -> str:
         """当前已确认的短句组合"""
         return "".join(self._committed_sentences)
     
+    @property
+    def remaining_text(self) -> str:
+        return self._temp_string
+    
     @property
     def latest_segment(self) -> str:
         """最新确认的完整段落"""
@@ -153,8 +170,8 @@ class TranscriptionSplitter:
     @staticmethod
     def split_by_punctuation(
         segments: List[TranscriptSegment], 
-        audio_buffer: np.ndarray, 
-        sample_rate: int = 16000
+        sample_rate: int = 16000,
+        segment_skip_index=  0
     ) -> Tuple[int, List[TranscriptSegment], List[TranscriptSegment], bool]:
         """
         根据标点符号将片段分为左侧（已确认）和右侧（待确认）
@@ -167,24 +184,26 @@ class TranscriptionSplitter:
         split_index = 0
         is_sentence_end = False
         
-        # 短音频使用所有标点符号作为分割依据
-        buffer_duration = len(audio_buffer) / sample_rate
-        markers = ALL_MARKERS if buffer_duration < 12 else SENTENCE_END_MARKERS
-        
-        for idx, seg in enumerate(segments):
+        # # 短音频使用所有标点符号作为分割依据
+        # buffer_duration = len(audio_buffer) / sample_rate
+        # markers = ALL_MARKERS if buffer_duration < 12 else SENTENCE_END_MARKERS
+        skip_segments = segments[:segment_skip_index+1]
+        skipped_segments = segments[segment_skip_index:]
+
+        markers = ALL_MARKERS
+        for idx, seg in enumerate(skipped_segments):
             left_segments.append(seg)
             if seg.text and seg.text[-1] in markers:
                 split_index = int(seg.t1 / 100 * sample_rate)
                 is_sentence_end = bool(SENTENCE_END_PATTERN.search(seg.text))
-                right_segments = segments[min(idx+1, len(segments)):]
+                right_segments = skipped_segments[min(idx+1, len(skipped_segments)):]
                 break
-                
+        left_segments = skip_segments+ left_segments
         return split_index, left_segments, right_segments, is_sentence_end
     
     @staticmethod
     def split_by_sequences(
         segments: List[TranscriptSegment], 
-        audio_buffer: np.ndarray, 
         sample_rate: int = 16000
     ) -> Tuple[int, Iterator[TranscriptSegment], Iterator[TranscriptSegment], bool]:
         """
@@ -210,14 +229,32 @@ class TranscriptionSplitter:
         return 0, iter([]), iter(segments), False
 
 
-class TranscriptionStabilizer:
+class TranscriptionStabilizer(TranscriptionSplitter):
     """
     转录结果稳定器，负责确认和管理转录片段
     """
     def __init__(self, sample_rate: int = 16000):
-        self.manager = TranscriptionManager()
-        self.stability_buffer = TextStabilityBuffer(max_history=2)
+        self.text_manager = TranscriptionManager()
         self.sample_rate = sample_rate
+
+    @property
+    def latest_segment(self):
+        return self.text_manager.latest_segment
+
+
+    @property
+    def segment_count(self):
+        return self.text_manager.segment_count
+    
+
+    @property
+    def remaining_text(self):
+        return self.text_manager.remaining_text
+    
+    @property
+    def stable_string(self):
+        return self.text_manager.current_sentence
+
         
     def process_segments(self, segments: List[TranscriptSegment]) -> Tuple[Optional[int], bool]:
         """
@@ -232,49 +269,37 @@ class TranscriptionStabilizer:
         # 查找第一个包含标点的片段作为分割点
         split_index = None
         stable_segments = []
+        force_split = False
+        if len(segments) < 20:
+            remaining_text = join_segment_text(segments)
+            self.text_manager.update_temp(remaining_text)
+            return split_index, False, join_segment_text(segments), self.text_manager.remaining_text
         
-        for idx, seg in enumerate(segments):
-            stable_segments.append(seg)
-            if REGEX_MARKERS.search(seg.text):
-                split_index = int(seg.t1 / 100 * self.sample_rate)
-                stable_idx = min(idx + 1, len(segments))
-                break
+        # 查找20个长度后的标点符号
+        split_index, left_segments, right_segments, is_sentence_end = self.split_by_punctuation(segments[20:],sample_rate=self.sample_rate)
         
-        if split_index:  # 找到标点，确认标点前的内容
-            stable_text = self._join_segment_text(segments[:stable_idx])
-            self.manager.update_temp(stable_text).commit_sentence()
+        if split_index is not None:  # 找到标点，确认标点前的内容
+            stable_text = join_segment_text(left_segments)
+            self.text_manager.update_temp(stable_text).commit_sentence()
             
             # 更新剩余文本
-            remaining_text = self._join_segment_text(segments[stable_idx:])
-            self.manager.update_temp(remaining_text)
+            remaining_text = join_segment_text(right_segments)
+            self.text_manager.update_temp(remaining_text)
         else:
-            # 没有找到标点，全部作为临时文本
-            self.manager.update_temp(self._join_segment_text(segments))
+            # 如果没有标点 但是累计超过22个字符 直接从20个字符的位置切掉
+            if len(segments) > 22 and not REGEX_MARKERS.search(join_segment_text(segments)):
+                split_index = int(segments[20].t1 / 100 * self.sample_rate)
+                stable_idx = 21  # 直接使用22个字符的索引
+                force_split = True
+                stable_text = join_segment_text(segments[:stable_idx])
+                self.text_manager.update_temp(stable_text).commit_sentence()
+                self.text_manager.update_temp(join_segment_text(segments[stable_idx:]))
+            else:
+                # 没有找到标点，全部作为临时文本
+                self.text_manager.update_temp(join_segment_text(segments))
             
         # 检查是否达到换行标准
-        should_linebreak = self.manager.sentence_length >= 20
-        
-        return split_index, should_linebreak
-    
-    def check_stability(self, text: str, index: int) -> Optional[int]:
-        """
-        检查文本是否稳定
+        should_linebreak = self.text_manager.sentence_length >= 20 or force_split
         
-        Args:
-            text: 当前文本
-            index: 当前索引
-            
-        Returns:
-            如果文本稳定，返回稳定的索引；否则返回None
-        """
-        self.stability_buffer.add_entry(text, index)
-        return self.stability_buffer.get_stable_index()
-    
-    def commit_segment(self, is_end_of_sentence: bool) -> None:
-        """提交转录片段"""
-        self.manager.commit_segment(is_end_of_sentence)
-    
-    @staticmethod
-    def _join_segment_text(segments: List[TranscriptSegment], separator: str = "") -> str:
-        """连接多个片段的文本"""
-        return separator.join(seg.text for seg in segments)
+        return split_index, should_linebreak, join_segment_text(stable_segments), self.text_manager.remaining_text
+ 

transcribe/whisper_llm_serve.py

@@ -12,7 +12,13 @@ from api_model import TransResult, Message
 from .server import ServeClientBase
 from .utils import log_block, save_to_wave
 from .translatepipes import TranslatePipes
-from .strategy import TextStabilityBuffer, TranscriptionManager, TranscriptionSplitter, TranscriptSegment
+from .strategy import (
+    TextStabilityBuffer, 
+    TranscriptionManager, 
+    TranscriptionSplitter, 
+    TranscriptSegment,
+    TranscriptionStabilizer,
+    join_segment_text)
 
 logger = getLogger("TranscriptionService")
 
@@ -50,6 +56,8 @@ class WhisperTranscriptionService(ServeClientBase):
         self.translate_thread = self._start_thread(self._transcription_processing_loop)
         self.frame_processing_thread = self._start_thread(self._frame_processing_loop)
 
+        self.text_stablizer = TranscriptionStabilizer()
+
     def _start_thread(self, target_function) -> threading.Thread:
         """启动守护线程执行指定函数"""
         thread = threading.Thread(target=target_function)
@@ -154,11 +162,26 @@ class WhisperTranscriptionService(ServeClientBase):
         result = self._translate_pipe.translate(text, self.source_language, self.target_language)
         translated_text = result.translate_content
         
-        log_block("Translation time", f"{(time.perf_counter() - start_time):.3f}", "s")
+        log_block("Translation time ", f"{(time.perf_counter() - start_time):.3f}", "s")
         log_block("Translation output", f"{translated_text}")
         
         return translated_text
 
+    def _find_best_split_position(self, segments: list, target_length: int = 20) -> int:
+        """找到最适合分割的位置，尽量靠近目标长度且在词/字的边界"""
+        if len(segments) <= target_length:
+            return 0
+            
+        # 从目标长度位置向前搜索适合的分割点
+        for i in range(target_length, min(target_length + 10, len(segments))):
+            # 对于中文，每个字符都可以作为分割点
+            # 对于英文，在空格处分割
+            if self.source_language == "zh" or segments[i] == " ":
+                return i
+                
+        # 如果找不到理想分割点，就在目标长度处分割
+        return target_length
+
     def _analyze_segments(self, segments: List[TranscriptSegment], audio_buffer: np.ndarray) -> Tuple[Optional[int], str, str, bool]:
         """
         分析转录片段，确定稳定部分和需要继续观察的部分
@@ -171,24 +194,35 @@ class WhisperTranscriptionService(ServeClientBase):
             segments, audio_buffer, self.sample_rate
         )
         
-        left_text = self.text_separator.join(seg.text for seg in left_segments)
-        right_text = self.text_separator.join(seg.text for seg in right_segments)
+        left_text = join_segment_text(left_segments, self.text_separator)
+        right_text =  join_segment_text(right_segments, self.text_separator)
         
         # 如果找到分割点，检查左侧文本稳定性
         if left_idx != 0:
             self._text_stability_buffer.add_entry(left_text, left_idx)
             stable_idx = self._text_stability_buffer.get_stable_index()
             if stable_idx:
-                return stable_idx, left_text, right_text, is_end
-        
+                should_break = True if (self._transcription_manager.sentence_length>= 20) else False
+                return stable_idx, left_text, right_text, should_break
+
+        # 如果基于标点的方法没有找到稳定点，尝试检查句子的长度
+        if len(segments) >= 20:  # 设置更长的阈值，确保有足够内容进行分割
+            # 尝试在约20字符处找一个词的边界进行分割
+            split_pos = self._find_best_split_position(segments)
+            if split_pos > 0:
+                left_text = join_segment_text(segments[:split_pos], self.text_separator)
+                right_text = join_segment_text(segments[split_pos:], self.text_separator)
+                audio_pos = int(segments[split_pos].t1 / 100 * self.sample_rate)
+                return audio_pos, left_text, right_text, True
+
         # 如果基于标点的方法未找到稳定点，尝试基于句子序列的方法
         left_idx, left_segments, right_segments, is_end = TranscriptionSplitter.split_by_sequences(
-            segments, audio_buffer, self.sample_rate
+            segments, self.sample_rate
         )
         
         if left_idx != 0:
-            left_text = self.text_separator.join(seg.text for seg in left_segments)
-            right_text = self.text_separator.join(seg.text for seg in right_segments)
+            left_text = join_segment_text(left_segments, self.text_separator)
+            right_text =  join_segment_text(right_segments, self.text_separator)
             return left_idx, left_text, right_text, is_end
         
         # 如果都没有找到分割点
@@ -196,6 +230,7 @@ class WhisperTranscriptionService(ServeClientBase):
 
     def _transcription_processing_loop(self) -> None:
         """主转录处理循环"""
+        c = 0
         while not self._translate_thread_stop.is_set():
             if self.exit:
                 logger.info("Exiting transcription thread")
@@ -203,26 +238,28 @@ class WhisperTranscriptionService(ServeClientBase):
                 
             # 等待音频数据
             if self.frames_np is None:
-                time.sleep(0.02)
+                time.sleep(0.2)
                 logger.info("Waiting for audio data...")
                 continue
                 
             # 获取音频块进行处理
             audio_buffer = self._get_audio_for_processing()
             if audio_buffer is None:
-                time.sleep(0.02)
+                time.sleep(0.2)
                 continue
-                
-            try:
-                logger.info(f"Processing audio buffer: {len(audio_buffer)/self.sample_rate:.2f}s")
-                segments = self._transcribe_audio(audio_buffer)
-                
-                # 处理转录结果并发送到客户端
-                for result in self._process_transcription_results(segments, audio_buffer):
-                    self._send_result_to_client(result)
+
+            c+= 1
+            save_to_wave(f"dev-{c}.wav", audio_buffer)
+            
+            # try:
+            segments = self._transcribe_audio(audio_buffer)
+            
+            # 处理转录结果并发送到客户端
+            for result in self._process_transcription_results(segments, audio_buffer):
+                self._send_result_to_client(result)
                     
-            except Exception as e:
-                logger.error(f"Error processing audio: {e}")
+            # except Exception as e:
+            #     logger.error(f"Error processing audio: {e}")
 
     def _process_transcription_results(self, segments: List[TranscriptSegment], audio_buffer: np.ndarray) -> Iterator[TransResult]:
         """
@@ -236,12 +273,7 @@ class WhisperTranscriptionService(ServeClientBase):
         if not full_text:
             return
             
-        # 更新转录管理器中的临时文本
-        self._transcription_manager.update_temp(full_text)
-        
-        # 分析片段，确定稳定部分和需要继续观察的部分
-        cut_index, stable_text, remaining_text, is_sentence_end = self._analyze_segments(segments, audio_buffer)
-        
+        cut_index, is_sentence_end, stable_text, remaining_text = self.text_stablizer.process_segments(segments)
         # 如果找到稳定的分割点
         if cut_index:
             # 更新音频缓冲区，移除已处理部分
@@ -249,13 +281,11 @@ class WhisperTranscriptionService(ServeClientBase):
             
             # 提交稳定的文本
             log_block("Stable transcription", f"{stable_text}")
-            self._transcription_manager.update_temp(stable_text).commit_segment(is_sentence_end)
-            self._transcription_manager.update_temp(remaining_text)
-            
+
             # 如果是句子结束，发送完整句子的翻译结果
             if is_sentence_end:
-                segment_text = self._transcription_manager.latest_segment
-                segment_id = self._transcription_manager.segment_count - 1
+                segment_text = self.text_stablizer.latest_segment
+                segment_id = self.text_stablizer.segment_count - 1
                 
                 # 生成已确认句子的翻译结果
                 yield TransResult(
@@ -268,19 +298,19 @@ class WhisperTranscriptionService(ServeClientBase):
                 )
                 
                 # 如果还有剩余部分，生成临时翻译结果
-                if self._transcription_manager.current_sentence.strip():
+                if self.text_stablizer.remaining_text.strip():
                     yield TransResult(
                         seg_id=segment_id + 1,
-                        context=self._transcription_manager.current_sentence,
+                        context=self.text_stablizer.remaining_text,
                         from_=self.source_language,
                         to=self.target_language,
-                        tran_content=self._translate_text(self._transcription_manager.current_sentence.strip()),
+                        tran_content=self._translate_text(self.text_stablizer.remaining_text.strip()),
                         partial=True
                     )
         else:
             # 没有找到稳定点，发送当前所有内容的临时翻译结果
-            segment_id = self._transcription_manager.segment_count
-            current_text = self._transcription_manager.current_sentence + self._transcription_manager.update_temp(remaining_text)._temp_string
+            segment_id = self.text_stablizer.segment_count
+            current_text = self.text_stablizer.stable_string +  self.text_stablizer.remaining_text
             
             yield TransResult(
                 seg_id=segment_id,