File size: 1,316 Bytes
748b6ee |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 |
# ingest.py
from sentence_transformers import SentenceTransformer
import whisper
import os
import chromadb
from chromadb.config import Settings
from chromadb.utils import embedding_functions
# Initialize models
whisper_model = whisper.load_model("base") # or "medium", "large" depending on system resources
embedding_model = SentenceTransformer("all-MiniLM-L6-v2")
# Set up Chroma DB
client = chromadb.Client(Settings(chroma_db_impl="duckdb+parquet", persist_directory="chroma_db"))
collection = client.get_or_create_collection(name="meeting_notes",
embedding_function=embedding_functions.SentenceTransformerEmbeddingFunction(embedding_model))
def transcribe_and_ingest(video_path, meeting_id):
print(f"Transcribing {video_path}...")
result = whisper_model.transcribe(video_path)
transcription = result["text"]
print("Splitting transcription...")
chunks = [transcription[i:i+500] for i in range(0, len(transcription), 500)]
ids = [f"{meeting_id}_{i}" for i in range(len(chunks))]
print("Storing embeddings in Chroma DB...")
collection.add(documents=chunks, ids=ids, metadatas=[{"meeting_id": meeting_id}] * len(chunks))
return transcription # return full transcription for PDF or future use
|