scripts/chunk_and_embed.py

from langchain.text_splitter import RecursiveCharacterTextSplitter
from pathlib import Path
import pickle
import os

def main():
    BASE_DIR = Path(__file__).resolve().parent.parent
    DOCS_PATH = BASE_DIR / "output" / "all_docs.pkl"
    CHUNKS_PATH = BASE_DIR / "output" / "chunks.pkl"
    
    with open(DOCS_PATH, "rb") as f:
        all_docs = pickle.load(f)
    
    splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
    chunks = splitter.split_documents(all_docs)
    print(f"✅ Split into {len(chunks)} chunks")
    
    with open(CHUNKS_PATH, "wb") as f:
        pickle.dump(chunks, f)
    print(f"📦 Chunks saved to {CHUNKS_PATH}")

if __name__ == "__main__":
    main()