import { InferenceClient } from "@huggingface/inference"; import { StateGraph, Annotation, START, END } from "@langchain/langgraph"; import { HumanMessage, AIMessage, BaseMessage, ToolMessage, } from "@langchain/core/messages"; import { observeConsoleTool } from "./tools"; import { mcpClientManager, setMCPWebSocketConnection } from "./mcp-client"; import { planTasksTool, updateTaskTool, viewTasksTool } from "./task-tracker"; import { documentationService } from "./documentation"; import { StreamingToolCallParser, extractToolCalls, } from "../utils/tool-call-parser"; import { virtualFileSystem } from "../services/virtual-fs"; import type { WebSocket } from "ws"; const AgentState = Annotation.Root({ messages: Annotation({ reducer: (x, y) => x.concat(y), }), hasToolCalls: Annotation({ reducer: (_, y) => y, default: () => false, }), }); export class LangGraphAgent { private client: InferenceClient | null = null; private graph!: ReturnType; private model: string = "Qwen/Qwen3-Next-80B-A3B-Instruct"; private documentation: string = ""; private ws: WebSocket | null = null; constructor() { this.setupGraph(); } async initialize(hfToken: string, ws?: WebSocket) { if (!hfToken) { throw new Error("Hugging Face authentication required"); } this.client = new InferenceClient(hfToken); if (ws) { this.ws = ws; setMCPWebSocketConnection(ws); } await mcpClientManager.initialize(); const docs = await documentationService.load(); this.documentation = docs || ""; } private setupGraph() { const graph = new StateGraph(AgentState); graph.addNode("agent", async (state, config) => { const systemPrompt = this.buildSystemPrompt(); const messages = this.formatMessages(state.messages, systemPrompt); let fullResponse = ""; const parser = new StreamingToolCallParser(); let currentSegmentId: string | null = null; const messageId = config?.metadata?.messageId; const abortSignal = config?.metadata?.abortSignal as | AbortSignal | undefined; for await (const token of this.streamModelResponse( messages, abortSignal, )) { fullResponse += token; config?.writer?.({ type: "token", content: token }); // Process token through the parser const parseResult = parser.process(token); // Only stream safe text (without tool calls) if (parseResult.safeText) { // Start a text segment if needed if (!currentSegmentId && parseResult.safeText.trim() && this.ws) { currentSegmentId = `seg_${Date.now()}_${Math.random()}`; this.ws.send( JSON.stringify({ type: "segment_start", payload: { segmentId: currentSegmentId, segmentType: "text", messageId, }, timestamp: Date.now(), }), ); } // Stream the safe text if (currentSegmentId && this.ws) { this.ws.send( JSON.stringify({ type: "segment_token", payload: { segmentId: currentSegmentId, token: parseResult.safeText, messageId, }, timestamp: Date.now(), }), ); } } // If we detected a tool call start, end the current text segment if (parseResult.pendingToolCall && currentSegmentId && this.ws) { this.ws.send( JSON.stringify({ type: "segment_end", payload: { segmentId: currentSegmentId, messageId, }, timestamp: Date.now(), }), ); currentSegmentId = null; } } // Finalize any remaining text segment if (currentSegmentId && this.ws) { // Process any remaining buffered content const remainingBuffer = parser.getBuffer(); if (remainingBuffer && !parser.isInToolCall()) { const finalResult = parser.process(""); if (finalResult.safeText) { this.ws.send( JSON.stringify({ type: "segment_token", payload: { segmentId: currentSegmentId, token: finalResult.safeText, messageId, }, timestamp: Date.now(), }), ); } } this.ws.send( JSON.stringify({ type: "segment_end", payload: { segmentId: currentSegmentId, messageId, }, timestamp: Date.now(), }), ); } const toolCalls = extractToolCalls(fullResponse); if (toolCalls.length > 0) { const toolResults = await this.executeToolsWithSegments( toolCalls, config?.metadata?.messageId as string | undefined, ); return { messages: [ new AIMessage({ content: fullResponse, additional_kwargs: { has_tool_calls: true }, }), ...toolResults, ], hasToolCalls: true, }; } if (state.messages.length > 0) { const lastUserMessage = state.messages[state.messages.length - 1]; if (lastUserMessage instanceof HumanMessage) { const needsTools = this.shouldUseTools( lastUserMessage.content as string, ); if (needsTools && !state.hasToolCalls) { const reminderMessage = new AIMessage( "I need to use tools to complete this task. Let me try again with the appropriate tool.", ); return { messages: [reminderMessage], hasToolCalls: false, }; } } } return { messages: [new AIMessage(fullResponse)], hasToolCalls: false, }; }); // @ts-expect-error - LangGraph type mismatch with START constant graph.addEdge(START, "agent"); // @ts-expect-error - LangGraph type mismatch with conditional edges graph.addConditionalEdges("agent", (state) => this.shouldContinue(state), { continue: "agent", end: END, }); this.graph = graph.compile(); } private shouldContinue(state: typeof AgentState.State): string { const lastMessage = state.messages[state.messages.length - 1]; if (lastMessage instanceof ToolMessage) { return "continue"; } if ( lastMessage instanceof AIMessage && lastMessage.additional_kwargs?.has_tool_calls ) { return "continue"; } return "end"; } private buildSystemPrompt(): string { return `## Role & Primary Guidance You are a VibeGame Engine Specialist operating in a single-file editor environment. You work with ONE game file that users can see and edit in real-time, similar to JSFiddle or CodePen. ## Live Coding Environment Context - **SINGLE EDITOR**: There is exactly ONE editor file containing the game's XML/HTML code - **"The code" ALWAYS refers to**: The content currently in the editor - **Live Preview**: Changes to the editor automatically reload the game - **User's View**: Users see the same editor you're modifying - **GAME is PRE-IMPORTED**: The GAME object is automatically available - NEVER write \`import * as GAME from 'vibegame'\` - **Auto-run enabled**: GAME.run() is called automatically - use \`GAME.withPlugin(MyPlugin)\` NOT \`GAME.withPlugin(MyPlugin).run()\` ## VibeGame Expert Knowledge ${this.documentation} ## MCP Tool Execution Protocol ### Format Requirements - MANDATORY format: {"param": "value"} - JSON arguments must be valid JSON (use double quotes for strings) - **CRITICAL**: Execute ONE tool at a time and wait for the result - The parser will handle unclosed tags gracefully for recovery - **NEVER** generate multiple tool calls in a single response - After each tool execution, analyze the result before deciding next action - ALWAYS check console after making changes - The game auto-reloads after editor changes ### Tool Execution Rules 1. Execute a single tool 2. Read and analyze the tool's response 3. Only then decide if another tool is needed 4. If multiple edits are needed, use plan_tasks first to organize them ### Available MCP Tools (All operate on the SINGLE editor file) #### Understanding the Editor Content - search_editor: Find text/patterns in the current game code Parameters: - query: string - Text or regex pattern to search for - mode: "text" | "regex" - Search mode (optional, default: "text") - contextLines: number - Lines of context (optional, default: 2, max: 5) Example: {"query": "dynamic-part", "mode": "text"} - read_editor_lines: Read specific lines from the current game code Parameters: - startLine: number - Starting line (1-indexed) - endLine: number - Ending line (optional, defaults to startLine) Example: {"startLine": 10, "endLine": 20} - read_editor: Read the complete game code currently in the editor Parameters: none Example: {} Note: Use when user asks to "explain the code", "show the code", or needs full context #### Modifying the Game - edit_editor: Make targeted changes to the game code Parameters: - oldText: string - Exact text to find and replace (max ~20 lines). MUST include actual content, not just whitespace - newText: string - Replacement text Example: {"oldText": "color='#ff0000'", "newText": "color='#00ff00'"} IMPORTANT: Always include meaningful content (tags, comments, or code) in oldText, never just spaces/newlines Note: Returns standardized response with game status and console output - write_editor: Replace all game code with new content Parameters: - content: string - Complete new game code Example: {"content": "..."} Note: Use ONLY for starting fresh or complete rewrites #### Task Management - plan_tasks: Create task list for complex operations Parameters: - tasks: string[] - Array of task descriptions Example: {"tasks": ["Add physics component", "Create gravity system", "Test gravity effect"]} - update_task: Update task status Parameters: - taskId: number - Task ID to update - status: "pending" | "in_progress" | "completed" Example: {"taskId": 1, "status": "completed"} - view_tasks: View current task list Parameters: none Example: {} #### Runtime Monitoring - observe_console: Check console messages and game state Parameters: none Example: {} Note: Returns recent console messages and game status ## Tool Response Format All editor modification tools return: - Success indicator (✅ or ❌) - Action description - Game status (Running/Error/Loading) - Console output from the game ## Common User Requests (Context Clarification) When users say: - "explain the code" → Read and explain the CURRENT editor content - "what's in the game?" → Describe the entities/elements in the editor - "fix the error" → Check console, then modify the editor content - "add a..." → Add new elements to the existing editor content - "change the..." → Modify existing elements in the editor ## Execution Patterns ### Code Writing Rules (Live Environment) - **NO IMPORTS**: GAME is pre-imported globally - NEVER write \`import * as GAME from 'vibegame'\` - **NO .run()**: Auto-run is enabled - write \`GAME.withPlugin(MyPlugin)\` not \`GAME.withPlugin(MyPlugin).run()\` - **Direct access**: Use GAME.defineComponent, GAME.Types, etc. directly - **Script tags**: When adding JavaScript, use \`