I think we're close to getting tooling to work

close?

src/public/app/widgets/llm_chat_panel.ts

@@ -176,72 +176,82 @@ export default class LlmChatPanel extends BasicWidget {
     }
 
     /**
-     * Load saved chat data from the note
+     * Save current chat data to the note attribute
-     */
-    async loadSavedData() {
-        if (!this.onGetData) {
-            console.log("No getData callback available");
-            return;
-        }
-
-        try {
-            const data = await this.onGetData();
-            console.log(`Loading chat data for noteId: ${this.currentNoteId}`, data);
-
-            // Make sure we're loading data for the correct note
-            if (data && data.noteId && data.noteId !== this.currentNoteId) {
-                console.warn(`Data noteId ${data.noteId} doesn't match current noteId ${this.currentNoteId}`);
-            }
-
-            if (data && data.messages && Array.isArray(data.messages)) {
-                // Clear existing messages in the UI
-                this.noteContextChatMessages.innerHTML = '';
-                this.messages = [];
-
-                // Add each message to the UI
-                data.messages.forEach((message: {role: string; content: string}) => {
-                    if (message.role === 'user' || message.role === 'assistant') {
-                        this.addMessageToChat(message.role, message.content);
-                        // Track messages in our local array too
-                        this.messages.push(message);
-                    }
-                });
-
-                // Scroll to bottom
-                this.chatContainer.scrollTop = this.chatContainer.scrollHeight;
-                console.log(`Successfully loaded ${data.messages.length} messages for noteId: ${this.currentNoteId}`);
-
-                return true;
-            }
-        } catch (e) {
-            console.error(`Error loading saved chat data for noteId: ${this.currentNoteId}:`, e);
-        }
-
-        return false;
-    }
-
-    /**
-     * Save the current chat data to the note
      */
     async saveCurrentData() {
         if (!this.onSaveData) {
-            console.log("No saveData callback available");
             return;
         }
 
         try {
-            // Include the current note ID for tracking purposes
+            const dataToSave = {
-            await this.onSaveData({
                 messages: this.messages,
-                lastUpdated: new Date(),
+                sessionId: this.sessionId
-                noteId: this.currentNoteId // Include the note ID to help with debugging
+            };
-            });
+
-            console.log(`Saved chat data for noteId: ${this.currentNoteId} with ${this.messages.length} messages`);
+            console.log(`Saving chat data with sessionId: ${this.sessionId}`);
-            return true;
+
-        } catch (e) {
+            await this.onSaveData(dataToSave);
-            console.error(`Error saving chat data for noteId: ${this.currentNoteId}:`, e);
+        } catch (error) {
+            console.error('Failed to save chat data', error);
+        }
+    }
+
+    /**
+     * Load saved chat data from the note attribute
+     */
+    async loadSavedData(): Promise<boolean> {
+        if (!this.onGetData) {
             return false;
         }
+
+        try {
+            const savedData = await this.onGetData();
+
+            if (savedData?.messages?.length > 0) {
+                // Load messages
+                this.messages = savedData.messages;
+
+                // Clear and rebuild the chat UI
+                this.noteContextChatMessages.innerHTML = '';
+
+                this.messages.forEach(message => {
+                    const role = message.role as 'user' | 'assistant';
+                    this.addMessageToChat(role, message.content);
+                });
+
+                // Load session ID if available
+                if (savedData.sessionId) {
+                    try {
+                        // Verify the session still exists
+                        const sessionCheck = await server.get<any>(`llm/sessions/${savedData.sessionId}`);
+
+                        if (sessionCheck && sessionCheck.id) {
+                            console.log(`Restored session ${savedData.sessionId}`);
+                            this.sessionId = savedData.sessionId;
+                        } else {
+                            console.log(`Saved session ${savedData.sessionId} not found, will create new one`);
+                            this.sessionId = null;
+                            await this.createChatSession();
+                        }
+                    } catch (error) {
+                        console.log(`Error checking saved session ${savedData.sessionId}, will create new one`);
+                        this.sessionId = null;
+                        await this.createChatSession();
+                    }
+                } else {
+                    // No saved session ID, create a new one
+                    this.sessionId = null;
+                    await this.createChatSession();
+                }
+
+                return true;
+            }
+        } catch (error) {
+            console.error('Failed to load saved chat data', error);
+        }
+
+        return false;
     }
 
     async refresh() {
@@ -301,13 +311,38 @@ export default class LlmChatPanel extends BasicWidget {
      * Handle sending a user message to the LLM service
      */
     private async sendMessage(content: string) {
-        if (!content.trim() || !this.sessionId) {
+        if (!content.trim()) {
             return;
         }
 
         // Check for provider validation issues before sending
         await this.validateEmbeddingProviders();
 
+        // Make sure we have a valid session
+        if (!this.sessionId) {
+            // If no session ID, create a new session
+            await this.createChatSession();
+
+            if (!this.sessionId) {
+                // If still no session ID, show error and return
+                console.error("Failed to create chat session");
+                toastService.showError("Failed to create chat session");
+                return;
+            }
+        } else {
+            // Verify the session exists on the server
+            try {
+                const sessionCheck = await server.get<any>(`llm/sessions/${this.sessionId}`);
+                if (!sessionCheck || !sessionCheck.id) {
+                    console.log(`Session ${this.sessionId} not found, creating a new one`);
+                    await this.createChatSession();
+                }
+            } catch (error) {
+                console.log(`Error checking session ${this.sessionId}, creating a new one`);
+                await this.createChatSession();
+            }
+        }
+
         // Process the user message
         await this.processUserMessage(content);
 
@@ -321,7 +356,7 @@ export default class LlmChatPanel extends BasicWidget {
             const showThinking = this.showThinkingCheckbox.checked;
 
             // Add logging to verify parameters
-            console.log(`Sending message with: useAdvancedContext=${useAdvancedContext}, showThinking=${showThinking}, noteId=${this.currentNoteId}`);
+            console.log(`Sending message with: useAdvancedContext=${useAdvancedContext}, showThinking=${showThinking}, noteId=${this.currentNoteId}, sessionId=${this.sessionId}`);
 
             // Create the message parameters
             const messageParams = {

src/services/llm/ai_service_manager.ts

@@ -1,14 +1,14 @@
 import options from '../options.js';
-import type { AIService, ChatCompletionOptions, ChatResponse, Message, SemanticContextService } from './ai_interface.js';
+import type { AIService, ChatCompletionOptions, ChatResponse, Message } from './ai_interface.js';
-import { OpenAIService } from './providers/openai_service.js';
 import { AnthropicService } from './providers/anthropic_service.js';
-import { OllamaService } from './providers/ollama_service.js';
-import log from '../log.js';
 import { ContextExtractor } from './context/index.js';
-import contextService from './context_service.js';
-import indexService from './index_service.js';
-import { getEmbeddingProvider, getEnabledEmbeddingProviders } from './providers/providers.js';
 import agentTools from './context_extractors/index.js';
+import contextService from './context/services/context_service.js';
+import { getEmbeddingProvider, getEnabledEmbeddingProviders } from './providers/providers.js';
+import indexService from './index_service.js';
+import log from '../log.js';
+import { OllamaService } from './providers/ollama_service.js';
+import { OpenAIService } from './providers/openai_service.js';
 
 // Import interfaces
 import type {
@@ -277,7 +277,7 @@ export class AIServiceManager implements IAIServiceManager {
      */
     async initializeAgentTools(): Promise<void> {
         try {
-            await agentTools.initialize(this);
+            await agentTools.initialize(true);
             log.info("Agent tools initialized successfully");
         } catch (error: any) {
             log.error(`Error initializing agent tools: ${error.message}`);
@@ -296,28 +296,32 @@ export class AIServiceManager implements IAIServiceManager {
      * Get the vector search tool for semantic similarity search
      */
     getVectorSearchTool() {
-        return agentTools.getVectorSearchTool();
+        const tools = agentTools.getTools();
+        return tools.vectorSearch;
     }
 
     /**
      * Get the note navigator tool for hierarchical exploration
      */
     getNoteNavigatorTool() {
-        return agentTools.getNoteNavigatorTool();
+        const tools = agentTools.getTools();
+        return tools.noteNavigator;
     }
 
     /**
      * Get the query decomposition tool for complex queries
      */
     getQueryDecompositionTool() {
-        return agentTools.getQueryDecompositionTool();
+        const tools = agentTools.getTools();
+        return tools.queryDecomposition;
     }
 
     /**
      * Get the contextual thinking tool for transparent reasoning
      */
     getContextualThinkingTool() {
-        return agentTools.getContextualThinkingTool();
+        const tools = agentTools.getTools();
+        return tools.contextualThinking;
     }
 
     /**
@@ -391,7 +395,7 @@ export class AIServiceManager implements IAIServiceManager {
             await this.getIndexService().initialize();
 
             // Initialize agent tools with this service manager instance
-            await agentTools.initialize(this);
+            await agentTools.initialize(true);
 
             // Initialize LLM tools - this is the single place where tools are initialized
             const toolInitializer = await import('./tools/tool_initializer.js');
@@ -407,24 +411,95 @@ export class AIServiceManager implements IAIServiceManager {
     }
 
     /**
-     * Get context from agent tools
+     * Get description of available agent tools
+     */
+    async getAgentToolsDescription(): Promise<string> {
+        try {
+            // Get all available tools
+            const tools = agentTools.getAllTools();
+
+            if (!tools || tools.length === 0) {
+                return "";
+            }
+
+            // Format tool descriptions
+            const toolDescriptions = tools.map(tool =>
+                `- ${tool.name}: ${tool.description}`
+            ).join('\n');
+
+            return `Available tools:\n${toolDescriptions}`;
+        } catch (error) {
+            log.error(`Error getting agent tools description: ${error}`);
+            return "";
+        }
+    }
+
+    /**
+     * Get agent tools context for a specific note
+     * This context enriches LLM prompts with tools that can interact with Trilium
+     *
+     * @param noteId - The note ID
+     * @param query - The user's query
+     * @param showThinking - Whether to show LLM's thinking process
+     * @param relevantNotes - Optional notes already found to be relevant
+     * @returns Enhanced context with agent tools information
      */
     async getAgentToolsContext(
         noteId: string,
         query: string,
         showThinking: boolean = false,
-        relevantNotes: NoteSearchResult[] = []
+        relevantNotes: Array<any> = []
     ): Promise<string> {
         try {
-            if (!this.getAIEnabled()) {
+            // Create agent tools message
-                return '';
+            const toolsMessage = await this.getAgentToolsDescription();
+
+            // Initialize and use the agent tools
+            await this.initializeAgentTools();
+
+            // If we have notes that were already found to be relevant, use them directly
+            let contextNotes = relevantNotes;
+
+            // If no notes provided, find relevant ones
+            if (!contextNotes || contextNotes.length === 0) {
+                try {
+                    // Get the default LLM service for context enhancement
+                    const provider = this.getPreferredProvider();
+                    const llmService = this.getService(provider);
+
+                    // Find relevant notes
+                    contextNotes = await contextService.findRelevantNotes(
+                        query,
+                        noteId,
+                        {
+                            maxResults: 5,
+                            summarize: true,
+                            llmService
+                        }
+                    );
+
+                    log.info(`Found ${contextNotes.length} relevant notes for context`);
+                } catch (error) {
+                    log.error(`Failed to find relevant notes: ${error}`);
+                    // Continue without context notes
+                    contextNotes = [];
+                }
             }
 
-            await this.initializeAgentTools();
+            // Format notes into context string if we have any
-            return await contextService.getAgentToolsContext(noteId, query, showThinking);
+            let contextStr = "";
+            if (contextNotes && contextNotes.length > 0) {
+                contextStr = "\n\nRelevant context:\n";
+                contextNotes.forEach((note, index) => {
+                    contextStr += `[${index + 1}] "${note.title}"\n${note.content || 'No content available'}\n\n`;
+                });
+            }
+
+            // Combine tool message with context
+            return toolsMessage + contextStr;
         } catch (error) {
             log.error(`Error getting agent tools context: ${error}`);
-            return '';
+            return "";
         }
     }
 
@@ -562,7 +637,7 @@ export default {
         noteId: string,
         query: string,
         showThinking: boolean = false,
-        relevantNotes: NoteSearchResult[] = []
+        relevantNotes: Array<any> = []
     ): Promise<string> {
         return getInstance().getAgentToolsContext(
             noteId,

src/services/llm/context/index.ts

@@ -31,7 +31,14 @@ async function getSemanticContext(
             return "Semantic context service not available.";
         }
 
-        return await contextService.getSemanticContext(noteId, "", options.maxSimilarNotes || 5);
+        // Get an LLM service
+        const llmService = aiServiceManager.getInstance().getService();
+
+        const result = await contextService.processQuery("", llmService, {
+            maxResults: options.maxSimilarNotes || 5,
+            contextNoteId: noteId
+        });
+        return result.context;
     } catch (error) {
         console.error("Error getting semantic context:", error);
         return "Error retrieving semantic context.";
@@ -489,15 +496,26 @@ export class ContextExtractor {
      */
     static async getProgressiveContext(noteId: string, depth = 1): Promise<string> {
         try {
-            // Use the new context service
             const { default: aiServiceManager } = await import('../ai_service_manager.js');
             const contextService = aiServiceManager.getInstance().getContextService();
 
             if (!contextService) {
-                return ContextExtractor.extractContext(noteId);
+                return "Context service not available.";
             }
 
-            return await contextService.getProgressiveContext(noteId, depth);
+            const results = await contextService.findRelevantNotes(
+                "", // Empty query to get general context
+                noteId,
+                { maxResults: depth * 5 }
+            );
+
+            // Format the results
+            let contextText = `Progressive context for note (depth ${depth}):\n\n`;
+            results.forEach((note, index) => {
+                contextText += `[${index + 1}] ${note.title}\n${note.content || 'No content'}\n\n`;
+            });
+
+            return contextText;
         } catch (error) {
             // Fall back to regular context if progressive loading fails
             console.error('Error in progressive context loading:', error);
@@ -522,15 +540,21 @@ export class ContextExtractor {
      */
     static async getSmartContext(noteId: string, query: string): Promise<string> {
         try {
-            // Use the new context service
             const { default: aiServiceManager } = await import('../ai_service_manager.js');
             const contextService = aiServiceManager.getInstance().getContextService();
+            const llmService = aiServiceManager.getInstance().getService();
 
             if (!contextService) {
-                return ContextExtractor.extractContext(noteId);
+                return "Context service not available.";
             }
 
-            return await contextService.getSmartContext(noteId, query);
+            const result = await contextService.processQuery(
+                query,
+                llmService,
+                { contextNoteId: noteId }
+            );
+
+            return result.context;
         } catch (error) {
             // Fall back to regular context if smart context fails
             console.error('Error in smart context selection:', error);

src/services/llm/context/modules/context_service.ts

@@ -1,898 +0,0 @@
-import log from '../../../log.js';
-import providerManager from './provider_manager.js';
-import cacheManager from './cache_manager.js';
-import semanticSearch from './semantic_search.js';
-import queryEnhancer from './query_enhancer.js';
-import contextFormatter from './context_formatter.js';
-import aiServiceManager from '../../ai_service_manager.js';
-import { ContextExtractor } from '../index.js';
-import { CONTEXT_PROMPTS } from '../../constants/llm_prompt_constants.js';
-import becca from '../../../../becca/becca.js';
-import type { NoteSearchResult } from '../../interfaces/context_interfaces.js';
-import type { LLMServiceInterface } from '../../interfaces/agent_tool_interfaces.js';
-import type { Message } from '../../ai_interface.js';
-
-/**
- * Main context service that integrates all context-related functionality
- * This service replaces the old TriliumContextService and SemanticContextService
- */
-export class ContextService {
-    private initialized = false;
-    private initPromise: Promise<void> | null = null;
-    private contextExtractor: ContextExtractor;
-
-    constructor() {
-        this.contextExtractor = new ContextExtractor();
-    }
-
-    /**
-     * Initialize the service
-     */
-    async initialize(): Promise<void> {
-        if (this.initialized) return;
-
-        // Use a promise to prevent multiple simultaneous initializations
-        if (this.initPromise) return this.initPromise;
-
-        this.initPromise = (async () => {
-            try {
-                // Initialize provider
-                const provider = await providerManager.getPreferredEmbeddingProvider();
-                if (!provider) {
-                    throw new Error(`No embedding provider available. Could not initialize context service.`);
-                }
-
-                // Initialize agent tools to ensure they're ready
-                try {
-                    await aiServiceManager.getInstance().initializeAgentTools();
-                    log.info("Agent tools initialized for use with ContextService");
-                } catch (toolError) {
-                    log.error(`Error initializing agent tools: ${toolError}`);
-                    // Continue even if agent tools fail to initialize
-                }
-
-                this.initialized = true;
-                log.info(`Context service initialized with provider: ${provider.name}`);
-            } catch (error: unknown) {
-                const errorMessage = error instanceof Error ? error.message : String(error);
-                log.error(`Failed to initialize context service: ${errorMessage}`);
-                throw error;
-            } finally {
-                this.initPromise = null;
-            }
-        })();
-
-        return this.initPromise;
-    }
-
-    /**
-     * Process a user query to find relevant context in Trilium notes
-     *
-     * @param userQuestion - The user's query
-     * @param llmService - The LLM service to use
-     * @param contextNoteId - Optional note ID to restrict search to a branch
-     * @param showThinking - Whether to show the thinking process in output
-     * @returns Context information and relevant notes
-     */
-    async processQuery(
-        userQuestion: string,
-        llmService: LLMServiceInterface,
-        contextNoteId: string | null = null,
-        showThinking: boolean = false
-    ): Promise<{ context: string; sources: NoteSearchResult[]; thinking?: string }> {
-        log.info(`Processing query with: question="${userQuestion.substring(0, 50)}...", noteId=${contextNoteId}, showThinking=${showThinking}`);
-
-        if (!this.initialized) {
-            try {
-                await this.initialize();
-            } catch (error) {
-                log.error(`Failed to initialize ContextService: ${error}`);
-                // Return a fallback response if initialization fails
-                return {
-                    context: CONTEXT_PROMPTS.NO_NOTES_CONTEXT,
-                    sources: [],
-                    thinking: undefined
-                };
-            }
-        }
-
-        try {
-            // Step 1: Generate search queries (skip if tool calling might be enabled)
-            let searchQueries: string[];
-
-            // Check if llmService has tool calling enabled
-            const isToolsEnabled = llmService &&
-                typeof llmService === 'object' &&
-                'constructor' in llmService &&
-                llmService.constructor.name === 'OllamaService';
-
-            if (isToolsEnabled) {
-                // Skip query generation if tools might be used to avoid race conditions
-                log.info(`Skipping query enhancement for potential tool-enabled service: ${llmService.constructor.name}`);
-                searchQueries = [userQuestion]; // Use simple fallback
-            } else {
-                try {
-                    searchQueries = await queryEnhancer.generateSearchQueries(userQuestion, llmService);
-                } catch (error) {
-                    log.error(`Error generating search queries, using fallback: ${error}`);
-                    searchQueries = [userQuestion]; // Fallback to using the original question
-                }
-            }
-
-            log.info(`Generated search queries: ${JSON.stringify(searchQueries)}`);
-
-            // Step 2: Find relevant notes using the pipeline's VectorSearchStage
-            let relevantNotes: NoteSearchResult[] = [];
-            try {
-                log.info(`Using VectorSearchStage pipeline component to find relevant notes`);
-
-                // Create or import the vector search stage
-                const VectorSearchStage = (await import('../../pipeline/stages/vector_search_stage.js')).VectorSearchStage;
-                const vectorSearchStage = new VectorSearchStage();
-
-                // Use multi-query approach through the pipeline
-                const allResults: Map<string, NoteSearchResult> = new Map();
-
-                // Process searches using the pipeline stage
-                for (const query of searchQueries) {
-                    log.info(`Executing pipeline vector search for query: "${query.substring(0, 50)}..."`);
-
-                    // Use the pipeline stage directly
-                    const result = await vectorSearchStage.execute({
-                        query,
-                        noteId: contextNoteId,
-                        options: {
-                            maxResults: 5, // Limit per query
-                            useEnhancedQueries: false, // Don't enhance these - we already have enhanced queries
-                            threshold: 0.6,
-                            llmService // Pass the LLM service for potential use
-                        }
-                    });
-
-                    const results = result.searchResults;
-                    log.info(`Pipeline vector search found ${results.length} results for query "${query.substring(0, 50)}..."`);
-
-                    // Combine results, avoiding duplicates
-                    for (const result of results) {
-                        if (!allResults.has(result.noteId)) {
-                            allResults.set(result.noteId, result);
-                        } else {
-                            // If note already exists, update similarity to max of both values
-                            const existing = allResults.get(result.noteId);
-                            if (existing && result.similarity > existing.similarity) {
-                                existing.similarity = result.similarity;
-                                allResults.set(result.noteId, existing);
-                            }
-                        }
-                    }
-                }
-
-                // Convert map to array and limit to top results
-                relevantNotes = Array.from(allResults.values())
-                    .filter(note => {
-                        // Filter out notes with no content or very minimal content (less than 10 chars)
-                        const hasContent = note.content && note.content.trim().length > 10;
-                        if (!hasContent) {
-                            log.info(`Filtering out empty/minimal note: "${note.title}" (${note.noteId})`);
-                        }
-                        return hasContent;
-                    })
-                    .sort((a, b) => b.similarity - a.similarity)
-                    .slice(0, 20); // Increased from 8 to 20 notes
-
-                log.info(`After filtering out empty notes, ${relevantNotes.length} relevant notes remain`);
-            } catch (error) {
-                log.error(`Error finding relevant notes: ${error}`);
-                // Continue with empty notes list
-            }
-
-            // Step 3: Build context from the notes
-            const provider = await providerManager.getPreferredEmbeddingProvider();
-            const providerId = provider?.name || 'default';
-            const context = await contextFormatter.buildContextFromNotes(relevantNotes, userQuestion, providerId);
-
-            // DEBUG: Log the initial context built from notes
-            log.info(`Initial context from buildContextFromNotes: ${context.length} chars, starting with: "${context.substring(0, 150)}..."`);
-
-            // Step 4: Add agent tools context with thinking process if requested
-            let enhancedContext = context;
-            try {
-                // Pass 'root' as the default noteId when no specific note is selected
-                const noteIdToUse = contextNoteId || 'root';
-                log.info(`Calling getAgentToolsContext with noteId=${noteIdToUse}, showThinking=${showThinking}`);
-
-                const agentContext = await this.getAgentToolsContext(
-                    noteIdToUse,
-                    userQuestion,
-                    showThinking,
-                    relevantNotes
-                );
-
-                if (agentContext) {
-                    enhancedContext = enhancedContext + "\n\n" + agentContext;
-                }
-
-                // DEBUG: Log the final combined context
-                log.info(`FINAL COMBINED CONTEXT: ${enhancedContext.length} chars, with content structure: ${this.summarizeContextStructure(enhancedContext)}`);
-            } catch (error) {
-                log.error(`Error getting agent tools context: ${error}`);
-                // Continue with the basic context
-            }
-
-            return {
-                context: enhancedContext,
-                sources: relevantNotes,
-                thinking: showThinking ? this.summarizeContextStructure(enhancedContext) : undefined
-            };
-        } catch (error) {
-            log.error(`Error processing query: ${error}`);
-            return {
-                context: CONTEXT_PROMPTS.NO_NOTES_CONTEXT,
-                sources: [],
-                thinking: undefined
-            };
-        }
-    }
-
-    /**
-     * Get context with agent tools enhancement
-     *
-     * @param noteId - The relevant note ID
-     * @param query - The user's query
-     * @param showThinking - Whether to show thinking process
-     * @param relevantNotes - Optional pre-found relevant notes
-     * @returns Enhanced context string
-     */
-    async getAgentToolsContext(
-        noteId: string,
-        query: string,
-        showThinking: boolean = false,
-        relevantNotes: NoteSearchResult[] = []
-    ): Promise<string> {
-        try {
-            log.info(`Building enhanced agent tools context for query: "${query.substring(0, 50)}...", noteId=${noteId}, showThinking=${showThinking}`);
-
-            // Make sure agent tools are initialized
-            const agentManager = aiServiceManager.getInstance();
-
-            // Initialize all tools if not already done
-            if (!agentManager.getAgentTools().isInitialized()) {
-                await agentManager.initializeAgentTools();
-                log.info("Agent tools initialized on-demand in getAgentToolsContext");
-            }
-
-            // Get all agent tools
-            const vectorSearchTool = agentManager.getVectorSearchTool();
-            const noteNavigatorTool = agentManager.getNoteNavigatorTool();
-            const queryDecompositionTool = agentManager.getQueryDecompositionTool();
-            const contextualThinkingTool = agentManager.getContextualThinkingTool();
-
-            // Step 1: Start a thinking process
-            const thinkingId = contextualThinkingTool.startThinking(query);
-            contextualThinkingTool.addThinkingStep(thinkingId, {
-                type: 'observation',
-                content: `Analyzing query: "${query}" for note ID: ${noteId}`
-            });
-
-            // Step 2: Decompose the query into sub-questions
-            const decomposedQuery = queryDecompositionTool.decomposeQuery(query);
-            contextualThinkingTool.addThinkingStep(thinkingId, {
-                type: 'observation',
-                content: `Query complexity: ${decomposedQuery.complexity}/10. Decomposed into ${decomposedQuery.subQueries.length} sub-queries.`
-            });
-
-            // Log each sub-query as a thinking step
-            for (const subQuery of decomposedQuery.subQueries) {
-                contextualThinkingTool.addThinkingStep(thinkingId, {
-                    type: 'question',
-                    content: subQuery.text,
-                    metadata: {
-                        reason: subQuery.reason
-                    }
-                });
-            }
-
-            // Step 3: Use vector search to find related content
-            // Use an aggressive search with lower threshold to get more results
-            const searchOptions = {
-                threshold: 0.5,  // Lower threshold to include more matches
-                limit: 15        // Get more results
-            };
-
-            const vectorSearchPromises = [];
-
-            // Search for each sub-query that isn't just the original query
-            for (const subQuery of decomposedQuery.subQueries.filter(sq => sq.text !== query)) {
-                vectorSearchPromises.push(
-                    vectorSearchTool.search(subQuery.text, noteId, searchOptions)
-                        .then(results => {
-                            return {
-                                query: subQuery.text,
-                                results
-                            };
-                        })
-                );
-            }
-
-            // Wait for all searches to complete
-            const searchResults = await Promise.all(vectorSearchPromises);
-
-            // Record the search results in thinking steps
-            let totalResults = 0;
-            for (const result of searchResults) {
-                totalResults += result.results.length;
-
-                if (result.results.length > 0) {
-                    const stepId = contextualThinkingTool.addThinkingStep(thinkingId, {
-                        type: 'evidence',
-                        content: `Found ${result.results.length} relevant notes for sub-query: "${result.query}"`,
-                        metadata: {
-                            searchQuery: result.query
-                        }
-                    });
-
-                    // Add top results as children
-                    for (const note of result.results.slice(0, 3)) {
-                        contextualThinkingTool.addThinkingStep(thinkingId, {
-                            type: 'evidence',
-                            content: `Note "${note.title}" (similarity: ${Math.round(note.similarity * 100)}%) contains relevant information`,
-                            metadata: {
-                                noteId: note.noteId,
-                                similarity: note.similarity
-                            }
-                        }, stepId);
-                    }
-                } else {
-                    contextualThinkingTool.addThinkingStep(thinkingId, {
-                        type: 'observation',
-                        content: `No notes found for sub-query: "${result.query}"`,
-                        metadata: {
-                            searchQuery: result.query
-                        }
-                    });
-                }
-            }
-
-            // Step 4: Get note structure information
-            try {
-                const noteStructure = await noteNavigatorTool.getNoteStructure(noteId);
-
-                contextualThinkingTool.addThinkingStep(thinkingId, {
-                    type: 'observation',
-                    content: `Note structure: ${noteStructure.childCount} child notes, ${noteStructure.attributes.length} attributes, ${noteStructure.parentPath.length} levels in hierarchy`,
-                    metadata: {
-                        structure: noteStructure
-                    }
-                });
-
-                // Add information about parent path
-                if (noteStructure.parentPath.length > 0) {
-                    const parentPathStr = noteStructure.parentPath.map((p: {title: string, noteId: string}) => p.title).join(' > ');
-                    contextualThinkingTool.addThinkingStep(thinkingId, {
-                        type: 'observation',
-                        content: `Note hierarchy: ${parentPathStr}`,
-                        metadata: {
-                            parentPath: noteStructure.parentPath
-                        }
-                    });
-                }
-            } catch (error) {
-                log.error(`Error getting note structure: ${error}`);
-                contextualThinkingTool.addThinkingStep(thinkingId, {
-                    type: 'observation',
-                    content: `Unable to retrieve note structure information: ${error}`
-                });
-            }
-
-            // Step 5: Conclude thinking process
-            contextualThinkingTool.addThinkingStep(thinkingId, {
-                type: 'conclusion',
-                content: `Analysis complete. Found ${totalResults} relevant notes across ${searchResults.length} search queries.`,
-                metadata: {
-                    totalResults,
-                    queryCount: searchResults.length
-                }
-            });
-
-            // Complete the thinking process
-            contextualThinkingTool.completeThinking(thinkingId);
-
-            // Step 6: Build the context string combining all the information
-            let agentContext = '';
-
-            // Add note structure information
-            try {
-                const noteStructure = await noteNavigatorTool.getNoteStructure(noteId);
-                agentContext += `## Current Note Context\n`;
-                agentContext += `- Note Title: ${noteStructure.title}\n`;
-
-                if (noteStructure.parentPath.length > 0) {
-                    const parentPathStr = noteStructure.parentPath.map((p: {title: string, noteId: string}) => p.title).join(' > ');
-                    agentContext += `- Location: ${parentPathStr}\n`;
-                }
-
-                if (noteStructure.attributes.length > 0) {
-                    agentContext += `- Attributes: ${noteStructure.attributes.map((a: {name: string, value: string}) => `${a.name}=${a.value}`).join(', ')}\n`;
-                }
-
-                if (noteStructure.childCount > 0) {
-                    agentContext += `- Contains ${noteStructure.childCount} child notes\n`;
-                }
-
-                agentContext += `\n`;
-            } catch (error) {
-                log.error(`Error adding note structure to context: ${error}`);
-            }
-
-            // Combine the notes from both searches - the initial relevantNotes and from vector search
-            // Start with a Map to deduplicate by noteId
-            const allNotes = new Map<string, NoteSearchResult>();
-
-            // Add notes from the initial search in processQuery (relevantNotes parameter)
-            if (relevantNotes && relevantNotes.length > 0) {
-                log.info(`Adding ${relevantNotes.length} notes from initial search to combined results`);
-                for (const note of relevantNotes) {
-                    if (note.noteId) {
-                        allNotes.set(note.noteId, note);
-                    }
-                }
-            }
-
-            // Add notes from vector search of sub-queries
-            const vectorSearchNotes = searchResults.flatMap(r => r.results);
-            if (vectorSearchNotes.length > 0) {
-                log.info(`Adding ${vectorSearchNotes.length} notes from vector search to combined results`);
-                for (const note of vectorSearchNotes) {
-                    // If note already exists, keep the one with higher similarity
-                    const existing = allNotes.get(note.noteId);
-                    if (existing && note.similarity > existing.similarity) {
-                        existing.similarity = note.similarity;
-                    } else {
-                        allNotes.set(note.noteId, note);
-                    }
-                }
-            }
-
-            // Convert the combined Map to an array and sort by similarity
-            const combinedNotes = Array.from(allNotes.values())
-                .filter(note => {
-                    // Filter out notes with no content or very minimal content
-                    const hasContent = note.content && note.content.trim().length > 10;
-                    log.info(`Note "${note.title}" (${note.noteId}) has content: ${hasContent} and content length: ${note.content ? note.content.length : 0} chars`);
-                    if (!hasContent) {
-                        log.info(`Filtering out empty/minimal note from combined results: "${note.title}" (${note.noteId})`);
-                    }
-                    return hasContent;
-                })
-                .sort((a, b) => b.similarity - a.similarity);
-
-            log.info(`Combined ${relevantNotes.length} notes from initial search with ${vectorSearchNotes.length} notes from vector search, resulting in ${combinedNotes.length} unique notes after filtering out empty notes`);
-
-            // Just take the top notes by similarity
-            const finalNotes = combinedNotes.slice(0, 30); // Take top 30 notes
-
-            if (finalNotes.length > 0) {
-                agentContext += `## Relevant Information\n`;
-
-                for (const note of finalNotes) {
-                    agentContext += `### ${note.title}\n`;
-
-                    // Add relationship information for the note
-                    try {
-                        const noteObj = becca.getNote(note.noteId);
-                        if (noteObj) {
-                            // Get parent notes
-                            const parentNotes = noteObj.getParentNotes();
-                            if (parentNotes && parentNotes.length > 0) {
-                                agentContext += `**Parent notes:** ${parentNotes.map((p: any) => p.title).join(', ')}\n`;
-                            }
-
-                            // Get child notes
-                            const childNotes = noteObj.getChildNotes();
-                            if (childNotes && childNotes.length > 0) {
-                                agentContext += `**Child notes:** ${childNotes.map((c: any) => c.title).join(', ')}\n`;
-                            }
-
-                            // Get attributes
-                            const attributes = noteObj.getAttributes();
-                            if (attributes && attributes.length > 0) {
-                                const filteredAttrs = attributes.filter((a: any) => !a.name.startsWith('_')); // Filter out system attributes
-                                if (filteredAttrs.length > 0) {
-                                    agentContext += `**Attributes:** ${filteredAttrs.map((a: any) => `${a.name}=${a.value}`).join(', ')}\n`;
-                                }
-                            }
-
-                            // Get backlinks/related notes through relation attributes
-                            const relationAttrs = attributes?.filter((a: any) =>
-                                a.name.startsWith('relation:') ||
-                                a.name.startsWith('label:')
-                            );
-
-                            if (relationAttrs && relationAttrs.length > 0) {
-                                agentContext += `**Relationships:** ${relationAttrs.map((a: any) => {
-                                    const targetNote = becca.getNote(a.value);
-                                    const targetTitle = targetNote ? targetNote.title : a.value;
-                                    return `${a.name.substring(a.name.indexOf(':') + 1)} → ${targetTitle}`;
-                                }).join(', ')}\n`;
-                            }
-                        }
-                    } catch (error) {
-                        log.error(`Error getting relationship info for note ${note.noteId}: ${error}`);
-                    }
-
-                    agentContext += '\n';
-
-                    if (note.content) {
-                        // Extract relevant content instead of just taking first 2000 chars
-                        const relevantContent = await this.extractRelevantContent(note.content, query, 2000);
-                        agentContext += `${relevantContent}\n\n`;
-                    }
-                }
-            }
-
-
-            // Add thinking process if requested
-            if (showThinking) {
-                log.info(`Including thinking process in context (showThinking=true)`);
-                agentContext += `\n## Reasoning Process\n`;
-                const thinkingSummary = contextualThinkingTool.getThinkingSummary(thinkingId);
-                log.info(`Thinking summary length: ${thinkingSummary.length} characters`);
-                agentContext += thinkingSummary;
-            } else {
-                log.info(`Skipping thinking process in context (showThinking=false)`);
-            }
-
-            // Log stats about the context
-            log.info(`Agent tools context built: ${agentContext.length} chars, ${agentContext.split('\n').length} lines`);
-
-            // DEBUG: Log more detailed information about the agent tools context content
-            log.info(`Agent tools context content structure: ${this.summarizeContextStructure(agentContext)}`);
-            if (agentContext.length < 1000) {
-                log.info(`Agent tools context full content (short): ${agentContext}`);
-            } else {
-                log.info(`Agent tools context first 500 chars: ${agentContext.substring(0, 500)}...`);
-                log.info(`Agent tools context last 500 chars: ${agentContext.substring(agentContext.length - 500)}`);
-            }
-
-            return agentContext;
-        } catch (error) {
-            log.error(`Error getting agent tools context: ${error}`);
-            return `Error generating enhanced context: ${error}`;
-        }
-    }
-
-    /**
-     * Summarize the structure of a context string for debugging
-     * @param context - The context string to summarize
-     * @returns A summary of the context structure
-     */
-    private summarizeContextStructure(context: string): string {
-        if (!context) return "Empty context";
-
-        // Count sections and headers
-        const sections = context.split('##').length - 1;
-        const subSections = context.split('###').length - 1;
-
-        // Count notes referenced
-        const noteMatches = context.match(/### [^\n]+/g);
-        const noteCount = noteMatches ? noteMatches.length : 0;
-
-        // Extract note titles if present
-        let noteTitles = "";
-        if (noteMatches && noteMatches.length > 0) {
-            noteTitles = ` Note titles: ${noteMatches.slice(0, 3).map(m => m.substring(4)).join(', ')}${noteMatches.length > 3 ? '...' : ''}`;
-        }
-
-        return `${sections} main sections, ${subSections} subsections, ${noteCount} notes referenced.${noteTitles}`;
-    }
-
-    /**
-     * Get semantic context based on query
-     *
-     * @param noteId - Note ID to start from
-     * @param userQuery - User query for context
-     * @param maxResults - Maximum number of results
-     * @param messages - Optional conversation messages to adjust context size
-     * @returns Formatted context
-     */
-    async getSemanticContext(
-        noteId: string,
-        userQuery: string,
-        maxResults: number = 5,
-        messages: Message[] = []
-    ): Promise<string> {
-        if (!this.initialized) {
-            await this.initialize();
-        }
-
-        try {
-            // Get related notes from the context extractor
-            const [
-                parentNotes,
-                childNotes,
-                linkedNotes
-            ] = await Promise.all([
-                this.contextExtractor.getParentNotes(noteId, 3),
-                this.contextExtractor.getChildContext(noteId, 10).then(context => {
-                    // Parse child notes from context string
-                    const lines = context.split('\n');
-                    const result: {noteId: string, title: string}[] = [];
-                    for (const line of lines) {
-                        const match = line.match(/- (.*)/);
-                        if (match) {
-                            // We don't have noteIds in the context string, so use titles only
-                            result.push({
-                                title: match[1],
-                                noteId: '' // Empty noteId since we can't extract it from context
-                            });
-                        }
-                    }
-                    return result;
-                }),
-                this.contextExtractor.getLinkedNotesContext(noteId, 10).then(context => {
-                    // Parse linked notes from context string
-                    const lines = context.split('\n');
-                    const result: {noteId: string, title: string}[] = [];
-                    for (const line of lines) {
-                        const match = line.match(/- \[(.*?)\]\(trilium:\/\/([a-zA-Z0-9]+)\)/);
-                        if (match) {
-                            result.push({
-                                title: match[1],
-                                noteId: match[2]
-                            });
-                        }
-                    }
-                    return result;
-                })
-            ]);
-
-            // Combine all related notes
-            const allRelatedNotes = [...parentNotes, ...childNotes, ...linkedNotes];
-
-            // If no related notes, return empty context
-            if (allRelatedNotes.length === 0) {
-                return '';
-            }
-
-            // Convert parent notes from {id, title} to {noteId, title} for consistency
-            const normalizedRelatedNotes = allRelatedNotes.map(note => {
-                return {
-                    noteId: 'id' in note ? note.id : note.noteId,
-                    title: note.title
-                };
-            });
-
-            // Rank notes by relevance to query
-            const rankedNotes = await semanticSearch.rankNotesByRelevance(
-                normalizedRelatedNotes as Array<{noteId: string, title: string}>,
-                userQuery
-            );
-
-            // Get content for the top N most relevant notes
-            const mostRelevantNotes = rankedNotes.slice(0, maxResults);
-
-            // Get relevant search results to pass to context formatter
-            const searchResults = await Promise.all(
-                mostRelevantNotes.map(async note => {
-                    const content = await this.contextExtractor.getNoteContent(note.noteId);
-                    if (!content) return null;
-
-                    // Create a properly typed NoteSearchResult object
-                    return {
-                        noteId: note.noteId,
-                        title: note.title,
-                        content,
-                        similarity: note.relevance
-                    };
-                })
-            );
-
-            // Filter out nulls and empty content
-            const validResults: NoteSearchResult[] = searchResults
-                .filter(result => result !== null && result.content && result.content.trim().length > 0)
-                .map(result => result as NoteSearchResult);
-
-            // If no content retrieved, return empty string
-            if (validResults.length === 0) {
-                return '';
-            }
-
-            // Get the provider information for formatting
-            const provider = await providerManager.getPreferredEmbeddingProvider();
-            const providerId = provider?.name || 'default';
-
-            // Format the context with the context formatter (which handles adjusting for conversation size)
-            return contextFormatter.buildContextFromNotes(validResults, userQuery, providerId, messages);
-        } catch (error) {
-            log.error(`Error getting semantic context: ${error}`);
-            return '';
-        }
-    }
-
-    /**
-     * Get progressive context loading based on depth
-     *
-     * @param noteId - The base note ID
-     * @param depth - Depth level (1-4)
-     * @returns Context string with progressively more information
-     */
-    async getProgressiveContext(noteId: string, depth: number = 1): Promise<string> {
-        if (!this.initialized) {
-            await this.initialize();
-        }
-
-        try {
-            // Use the existing context extractor method
-            return await this.contextExtractor.getProgressiveContext(noteId, depth);
-        } catch (error) {
-            log.error(`Error getting progressive context: ${error}`);
-            return '';
-        }
-    }
-
-    /**
-     * Get smart context that adapts to query complexity
-     *
-     * @param noteId - The base note ID
-     * @param userQuery - The user's query
-     * @returns Context string with appropriate level of detail
-     */
-    async getSmartContext(noteId: string, userQuery: string): Promise<string> {
-        if (!this.initialized) {
-            await this.initialize();
-        }
-
-        try {
-            // Determine query complexity to adjust context depth
-            const complexity = queryEnhancer.estimateQueryComplexity(userQuery);
-
-            // If it's a simple query with low complexity, use progressive context
-            if (complexity < 0.3) {
-                return await this.getProgressiveContext(noteId, 2); // Just note + parents
-            }
-            // For medium complexity, include more context
-            else if (complexity < 0.7) {
-                return await this.getProgressiveContext(noteId, 3); // Note + parents + children
-            }
-            // For complex queries, use semantic context
-            else {
-                return await this.getSemanticContext(noteId, userQuery, 7); // More results for complex queries
-            }
-        } catch (error) {
-            log.error(`Error getting smart context: ${error}`);
-            // Fallback to basic context extraction
-            return await this.contextExtractor.extractContext(noteId);
-        }
-    }
-
-    /**
-     * Clear all context caches
-     */
-    clearCaches(): void {
-        cacheManager.clearAllCaches();
-    }
-
-    /**
-     * Extract the most relevant portions from a note's content
-     * @param content - The full note content
-     * @param query - The user's query
-     * @param maxChars - Maximum characters to include
-     * @returns The most relevant content sections
-     */
-    private async extractRelevantContent(content: string, query: string, maxChars: number = 2000): Promise<string> {
-        if (!content || content.length <= maxChars) {
-            return content; // Return full content if it's already short enough
-        }
-
-        try {
-            // Get the vector search tool for relevance calculation
-            const agentManager = aiServiceManager.getInstance();
-            const vectorSearchTool = agentManager.getVectorSearchTool();
-
-            // Split content into chunks of reasonable size (300-500 chars with overlap)
-            const chunkSize = 400;
-            const overlap = 100;
-            const chunks: string[] = [];
-
-            for (let i = 0; i < content.length; i += (chunkSize - overlap)) {
-                const end = Math.min(i + chunkSize, content.length);
-                chunks.push(content.substring(i, end));
-                if (end === content.length) break;
-            }
-
-            log.info(`Split note content into ${chunks.length} chunks for relevance extraction`);
-
-            // Get embedding provider from service
-            const provider = await providerManager.getPreferredEmbeddingProvider();
-            if (!provider) {
-                throw new Error("No embedding provider available");
-            }
-
-            // Get embeddings for the query and all chunks
-            const queryEmbedding = await provider.generateEmbeddings(query);
-
-            // Process chunks in smaller batches to avoid overwhelming the provider
-            const batchSize = 5;
-            const chunkEmbeddings = [];
-
-            for (let i = 0; i < chunks.length; i += batchSize) {
-                const batch = chunks.slice(i, i + batchSize);
-                const batchEmbeddings = await Promise.all(
-                    batch.map(chunk => provider.generateEmbeddings(chunk))
-                );
-                chunkEmbeddings.push(...batchEmbeddings);
-            }
-
-            // Calculate similarity between query and each chunk
-            const similarities: Array<{index: number, similarity: number, content: string}> =
-                chunkEmbeddings.map((embedding, index) => {
-                    // Calculate cosine similarity manually since the method doesn't exist
-                    const similarity = this.calculateCosineSimilarity(queryEmbedding, embedding);
-                    return { index, similarity, content: chunks[index] };
-                });
-
-            // Sort chunks by similarity (most relevant first)
-            similarities.sort((a, b) => b.similarity - a.similarity);
-
-            // DEBUG: Log some info about the top chunks
-            log.info(`Top 3 most relevant chunks for query "${query.substring(0, 30)}..." (out of ${chunks.length} total):`);
-            similarities.slice(0, 3).forEach((chunk, idx) => {
-                log.info(`  Chunk ${idx+1}: Similarity ${Math.round(chunk.similarity * 100)}%, Content: "${chunk.content.substring(0, 50)}..."`);
-            });
-
-            // Take the most relevant chunks up to maxChars
-            let result = '';
-            let totalChars = 0;
-            let chunksIncluded = 0;
-
-            for (const chunk of similarities) {
-                if (totalChars + chunk.content.length > maxChars) {
-                    // If adding full chunk would exceed limit, add as much as possible
-                    const remainingSpace = maxChars - totalChars;
-                    if (remainingSpace > 100) { // Only add if we can include something meaningful
-                        result += `\n...\n${chunk.content.substring(0, remainingSpace)}...`;
-                        log.info(`  Added partial chunk with similarity ${Math.round(chunk.similarity * 100)}% (${remainingSpace} chars)`);
-                    }
-                    break;
-                }
-
-                if (result.length > 0) result += '\n...\n';
-                result += chunk.content;
-                totalChars += chunk.content.length;
-                chunksIncluded++;
-            }
-
-            log.info(`Extracted ${totalChars} chars of relevant content from ${content.length} chars total (${chunksIncluded} chunks included)`);
-            return result;
-        } catch (error) {
-            log.error(`Error extracting relevant content: ${error}`);
-            // Fallback to simple truncation if extraction fails
-            return content.substring(0, maxChars) + '...';
-        }
-    }
-
-    /**
-     * Calculate cosine similarity between two vectors
-     * @param vec1 - First vector
-     * @param vec2 - Second vector
-     * @returns Cosine similarity between the two vectors
-     */
-    private calculateCosineSimilarity(vec1: number[], vec2: number[]): number {
-        let dotProduct = 0;
-        let norm1 = 0;
-        let norm2 = 0;
-
-        for (let i = 0; i < vec1.length; i++) {
-            dotProduct += vec1[i] * vec2[i];
-            norm1 += vec1[i] * vec1[i];
-            norm2 += vec2[i] * vec2[i];
-        }
-
-        const magnitude = Math.sqrt(norm1) * Math.sqrt(norm2);
-        if (magnitude === 0) return 0;
-        return dotProduct / magnitude;
-    }
-}
-
-// Export singleton instance
-export default new ContextService();

src/services/llm/context/modules/query_enhancer.ts

@@ -1,126 +0,0 @@
-import log from '../../../log.js';
-import cacheManager from './cache_manager.js';
-import type { Message } from '../../ai_interface.js';
-import { CONTEXT_PROMPTS } from '../../constants/llm_prompt_constants.js';
-import type { LLMServiceInterface } from '../../interfaces/agent_tool_interfaces.js';
-import type { IQueryEnhancer } from '../../interfaces/context_interfaces.js';
-import JsonExtractor from '../../utils/json_extractor.js';
-
-/**
- * Provides utilities for enhancing queries and generating search queries
- */
-export class QueryEnhancer implements IQueryEnhancer {
-    // Use the centralized query enhancer prompt
-    private metaPrompt = CONTEXT_PROMPTS.QUERY_ENHANCER;
-
-    /**
-     * Get enhanced prompt with JSON formatting instructions
-     */
-    private getEnhancedPrompt(): string {
-        return `${this.metaPrompt}
-IMPORTANT: You must respond with valid JSON arrays. Always include commas between array elements.
-Format your answer as a valid JSON array without markdown code blocks, like this: ["item1", "item2", "item3"]`;
-    }
-
-    /**
-     * Generate search queries to find relevant information for the user question
-     *
-     * @param userQuestion - The user's question
-     * @param llmService - The LLM service to use for generating queries
-     * @returns Array of search queries
-     */
-    async generateSearchQueries(userQuestion: string, llmService: LLMServiceInterface): Promise<string[]> {
-        if (!userQuestion || userQuestion.trim() === '') {
-            return []; // Return empty array for empty input
-        }
-
-        try {
-            // Check cache with proper type checking
-            const cached = cacheManager.getQueryResults<string[]>(`searchQueries:${userQuestion}`);
-            if (cached && Array.isArray(cached)) {
-                return cached;
-            }
-
-            const messages: Array<{
-                role: 'user' | 'assistant' | 'system';
-                content: string;
-            }> = [
-                { role: "system", content: this.getEnhancedPrompt() },
-                { role: "user", content: userQuestion }
-            ];
-
-            const options = {
-                temperature: 0.3,
-                maxTokens: 300,
-                bypassFormatter: true, // Completely bypass formatter for query enhancement
-                expectsJsonResponse: true // Explicitly request JSON-formatted response
-            };
-
-            // Get the response from the LLM
-            const response = await llmService.generateChatCompletion(messages, options);
-            const responseText = response.text; // Extract the text from the response object
-
-            // Use the JsonExtractor to parse the response
-            const queries = JsonExtractor.extract<string[]>(responseText, {
-                extractArrays: true,
-                minStringLength: 3,
-                applyFixes: true,
-                useFallbacks: true
-            });
-
-            if (queries && queries.length > 0) {
-                log.info(`Extracted ${queries.length} queries using JsonExtractor`);
-                cacheManager.storeQueryResults(`searchQueries:${userQuestion}`, queries);
-                return queries;
-            }
-
-            // If all else fails, just use the original question
-            const fallback = [userQuestion];
-            log.info(`No queries extracted, using fallback: "${userQuestion}"`);
-            cacheManager.storeQueryResults(`searchQueries:${userQuestion}`, fallback);
-            return fallback;
-        } catch (error: unknown) {
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            log.error(`Error generating search queries: ${errorMessage}`);
-            return [userQuestion];
-        }
-    }
-
-    /**
-     * Estimate the complexity of a query
-     * This is used to determine the appropriate amount of context to provide
-     *
-     * @param query - The query to analyze
-     * @returns A complexity score from 0 (simple) to 1 (complex)
-     */
-    estimateQueryComplexity(query: string): number {
-        // Simple complexity estimation based on various factors
-
-        // Factor 1: Query length
-        const lengthScore = Math.min(query.length / 100, 0.4);
-
-        // Factor 2: Number of question words
-        const questionWords = ['what', 'how', 'why', 'when', 'where', 'who', 'which'];
-        const questionWordsCount = questionWords.filter(word =>
-            query.toLowerCase().includes(` ${word} `) ||
-            query.toLowerCase().startsWith(`${word} `)
-        ).length;
-        const questionWordsScore = Math.min(questionWordsCount * 0.15, 0.3);
-
-        // Factor 3: Contains comparison indicators
-        const comparisonWords = ['compare', 'difference', 'versus', 'vs', 'similarities', 'differences'];
-        const hasComparison = comparisonWords.some(word => query.toLowerCase().includes(word));
-        const comparisonScore = hasComparison ? 0.2 : 0;
-
-        // Factor 4: Request for detailed or in-depth information
-        const depthWords = ['explain', 'detail', 'elaborate', 'analysis', 'in-depth'];
-        const hasDepthRequest = depthWords.some(word => query.toLowerCase().includes(word));
-        const depthScore = hasDepthRequest ? 0.2 : 0;
-
-        // Combine scores with a maximum of 1.0
-        return Math.min(lengthScore + questionWordsScore + comparisonScore + depthScore, 1.0);
-    }
-}
-
-// Export singleton instance
-export default new QueryEnhancer();

src/services/llm/context/modules/semantic_search.ts

@@ -1,300 +0,0 @@
-import * as vectorStore from '../../embeddings/index.js';
-import { cosineSimilarity } from '../../embeddings/index.js';
-import log from '../../../log.js';
-import becca from '../../../../becca/becca.js';
-import providerManager from './provider_manager.js';
-import cacheManager from './cache_manager.js';
-import { ContextExtractor } from '../index.js';
-
-/**
- * Provides semantic search capabilities for finding relevant notes
- */
-export class SemanticSearch {
-    private contextExtractor: ContextExtractor;
-
-    constructor() {
-        this.contextExtractor = new ContextExtractor();
-    }
-
-    /**
-     * Rank notes by their semantic relevance to a query
-     *
-     * @param notes - Array of notes with noteId and title
-     * @param userQuery - The user's query to compare against
-     * @returns Sorted array of notes with relevance score
-     */
-    async rankNotesByRelevance(
-        notes: Array<{noteId: string, title: string}>,
-        userQuery: string
-    ): Promise<Array<{noteId: string, title: string, relevance: number}>> {
-        // Try to get from cache first
-        const cacheKey = `rank:${userQuery}:${notes.map(n => n.noteId).join(',')}`;
-        const cached = cacheManager.getNoteData('', cacheKey);
-        if (cached) {
-            return cached as Array<{noteId: string, title: string, relevance: number}>;
-        }
-
-        const queryEmbedding = await providerManager.generateQueryEmbedding(userQuery);
-        if (!queryEmbedding) {
-            // If embedding fails, return notes in original order
-            return notes.map(note => ({ ...note, relevance: 0 }));
-        }
-
-        const provider = await providerManager.getPreferredEmbeddingProvider();
-        if (!provider) {
-            return notes.map(note => ({ ...note, relevance: 0 }));
-        }
-
-        const rankedNotes = [];
-
-        for (const note of notes) {
-            // Get note embedding from vector store or generate it if not exists
-            let noteEmbedding = null;
-            try {
-                const embeddingResult = await vectorStore.getEmbeddingForNote(
-                    note.noteId,
-                    provider.name,
-                    provider.getConfig().model || ''
-                );
-
-                if (embeddingResult) {
-                    noteEmbedding = embeddingResult.embedding;
-                }
-            } catch (error) {
-                log.error(`Error retrieving embedding for note ${note.noteId}: ${error}`);
-            }
-
-            if (!noteEmbedding) {
-                // If note doesn't have an embedding yet, get content and generate one
-                const content = await this.contextExtractor.getNoteContent(note.noteId);
-                if (content && provider) {
-                    try {
-                        noteEmbedding = await provider.generateEmbeddings(content);
-                        // Store the embedding for future use
-                        await vectorStore.storeNoteEmbedding(
-                            note.noteId,
-                            provider.name,
-                            provider.getConfig().model || '',
-                            noteEmbedding
-                        );
-                    } catch (error) {
-                        log.error(`Error generating embedding for note ${note.noteId}: ${error}`);
-                    }
-                }
-            }
-
-            let relevance = 0;
-            if (noteEmbedding) {
-                // Calculate cosine similarity between query and note
-                relevance = cosineSimilarity(queryEmbedding, noteEmbedding);
-            }
-
-            rankedNotes.push({
-                ...note,
-                relevance
-            });
-        }
-
-        // Sort by relevance (highest first)
-        const result = rankedNotes.sort((a, b) => b.relevance - a.relevance);
-
-        // Cache results
-        cacheManager.storeNoteData('', cacheKey, result);
-
-        return result;
-    }
-
-    /**
-     * Find notes that are semantically relevant to a query
-     *
-     * @param query - The search query
-     * @param contextNoteId - Optional note ID to restrict search to a branch
-     * @param limit - Maximum number of results to return
-     * @returns Array of relevant notes with similarity scores
-     */
-    async findRelevantNotes(
-        query: string,
-        contextNoteId: string | null = null,
-        limit = 10
-    ): Promise<{noteId: string, title: string, content: string | null, similarity: number}[]> {
-        try {
-            // Check cache first
-            const cacheKey = `find:${query}:${contextNoteId || 'all'}:${limit}`;
-            const cached = cacheManager.getQueryResults(cacheKey);
-            if (cached) {
-                return cached as Array<{noteId: string, title: string, content: string | null, similarity: number}>;
-            }
-
-            // Get embedding for query
-            const queryEmbedding = await providerManager.generateQueryEmbedding(query);
-            if (!queryEmbedding) {
-                log.error('Failed to generate query embedding');
-                return [];
-            }
-
-            let results: {noteId: string, similarity: number}[] = [];
-
-            // Get provider information
-            const provider = await providerManager.getPreferredEmbeddingProvider();
-            if (!provider) {
-                log.error('No embedding provider available');
-                return [];
-            }
-
-            // If contextNoteId is provided, search only within that branch
-            if (contextNoteId) {
-                results = await this.findNotesInBranch(queryEmbedding, contextNoteId, limit);
-            } else {
-                // Otherwise search across all notes with embeddings
-                results = await vectorStore.findSimilarNotes(
-                    queryEmbedding,
-                    provider.name,
-                    provider.getConfig().model || '',
-                    limit
-                );
-            }
-
-            // Get note details for results
-            const enrichedResults = await Promise.all(
-                results.map(async result => {
-                    const note = becca.getNote(result.noteId);
-                    if (!note) {
-                        return null;
-                    }
-
-                    // Get note content
-                    const content = await this.contextExtractor.getNoteContent(result.noteId);
-
-                    // Adjust similarity score based on content quality
-                    let adjustedSimilarity = result.similarity;
-
-                    // Penalize notes with empty or minimal content
-                    if (!content || content.trim().length <= 10) {
-                        // Reduce similarity by 80% for empty/minimal notes
-                        adjustedSimilarity *= 0.2;
-                        log.info(`Adjusting similarity for empty/minimal note "${note.title}" from ${Math.round(result.similarity * 100)}% to ${Math.round(adjustedSimilarity * 100)}%`);
-                    }
-                    // Slightly boost notes with substantial content
-                    else if (content.length > 100) {
-                        // Small boost of 10% for notes with substantial content
-                        adjustedSimilarity = Math.min(1.0, adjustedSimilarity * 1.1);
-                    }
-
-                    return {
-                        noteId: result.noteId,
-                        title: note.title,
-                        content,
-                        similarity: adjustedSimilarity
-                    };
-                })
-            );
-
-            // Filter out null results
-            const filteredResults = enrichedResults.filter(result => {
-                // Filter out null results and notes with empty or minimal content
-                if (!result) return false;
-
-                // Instead of hard filtering by content length, now we use an adjusted
-                // similarity score, but we can still filter extremely low scores
-                return result.similarity > 0.2;
-            }) as {
-                noteId: string,
-                title: string,
-                content: string | null,
-                similarity: number
-            }[];
-
-            // Sort results by adjusted similarity
-            filteredResults.sort((a, b) => b.similarity - a.similarity);
-
-            // Cache results
-            cacheManager.storeQueryResults(cacheKey, filteredResults);
-
-            return filteredResults;
-        } catch (error) {
-            log.error(`Error finding relevant notes: ${error}`);
-            return [];
-        }
-    }
-
-    /**
-     * Find notes in a specific branch (subtree) that are relevant to a query
-     *
-     * @param embedding - The query embedding
-     * @param contextNoteId - Root note ID of the branch
-     * @param limit - Maximum results to return
-     * @returns Array of note IDs with similarity scores
-     */
-    private async findNotesInBranch(
-        embedding: Float32Array,
-        contextNoteId: string,
-        limit = 5
-    ): Promise<{noteId: string, similarity: number}[]> {
-        try {
-            // Get all notes in the subtree
-            const noteIds = await this.getSubtreeNoteIds(contextNoteId);
-
-            if (noteIds.length === 0) {
-                return [];
-            }
-
-            // Get provider information
-            const provider = await providerManager.getPreferredEmbeddingProvider();
-            if (!provider) {
-                log.error('No embedding provider available');
-                return [];
-            }
-
-            // Get model configuration
-            const model = provider.getConfig().model || '';
-            const providerName = provider.name;
-
-            // Use vectorStore to find similar notes within this subset
-            // Ideally we'd have a method to find within a specific set, but we'll use the general findSimilarNotes
-            return await vectorStore.findSimilarNotes(
-                embedding,
-                providerName,
-                model,
-                limit
-            ).then(results => {
-                // Filter to only include notes within our noteIds set
-                return results.filter(result => noteIds.includes(result.noteId));
-            });
-        } catch (error) {
-            log.error(`Error finding notes in branch: ${error}`);
-            return [];
-        }
-    }
-
-    /**
-     * Get all note IDs in a subtree
-     *
-     * @param rootNoteId - The root note ID
-     * @returns Array of note IDs in the subtree
-     */
-    private async getSubtreeNoteIds(rootNoteId: string): Promise<string[]> {
-        const noteIds = new Set<string>();
-        noteIds.add(rootNoteId); // Include the root note itself
-
-        const collectChildNotes = (noteId: string) => {
-            const note = becca.getNote(noteId);
-            if (!note) {
-                return;
-            }
-
-            const childNotes = note.getChildNotes();
-            for (const childNote of childNotes) {
-                if (!noteIds.has(childNote.noteId)) {
-                    noteIds.add(childNote.noteId);
-                    collectChildNotes(childNote.noteId);
-                }
-            }
-        };
-
-        collectChildNotes(rootNoteId);
-        return Array.from(noteIds);
-    }
-}
-
-// Export singleton instance
-export default new SemanticSearch();

src/services/llm/context/services/context_service.ts

@@ -0,0 +1,336 @@
+/**
+ * Unified Context Service
+ *
+ * Consolidates functionality from:
+ * - context_service.ts (old version)
+ * - semantic_search.ts
+ * - vector_search_stage.ts
+ *
+ * This service provides a central interface for all context extraction operations,
+ * supporting both full and summarized note content extraction.
+ */
+
+import log from '../../../log.js';
+import providerManager from '../modules/provider_manager.js';
+import cacheManager from '../modules/cache_manager.js';
+import vectorSearchService from './vector_search_service.js';
+import queryProcessor from './query_processor.js';
+import contextFormatter from '../modules/context_formatter.js';
+import aiServiceManager from '../../ai_service_manager.js';
+import { ContextExtractor } from '../index.js';
+import { CONTEXT_PROMPTS } from '../../constants/llm_prompt_constants.js';
+import type { NoteSearchResult } from '../../interfaces/context_interfaces.js';
+import type { LLMServiceInterface } from '../../interfaces/agent_tool_interfaces.js';
+
+// Options for context processing
+export interface ContextOptions {
+  // Content options
+  summarizeContent?: boolean;
+  maxResults?: number;
+  contextNoteId?: string | null;
+
+  // Processing options
+  useQueryEnhancement?: boolean;
+  useQueryDecomposition?: boolean;
+
+  // Debugging options
+  showThinking?: boolean;
+}
+
+export class ContextService {
+  private initialized = false;
+  private initPromise: Promise<void> | null = null;
+  private contextExtractor: ContextExtractor;
+
+  constructor() {
+    this.contextExtractor = new ContextExtractor();
+  }
+
+  /**
+   * Initialize the service
+   */
+  async initialize(): Promise<void> {
+    if (this.initialized) return;
+
+    // Use a promise to prevent multiple simultaneous initializations
+    if (this.initPromise) return this.initPromise;
+
+    this.initPromise = (async () => {
+      try {
+        // Initialize provider
+        const provider = await providerManager.getPreferredEmbeddingProvider();
+        if (!provider) {
+          throw new Error(`No embedding provider available. Could not initialize context service.`);
+        }
+
+        // Initialize agent tools to ensure they're ready
+        try {
+          await aiServiceManager.getInstance().initializeAgentTools();
+          log.info("Agent tools initialized for use with ContextService");
+        } catch (toolError) {
+          log.error(`Error initializing agent tools: ${toolError}`);
+          // Continue even if agent tools fail to initialize
+        }
+
+        this.initialized = true;
+        log.info(`Context service initialized with provider: ${provider.name}`);
+      } catch (error: unknown) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        log.error(`Failed to initialize context service: ${errorMessage}`);
+        throw error;
+      } finally {
+        this.initPromise = null;
+      }
+    })();
+
+    return this.initPromise;
+  }
+
+  /**
+   * Process a user query to find relevant context in Trilium notes
+   *
+   * @param userQuestion - The user's query
+   * @param llmService - The LLM service to use
+   * @param options - Context processing options
+   * @returns Context information and relevant notes
+   */
+  async processQuery(
+    userQuestion: string,
+    llmService: LLMServiceInterface,
+    options: ContextOptions = {}
+  ): Promise<{
+    context: string;
+    sources: NoteSearchResult[];
+    thinking?: string;
+    decomposedQuery?: any;
+  }> {
+    // Set default options
+    const {
+      summarizeContent = false,
+      maxResults = 10,
+      contextNoteId = null,
+      useQueryEnhancement = true,
+      useQueryDecomposition = false,
+      showThinking = false
+    } = options;
+
+    log.info(`Processing query: "${userQuestion.substring(0, 50)}..."`);
+    log.info(`Options: summarize=${summarizeContent}, maxResults=${maxResults}, contextNoteId=${contextNoteId || 'global'}`);
+    log.info(`Processing: enhancement=${useQueryEnhancement}, decomposition=${useQueryDecomposition}, showThinking=${showThinking}`);
+
+    if (!this.initialized) {
+      try {
+        await this.initialize();
+      } catch (error) {
+        log.error(`Failed to initialize ContextService: ${error}`);
+        // Return a fallback response if initialization fails
+        return {
+          context: CONTEXT_PROMPTS.NO_NOTES_CONTEXT,
+          sources: [],
+          thinking: undefined
+        };
+      }
+    }
+
+    try {
+      let decomposedQuery;
+      let searchQueries: string[] = [userQuestion];
+      let relevantNotes: NoteSearchResult[] = [];
+
+      // Step 1: Decompose query if requested
+      if (useQueryDecomposition) {
+        log.info(`Decomposing query for better understanding`);
+        decomposedQuery = queryProcessor.decomposeQuery(userQuestion);
+
+        // Extract sub-queries to use for search
+        if (decomposedQuery.subQueries.length > 0) {
+          searchQueries = decomposedQuery.subQueries
+            .map(sq => sq.text)
+            .filter(text => text !== userQuestion); // Remove the original query to avoid duplication
+
+          // Always include the original query
+          searchQueries.unshift(userQuestion);
+
+          log.info(`Query decomposed into ${searchQueries.length} search queries`);
+        }
+      }
+      // Step 2: Or enhance query if requested
+      else if (useQueryEnhancement) {
+        try {
+          log.info(`Enhancing query for better semantic matching`);
+          searchQueries = await queryProcessor.generateSearchQueries(userQuestion, llmService);
+          log.info(`Generated ${searchQueries.length} enhanced search queries`);
+        } catch (error) {
+          log.error(`Error generating search queries, using fallback: ${error}`);
+          searchQueries = [userQuestion]; // Fallback to using the original question
+        }
+      }
+
+      // Step 3: Find relevant notes using vector search
+      const allResults = new Map<string, NoteSearchResult>();
+
+      for (const query of searchQueries) {
+        try {
+          log.info(`Searching for: "${query.substring(0, 50)}..."`);
+
+          // Use the unified vector search service
+          const results = await vectorSearchService.findRelevantNotes(
+            query,
+            contextNoteId,
+            {
+              maxResults: maxResults,
+              summarizeContent: summarizeContent,
+              llmService: summarizeContent ? llmService : null
+            }
+          );
+
+          log.info(`Found ${results.length} results for query "${query.substring(0, 30)}..."`);
+
+          // Combine results, avoiding duplicates
+          for (const result of results) {
+            if (!allResults.has(result.noteId)) {
+              allResults.set(result.noteId, result);
+            } else {
+              // If note already exists, update similarity to max of both values
+              const existing = allResults.get(result.noteId);
+              if (existing && result.similarity > existing.similarity) {
+                existing.similarity = result.similarity;
+                allResults.set(result.noteId, existing);
+              }
+            }
+          }
+        } catch (error) {
+          log.error(`Error searching for query "${query}": ${error}`);
+        }
+      }
+
+      // Convert to array and sort by similarity
+      relevantNotes = Array.from(allResults.values())
+        .sort((a, b) => b.similarity - a.similarity)
+        .slice(0, maxResults);
+
+      log.info(`Final combined results: ${relevantNotes.length} relevant notes`);
+
+      // Step 4: Build context from the notes
+      const provider = await providerManager.getPreferredEmbeddingProvider();
+      const providerId = provider?.name || 'default';
+
+      const context = await contextFormatter.buildContextFromNotes(
+        relevantNotes,
+        userQuestion,
+        providerId
+      );
+
+      // Step 5: Add agent tools context if requested
+      let enhancedContext = context;
+      let thinkingProcess: string | undefined = undefined;
+
+      if (showThinking) {
+        thinkingProcess = this.generateThinkingProcess(
+          userQuestion,
+          searchQueries,
+          relevantNotes,
+          decomposedQuery
+        );
+      }
+
+      return {
+        context: enhancedContext,
+        sources: relevantNotes,
+        thinking: thinkingProcess,
+        decomposedQuery
+      };
+    } catch (error) {
+      log.error(`Error processing query: ${error}`);
+      return {
+        context: CONTEXT_PROMPTS.NO_NOTES_CONTEXT,
+        sources: [],
+        thinking: undefined
+      };
+    }
+  }
+
+  /**
+   * Generate a thinking process for debugging and transparency
+   */
+  private generateThinkingProcess(
+    originalQuery: string,
+    searchQueries: string[],
+    relevantNotes: NoteSearchResult[],
+    decomposedQuery?: any
+  ): string {
+    let thinking = `## Query Processing\n\n`;
+    thinking += `Original query: "${originalQuery}"\n\n`;
+
+    // Add decomposition analysis if available
+    if (decomposedQuery) {
+      thinking += `Query complexity: ${decomposedQuery.complexity}/10\n\n`;
+      thinking += `### Decomposed into ${decomposedQuery.subQueries.length} sub-queries:\n`;
+
+      decomposedQuery.subQueries.forEach((sq: any, i: number) => {
+        thinking += `${i+1}. ${sq.text}\n   Reason: ${sq.reason}\n\n`;
+      });
+    }
+
+    // Add search queries
+    thinking += `### Search Queries Used:\n`;
+    searchQueries.forEach((q, i) => {
+      thinking += `${i+1}. "${q}"\n`;
+    });
+
+    // Add found sources
+    thinking += `\n## Sources Retrieved (${relevantNotes.length})\n\n`;
+
+    relevantNotes.slice(0, 5).forEach((note, i) => {
+      thinking += `${i+1}. "${note.title}" (Score: ${Math.round(note.similarity * 100)}%)\n`;
+      thinking += `   ID: ${note.noteId}\n`;
+
+      // Check if parentPath exists before using it
+      if ('parentPath' in note && note.parentPath) {
+        thinking += `   Path: ${note.parentPath}\n`;
+      }
+
+      if (note.content) {
+        const contentPreview = note.content.length > 100
+          ? note.content.substring(0, 100) + '...'
+          : note.content;
+        thinking += `   Preview: ${contentPreview}\n`;
+      }
+
+      thinking += '\n';
+    });
+
+    if (relevantNotes.length > 5) {
+      thinking += `... and ${relevantNotes.length - 5} more sources\n`;
+    }
+
+    return thinking;
+  }
+
+  /**
+   * Find notes semantically related to a query
+   * (Shorthand method that directly uses vectorSearchService)
+   */
+  async findRelevantNotes(
+    query: string,
+    contextNoteId: string | null = null,
+    options: {
+      maxResults?: number,
+      summarize?: boolean,
+      llmService?: LLMServiceInterface | null
+    } = {}
+  ): Promise<NoteSearchResult[]> {
+    return vectorSearchService.findRelevantNotes(
+      query,
+      contextNoteId,
+      {
+        maxResults: options.maxResults,
+        summarizeContent: options.summarize,
+        llmService: options.llmService
+      }
+    );
+  }
+}
+
+// Export a singleton instance
+export default new ContextService();

src/services/llm/context/services/index.ts

@@ -0,0 +1,28 @@
+/**
+ * Consolidated Context Services
+ *
+ * This file exports the centralized context-related services that have been
+ * consolidated from previously overlapping implementations:
+ *
+ * - ContextService: Main entry point for context extraction operations
+ * - VectorSearchService: Unified semantic search functionality
+ * - QueryProcessor: Query enhancement and decomposition
+ */
+
+import contextService from './context_service.js';
+import vectorSearchService from './vector_search_service.js';
+import queryProcessor from './query_processor.js';
+
+export {
+  contextService,
+  vectorSearchService,
+  queryProcessor
+};
+
+// Export types
+export type { ContextOptions } from './context_service.js';
+export type { VectorSearchOptions } from './vector_search_service.js';
+export type { SubQuery, DecomposedQuery } from './query_processor.js';
+
+// Default export for backwards compatibility
+export default contextService;

src/services/llm/context/services/query_processor.ts

@@ -0,0 +1,516 @@
+/**
+ * Unified Query Processor Service
+ *
+ * Consolidates functionality from:
+ * - query_enhancer.ts
+ * - query_decomposition_tool.ts
+ *
+ * This service provides a central interface for all query processing operations,
+ * including enhancement, decomposition, and complexity analysis.
+ */
+
+import log from '../../../log.js';
+import cacheManager from '../modules/cache_manager.js';
+import { CONTEXT_PROMPTS } from '../../constants/llm_prompt_constants.js';
+import { QUERY_DECOMPOSITION_STRINGS } from '../../constants/query_decomposition_constants.js';
+import JsonExtractor from '../../utils/json_extractor.js';
+import type { LLMServiceInterface } from '../../interfaces/agent_tool_interfaces.js';
+
+// Interfaces
+export interface SubQuery {
+  id: string;
+  text: string;
+  reason: string;
+  isAnswered: boolean;
+  answer?: string;
+}
+
+export interface DecomposedQuery {
+  originalQuery: string;
+  subQueries: SubQuery[];
+  status: 'pending' | 'in_progress' | 'completed';
+  complexity: number;
+}
+
+export class QueryProcessor {
+  private static queryCounter: number = 0;
+
+  // Prompt templates
+  private enhancerPrompt = CONTEXT_PROMPTS.QUERY_ENHANCER;
+
+  /**
+   * Generate enhanced search queries for better semantic matching
+   *
+   * @param userQuestion - The user's question
+   * @param llmService - The LLM service to use for generating queries
+   * @returns Array of search queries
+   */
+  async generateSearchQueries(
+    userQuestion: string,
+    llmService: LLMServiceInterface
+  ): Promise<string[]> {
+    if (!userQuestion || userQuestion.trim() === '') {
+      return []; // Return empty array for empty input
+    }
+
+    try {
+      // Check cache
+      const cacheKey = `searchQueries:${userQuestion}`;
+      const cached = cacheManager.getQueryResults<string[]>(cacheKey);
+      if (cached && Array.isArray(cached)) {
+        return cached;
+      }
+
+      // Prepare the prompt with JSON formatting instructions
+      const enhancedPrompt = `${this.enhancerPrompt}
+IMPORTANT: You must respond with valid JSON arrays. Always include commas between array elements.
+Format your answer as a valid JSON array without markdown code blocks, like this: ["item1", "item2", "item3"]`;
+
+      const messages = [
+        { role: "system" as const, content: enhancedPrompt },
+        { role: "user" as const, content: userQuestion }
+      ];
+
+      const options = {
+        temperature: 0.3,
+        maxTokens: 300,
+        bypassFormatter: true,
+        expectsJsonResponse: true,
+        _bypassContextProcessing: true // Prevent recursive calls
+      };
+
+      // Get the response from the LLM
+      const response = await llmService.generateChatCompletion(messages, options);
+      const responseText = response.text;
+
+      // Use the JsonExtractor to parse the response
+      const queries = JsonExtractor.extract<string[]>(responseText, {
+        extractArrays: true,
+        minStringLength: 3,
+        applyFixes: true,
+        useFallbacks: true
+      });
+
+      if (queries && queries.length > 0) {
+        log.info(`Extracted ${queries.length} queries using JsonExtractor`);
+        cacheManager.storeQueryResults(cacheKey, queries);
+        return queries;
+      }
+
+      // Fallback to original question
+      const fallback = [userQuestion];
+      log.info(`No queries extracted, using fallback: "${userQuestion}"`);
+      cacheManager.storeQueryResults(cacheKey, fallback);
+      return fallback;
+    } catch (error: unknown) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      log.error(`Error generating search queries: ${errorMessage}`);
+      return [userQuestion];
+    }
+  }
+
+  /**
+   * Break down a complex query into smaller, more manageable sub-queries
+   *
+   * @param query The original user query
+   * @param context Optional context about the current note being viewed
+   * @returns A decomposed query object with sub-queries
+   */
+  decomposeQuery(query: string, context?: string): DecomposedQuery {
+    try {
+      // Log the decomposition attempt
+      log.info(`Decomposing query: "${query}"`);
+
+      if (!query || query.trim().length === 0) {
+        log.info(`Query is empty, skipping decomposition`);
+        return {
+          originalQuery: query,
+          subQueries: [],
+          status: 'pending',
+          complexity: 0
+        };
+      }
+
+      // Assess query complexity
+      const complexity = this.assessQueryComplexity(query);
+      log.info(`Query complexity assessment: ${complexity}/10`);
+
+      // For simple queries, just return the original as a single sub-query
+      if (complexity < 3) {
+        log.info(`Simple query detected (complexity: ${complexity}), using direct approach`);
+
+        const mainSubQuery = {
+          id: this.generateSubQueryId(),
+          text: query,
+          reason: "Direct question that can be answered without decomposition",
+          isAnswered: false
+        };
+
+        // Add a generic exploration query for context
+        const genericQuery = {
+          id: this.generateSubQueryId(),
+          text: `What information is related to ${query}?`,
+          reason: "General exploration to find related content",
+          isAnswered: false
+        };
+
+        return {
+          originalQuery: query,
+          subQueries: [mainSubQuery, genericQuery],
+          status: 'pending',
+          complexity
+        };
+      }
+
+      // For complex queries, break it down into sub-queries
+      const subQueries = this.createSubQueries(query, context);
+      log.info(`Decomposed query into ${subQueries.length} sub-queries`);
+
+      return {
+        originalQuery: query,
+        subQueries,
+        status: 'pending',
+        complexity
+      };
+    } catch (error: any) {
+      log.error(`Error decomposing query: ${error.message}`);
+
+      // Fallback to treating it as a simple query
+      return {
+        originalQuery: query,
+        subQueries: [{
+          id: this.generateSubQueryId(),
+          text: query,
+          reason: "Error occurred during decomposition, using original query",
+          isAnswered: false
+        }],
+        status: 'pending',
+        complexity: 1
+      };
+    }
+  }
+
+  /**
+   * Create sub-queries from a complex query
+   *
+   * @param query The original complex query
+   * @param context Optional context to help with decomposition
+   * @returns Array of sub-queries
+   */
+  private createSubQueries(query: string, context?: string): SubQuery[] {
+    // Analyze the query to identify potential aspects to explore
+    const questionParts = this.identifyQuestionParts(query);
+    const subQueries: SubQuery[] = [];
+
+    // Add the main query as the first sub-query
+    subQueries.push({
+      id: this.generateSubQueryId(),
+      text: query,
+      reason: "Main question (for direct matching)",
+      isAnswered: false
+    });
+
+    // Add sub-queries for each identified question part
+    for (const part of questionParts) {
+      subQueries.push({
+        id: this.generateSubQueryId(),
+        text: part,
+        reason: "Sub-aspect of the main question",
+        isAnswered: false
+      });
+    }
+
+    // Add a generic exploration query to find related information
+    subQueries.push({
+      id: this.generateSubQueryId(),
+      text: `What information is related to ${query}?`,
+      reason: "General exploration to find related content",
+      isAnswered: false
+    });
+
+    // If we have context, add a specific query for that context
+    if (context) {
+      subQueries.push({
+        id: this.generateSubQueryId(),
+        text: `How does "${context}" relate to ${query}?`,
+        reason: "Contextual relationship exploration",
+        isAnswered: false
+      });
+    }
+
+    return subQueries;
+  }
+
+  /**
+   * Identify parts of a complex question that could be individual sub-questions
+   *
+   * @param query The complex query to analyze
+   * @returns Array of potential sub-questions
+   */
+  private identifyQuestionParts(query: string): string[] {
+    const parts: string[] = [];
+
+    // Check for multiple question marks
+    const questionSentences = query.split(/(?<=\?)/).filter(s => s.includes('?'));
+    if (questionSentences.length > 1) {
+      // Multiple explicit questions detected
+      return questionSentences.map(s => s.trim());
+    }
+
+    // Check for conjunctions that might separate multiple questions
+    const conjunctions = ['and', 'or', 'but', 'plus', 'also'];
+    for (const conjunction of conjunctions) {
+      const pattern = new RegExp(`\\b${conjunction}\\b`, 'i');
+      if (pattern.test(query)) {
+        // Split by conjunction and check if each part could be a question
+        const splitParts = query.split(pattern);
+        for (const part of splitParts) {
+          const trimmed = part.trim();
+          if (trimmed.length > 10) { // Avoid tiny fragments
+            parts.push(trimmed);
+          }
+        }
+        if (parts.length > 0) {
+          return parts;
+        }
+      }
+    }
+
+    // Check for comparison indicators
+    const comparisonTerms = ['compare', 'difference', 'differences', 'versus', 'vs'];
+    for (const term of comparisonTerms) {
+      if (query.toLowerCase().includes(term)) {
+        // This is likely a comparison question, extract the items being compared
+        const beforeAfter = query.split(new RegExp(`\\b${term}\\b`, 'i'));
+        if (beforeAfter.length === 2) {
+          // Try to extract compared items
+          const aspects = this.extractComparisonAspects(beforeAfter[0], beforeAfter[1]);
+          if (aspects.length > 0) {
+            for (const aspect of aspects) {
+              parts.push(`What are the key points about ${aspect}?`);
+            }
+            parts.push(`What are the differences between ${aspects.join(' and ')}?`);
+            return parts;
+          }
+        }
+      }
+    }
+
+    // Check for "multiple aspects" questions
+    const aspectPatterns = [
+      /what (?:are|is) the (\w+) (?:of|about|for|in) /i,
+      /how (?:to|do|does|can) .+ (\w+)/i
+    ];
+
+    for (const pattern of aspectPatterns) {
+      const match = query.match(pattern);
+      if (match && match[1]) {
+        const aspect = match[1];
+        parts.push(`What is the ${aspect}?`);
+        parts.push(`How does ${aspect} relate to the main topic?`);
+      }
+    }
+
+    return parts;
+  }
+
+  /**
+   * Extract items being compared from a comparison question
+   *
+   * @param before Text before the comparison term
+   * @param after Text after the comparison term
+   * @returns Array of items being compared
+   */
+  private extractComparisonAspects(before: string, after: string): string[] {
+    const aspects: string[] = [];
+
+    // Look for "between A and B" pattern
+    const betweenMatch = after.match(/between (.+?) and (.+?)(?:\?|$)/i);
+    if (betweenMatch) {
+      aspects.push(betweenMatch[1].trim());
+      aspects.push(betweenMatch[2].trim());
+      return aspects;
+    }
+
+    // Look for A vs B pattern
+    const directComparison = after.match(/(.+?) (?:and|vs|versus) (.+?)(?:\?|$)/i);
+    if (directComparison) {
+      aspects.push(directComparison[1].trim());
+      aspects.push(directComparison[2].trim());
+      return aspects;
+    }
+
+    // Fall back to looking for named entities or key terms in both parts
+    const beforeTerms = before.match(/(\w+(?:\s+\w+){0,2})/g) || [];
+    const afterTerms = after.match(/(\w+(?:\s+\w+){0,2})/g) || [];
+
+    // Look for substantial terms (longer than 3 chars)
+    const candidateTerms = [...beforeTerms, ...afterTerms]
+      .filter(term => term.length > 3)
+      .map(term => term.trim());
+
+    // Take up to 2 distinct terms
+    return [...new Set(candidateTerms)].slice(0, 2);
+  }
+
+  /**
+   * Generate a unique ID for a sub-query
+   *
+   * @returns A unique sub-query ID
+   */
+  private generateSubQueryId(): string {
+    QueryProcessor.queryCounter++;
+    return `sq_${Date.now()}_${QueryProcessor.queryCounter}`;
+  }
+
+  /**
+   * Assess the complexity of a query on a scale of 1-10
+   * This helps determine if decomposition is needed
+   *
+   * @param query The query to assess
+   * @returns A complexity score from 1-10
+   */
+  assessQueryComplexity(query: string): number {
+    let score = 0;
+
+    // Factor 1: Length - longer queries tend to be more complex
+    // 0-1.5 points for length
+    const lengthScore = Math.min(query.length / 100, 1.5);
+    score += lengthScore;
+
+    // Factor 2: Question marks - multiple questions are more complex
+    // 0-2 points for question marks
+    const questionMarkCount = (query.match(/\?/g) || []).length;
+    score += Math.min(questionMarkCount * 0.8, 2);
+
+    // Factor 3: Question words - multiple "wh" questions indicate complexity
+    // 0-2 points for question words
+    const questionWords = ['what', 'why', 'how', 'when', 'where', 'who', 'which'];
+    let questionWordCount = 0;
+
+    for (const word of questionWords) {
+      const regex = new RegExp(`\\b${word}\\b`, 'gi');
+      questionWordCount += (query.match(regex) || []).length;
+    }
+
+    score += Math.min(questionWordCount * 0.5, 2);
+
+    // Factor 4: Conjunctions - linking multiple concepts increases complexity
+    // 0-1.5 points for conjunctions
+    const conjunctions = ['and', 'or', 'but', 'however', 'although', 'nevertheless', 'despite', 'whereas'];
+    let conjunctionCount = 0;
+
+    for (const conj of conjunctions) {
+      const regex = new RegExp(`\\b${conj}\\b`, 'gi');
+      conjunctionCount += (query.match(regex) || []).length;
+    }
+
+    score += Math.min(conjunctionCount * 0.3, 1.5);
+
+    // Factor 5: Comparison terms - comparisons are complex
+    // 0-1.5 points for comparison terms
+    const comparisonTerms = ['compare', 'difference', 'differences', 'versus', 'vs', 'similarities', 'better', 'worse'];
+    let comparisonCount = 0;
+
+    for (const term of comparisonTerms) {
+      const regex = new RegExp(`\\b${term}\\b`, 'gi');
+      comparisonCount += (query.match(regex) || []).length;
+    }
+
+    score += Math.min(comparisonCount * 0.7, 1.5);
+
+    // Factor 6: Technical terms and depth indicators
+    // 0-1.5 points for depth indicators
+    const depthTerms = ['explain', 'detail', 'elaborate', 'in-depth', 'comprehensive', 'thoroughly', 'analysis'];
+    let depthCount = 0;
+
+    for (const term of depthTerms) {
+      const regex = new RegExp(`\\b${term}\\b`, 'gi');
+      depthCount += (query.match(regex) || []).length;
+    }
+
+    score += Math.min(depthCount * 0.5, 1.5);
+
+    // Return final score, capped at 10
+    return Math.min(Math.round(score), 10);
+  }
+
+  /**
+   * Update a sub-query with its answer
+   *
+   * @param decomposedQuery The decomposed query object
+   * @param subQueryId The ID of the sub-query to update
+   * @param answer The answer to the sub-query
+   * @returns The updated decomposed query
+   */
+  updateSubQueryAnswer(
+    decomposedQuery: DecomposedQuery,
+    subQueryId: string,
+    answer: string
+  ): DecomposedQuery {
+    const updatedSubQueries = decomposedQuery.subQueries.map(sq => {
+      if (sq.id === subQueryId) {
+        return {
+          ...sq,
+          answer,
+          isAnswered: true
+        };
+      }
+      return sq;
+    });
+
+    // Check if all sub-queries are answered
+    const allAnswered = updatedSubQueries.every(sq => sq.isAnswered);
+
+    return {
+      ...decomposedQuery,
+      subQueries: updatedSubQueries,
+      status: allAnswered ? 'completed' : 'in_progress'
+    };
+  }
+
+  /**
+   * Synthesize all sub-query answers into a comprehensive response
+   *
+   * @param decomposedQuery The decomposed query with all sub-queries answered
+   * @returns A synthesized answer to the original query
+   */
+  synthesizeAnswer(decomposedQuery: DecomposedQuery): string {
+    try {
+      // Ensure all sub-queries are answered
+      if (!decomposedQuery.subQueries.every(sq => sq.isAnswered)) {
+        return "Cannot synthesize answer until all sub-queries are answered.";
+      }
+
+      // For simple queries with just one sub-query, return the answer directly
+      if (decomposedQuery.subQueries.length === 1) {
+        return decomposedQuery.subQueries[0].answer || "";
+      }
+
+      // For complex queries, build a structured response
+      let synthesized = `Answer to: ${decomposedQuery.originalQuery}\n\n`;
+
+      // Group by themes if there are many sub-queries
+      if (decomposedQuery.subQueries.length > 3) {
+        synthesized += "Based on the information gathered:\n\n";
+
+        for (const sq of decomposedQuery.subQueries) {
+          synthesized += `${sq.answer}\n\n`;
+        }
+      } else {
+        // For fewer sub-queries, present each one with its question
+        for (const sq of decomposedQuery.subQueries) {
+          synthesized += `${sq.answer}\n\n`;
+        }
+      }
+
+      return synthesized.trim();
+    } catch (error: any) {
+      log.error(`Error synthesizing answer: ${error.message}`);
+      return "An error occurred while synthesizing the answer.";
+    }
+  }
+}
+
+// Export a singleton instance
+export default new QueryProcessor();

src/services/llm/context/services/vector_search_service.ts

@@ -0,0 +1,381 @@
+/**
+ * Unified Vector Search Service
+ *
+ * Consolidates functionality from:
+ * - semantic_search.ts
+ * - vector_search_stage.ts
+ *
+ * This service provides a central interface for all vector search operations,
+ * supporting both full and summarized note context extraction.
+ */
+
+import * as vectorStore from '../../embeddings/index.js';
+import { cosineSimilarity } from '../../embeddings/index.js';
+import log from '../../../log.js';
+import becca from '../../../../becca/becca.js';
+import providerManager from '../modules/provider_manager.js';
+import cacheManager from '../modules/cache_manager.js';
+import type { NoteSearchResult } from '../../interfaces/context_interfaces.js';
+import type { LLMServiceInterface } from '../../interfaces/agent_tool_interfaces.js';
+
+export interface VectorSearchOptions {
+  maxResults?: number;
+  threshold?: number;
+  useEnhancedQueries?: boolean;
+  summarizeContent?: boolean;
+  llmService?: LLMServiceInterface | null;
+}
+
+export class VectorSearchService {
+  private contextExtractor: any;
+
+  constructor() {
+    // Lazy load the context extractor to avoid circular dependencies
+    import('../index.js').then(module => {
+      this.contextExtractor = new module.ContextExtractor();
+    });
+  }
+
+  /**
+   * Find notes that are semantically relevant to a query
+   *
+   * @param query - The search query
+   * @param contextNoteId - Optional note ID to restrict search to a branch
+   * @param options - Search options including result limit and summarization preference
+   * @returns Array of relevant notes with similarity scores
+   */
+  async findRelevantNotes(
+    query: string,
+    contextNoteId: string | null = null,
+    options: VectorSearchOptions = {}
+  ): Promise<NoteSearchResult[]> {
+    const {
+      maxResults = 10,
+      threshold = 0.6,
+      useEnhancedQueries = false,
+      summarizeContent = false,
+      llmService = null
+    } = options;
+
+    log.info(`VectorSearchService: Finding relevant notes for "${query.substring(0, 50)}..."`);
+    log.info(`Parameters: contextNoteId=${contextNoteId || 'global'}, maxResults=${maxResults}, summarize=${summarizeContent}`);
+
+    try {
+      // Check cache first
+      const cacheKey = `find:${query}:${contextNoteId || 'all'}:${maxResults}:${summarizeContent}`;
+      const cached = cacheManager.getQueryResults<NoteSearchResult[]>(cacheKey);
+      if (cached && Array.isArray(cached)) {
+        log.info(`VectorSearchService: Returning ${cached.length} cached results`);
+        return cached;
+      }
+
+      // Get embedding for query
+      const queryEmbedding = await providerManager.generateQueryEmbedding(query);
+      if (!queryEmbedding) {
+        log.error('Failed to generate query embedding');
+        return [];
+      }
+
+      // Get provider information
+      const provider = await providerManager.getPreferredEmbeddingProvider();
+      if (!provider) {
+        log.error('No embedding provider available');
+        return [];
+      }
+
+      // Find similar notes based on embeddings
+      let noteResults: {noteId: string, similarity: number}[] = [];
+
+      // If contextNoteId is provided, search only within that branch
+      if (contextNoteId) {
+        noteResults = await this.findNotesInBranch(
+          queryEmbedding,
+          contextNoteId,
+          maxResults
+        );
+      } else {
+        // Otherwise search across all notes with embeddings
+        noteResults = await vectorStore.findSimilarNotes(
+          queryEmbedding,
+          provider.name,
+          provider.getConfig().model || '',
+          maxResults
+        );
+      }
+
+      // Ensure context extractor is loaded
+      if (!this.contextExtractor) {
+        const module = await import('../index.js');
+        this.contextExtractor = new module.ContextExtractor();
+      }
+
+      // Get note details for results
+      const enrichedResults = await Promise.all(
+        noteResults.map(async result => {
+          const note = becca.getNote(result.noteId);
+          if (!note) {
+            return null;
+          }
+
+          // Get note content - full or summarized based on option
+          let content: string | null = null;
+
+          if (summarizeContent) {
+            content = await this.getSummarizedNoteContent(result.noteId, llmService);
+          } else {
+            content = await this.contextExtractor.getNoteContent(result.noteId);
+          }
+
+          // Adjust similarity score based on content quality
+          let adjustedSimilarity = result.similarity;
+
+          // Penalize notes with empty or minimal content
+          if (!content || content.trim().length <= 10) {
+            adjustedSimilarity *= 0.2;
+          }
+          // Slightly boost notes with substantial content
+          else if (content.length > 100) {
+            adjustedSimilarity = Math.min(1.0, adjustedSimilarity * 1.1);
+          }
+
+          // Get primary parent note ID
+          const parentNotes = note.getParentNotes();
+          const parentId = parentNotes.length > 0 ? parentNotes[0].noteId : undefined;
+
+          // Create parent chain for context
+          const parentPath = await this.getParentPath(result.noteId);
+
+          return {
+            noteId: result.noteId,
+            title: note.title,
+            content,
+            similarity: adjustedSimilarity,
+            parentId,
+            parentPath
+          };
+        })
+      );
+
+      // Filter out null results and notes with very low similarity
+      const filteredResults = enrichedResults.filter(result =>
+        result !== null && result.similarity > threshold
+      ) as NoteSearchResult[];
+
+      // Sort results by adjusted similarity
+      filteredResults.sort((a, b) => b.similarity - a.similarity);
+
+      // Limit to requested number of results
+      const limitedResults = filteredResults.slice(0, maxResults);
+
+      // Cache results
+      cacheManager.storeQueryResults(cacheKey, limitedResults);
+
+      log.info(`VectorSearchService: Found ${limitedResults.length} relevant notes`);
+      return limitedResults;
+    } catch (error) {
+      log.error(`Error finding relevant notes: ${error}`);
+      return [];
+    }
+  }
+
+  /**
+   * Get a summarized version of note content
+   *
+   * @param noteId - The note ID to summarize
+   * @param llmService - Optional LLM service for summarization
+   * @returns Summarized content or full content if summarization fails
+   */
+  private async getSummarizedNoteContent(
+    noteId: string,
+    llmService: LLMServiceInterface | null
+  ): Promise<string | null> {
+    try {
+      // Get the full content first
+      const fullContent = await this.contextExtractor.getNoteContent(noteId);
+      if (!fullContent || fullContent.length < 500) {
+        // Don't summarize short content
+        return fullContent;
+      }
+
+      // Check if we have an LLM service for summarization
+      if (!llmService) {
+        // If no LLM service, truncate the content instead
+        return fullContent.substring(0, 500) + "...";
+      }
+
+      // Check cache for summarized content
+      const cacheKey = `summary:${noteId}:${fullContent.length}`;
+      const cached = cacheManager.getNoteData(noteId, cacheKey);
+      if (cached) {
+        return cached as string;
+      }
+
+      const note = becca.getNote(noteId);
+      if (!note) return null;
+
+      // Prepare a summarization prompt
+      const messages = [
+        {
+          role: "system" as const,
+          content: "Summarize the following note content concisely while preserving key information. Keep your summary to about 20% of the original length."
+        },
+        {
+          role: "user" as const,
+          content: `Note title: ${note.title}\n\nContent:\n${fullContent}`
+        }
+      ];
+
+      // Request summarization with safeguards to prevent recursion
+      const result = await llmService.generateChatCompletion(messages, {
+        temperature: 0.3,
+        maxTokens: 500,
+        // Use any to bypass type checking for these special options
+        // that are recognized by the LLM service but not in the interface
+        ...(({
+          bypassFormatter: true,
+          bypassContextProcessing: true,
+          enableTools: false
+        } as any))
+      });
+
+      const summary = result.text;
+
+      // Cache the summarization result
+      cacheManager.storeNoteData(noteId, cacheKey, summary);
+
+      return summary;
+    } catch (error) {
+      log.error(`Error summarizing note content: ${error}`);
+      // Fall back to getting the full content
+      return this.contextExtractor.getNoteContent(noteId);
+    }
+  }
+
+  /**
+   * Find notes in a specific branch (subtree) that are relevant to a query
+   *
+   * @param embedding - The query embedding
+   * @param contextNoteId - Root note ID of the branch
+   * @param limit - Maximum results to return
+   * @returns Array of note IDs with similarity scores
+   */
+  private async findNotesInBranch(
+    embedding: Float32Array,
+    contextNoteId: string,
+    limit = 5
+  ): Promise<{noteId: string, similarity: number}[]> {
+    try {
+      // Get all notes in the subtree
+      const noteIds = await this.getSubtreeNoteIds(contextNoteId);
+
+      if (noteIds.length === 0) {
+        return [];
+      }
+
+      // Get provider information
+      const provider = await providerManager.getPreferredEmbeddingProvider();
+      if (!provider) {
+        log.error('No embedding provider available');
+        return [];
+      }
+
+      // Get model configuration
+      const model = provider.getConfig().model || '';
+      const providerName = provider.name;
+
+      // Get embeddings for all notes in the branch
+      const results: {noteId: string, similarity: number}[] = [];
+
+      for (const noteId of noteIds) {
+        try {
+          // Get note embedding
+          const embeddingResult = await vectorStore.getEmbeddingForNote(
+            noteId,
+            providerName,
+            model
+          );
+
+          if (embeddingResult && embeddingResult.embedding) {
+            // Calculate similarity
+            const similarity = cosineSimilarity(embedding, embeddingResult.embedding);
+            results.push({ noteId, similarity });
+          }
+        } catch (error) {
+          log.error(`Error processing note ${noteId} for branch search: ${error}`);
+        }
+      }
+
+      // Sort by similarity and return top results
+      return results
+        .sort((a, b) => b.similarity - a.similarity)
+        .slice(0, limit);
+    } catch (error) {
+      log.error(`Error in branch search: ${error}`);
+      return [];
+    }
+  }
+
+  /**
+   * Get all note IDs in a subtree (branch)
+   *
+   * @param rootNoteId - The root note ID of the branch
+   * @returns Array of note IDs in the subtree
+   */
+  private async getSubtreeNoteIds(rootNoteId: string): Promise<string[]> {
+    try {
+      const note = becca.getNote(rootNoteId);
+      if (!note) return [];
+
+      const noteIds = new Set<string>([rootNoteId]);
+      const processChildNotes = async (noteId: string) => {
+        const childNotes = becca.getNote(noteId)?.getChildNotes() || [];
+        for (const childNote of childNotes) {
+          if (!noteIds.has(childNote.noteId)) {
+            noteIds.add(childNote.noteId);
+            await processChildNotes(childNote.noteId);
+          }
+        }
+      };
+
+      await processChildNotes(rootNoteId);
+      return Array.from(noteIds);
+    } catch (error) {
+      log.error(`Error getting subtree note IDs: ${error}`);
+      return [];
+    }
+  }
+
+  /**
+   * Get the parent path for a note (for additional context)
+   *
+   * @param noteId - The note ID to get the parent path for
+   * @returns String representation of the parent path
+   */
+  private async getParentPath(noteId: string): Promise<string> {
+    try {
+      const note = becca.getNote(noteId);
+      if (!note) return '';
+
+      const path: string[] = [];
+      const parentNotes = note.getParentNotes();
+      let currentNote = parentNotes.length > 0 ? parentNotes[0] : null;
+
+      // Build path up to 3 levels
+      let level = 0;
+      while (currentNote && level < 3) {
+        path.unshift(currentNote.title);
+        const grandParents = currentNote.getParentNotes();
+        currentNote = grandParents.length > 0 ? grandParents[0] : null;
+        level++;
+      }
+
+      return path.join(' > ');
+    } catch (error) {
+      log.error(`Error getting parent path: ${error}`);
+      return '';
+    }
+  }
+}
+
+// Export a singleton instance
+export default new VectorSearchService();

src/services/llm/context_extractors/index.ts

@@ -1,82 +1,108 @@
 /**
- * Agent Tools Index
+ * Context Extractors Module
  *
- * This file exports all available agent tools for use by the LLM.
+ * Provides tools for extracting context from notes, files, and other sources.
- * Tools are prioritized in order of importance/impact.
  */
 
-import { VectorSearchTool } from './vector_search_tool.js';
+import { ContextualThinkingTool } from './contextual_thinking_tool.js';
 import { NoteNavigatorTool } from './note_navigator_tool.js';
 import { QueryDecompositionTool } from './query_decomposition_tool.js';
-import { ContextualThinkingTool } from './contextual_thinking_tool.js';
+import { VectorSearchTool } from './vector_search_tool.js';
 
 // Import services needed for initialization
-import contextService from '../context_service.js';
+import contextService from '../context/services/context_service.js';
-import aiServiceManager from '../ai_service_manager.js';
 import log from '../../log.js';
 
 // Import interfaces
 import type {
-  IAgentToolsManager,
+  IContextualThinkingTool,
-  LLMServiceInterface,
-  IVectorSearchTool,
   INoteNavigatorTool,
   IQueryDecompositionTool,
-  IContextualThinkingTool
+  IVectorSearchTool
 } from '../interfaces/agent_tool_interfaces.js';
 
 /**
- * Manages all agent tools and provides a unified interface for the LLM agent
+ * Agent Tools Manager
+ *
+ * Manages and provides access to all available agent tools.
  */
-export class AgentToolsManager implements IAgentToolsManager {
+class AgentToolsManager {
   private vectorSearchTool: VectorSearchTool | null = null;
   private noteNavigatorTool: NoteNavigatorTool | null = null;
   private queryDecompositionTool: QueryDecompositionTool | null = null;
   private contextualThinkingTool: ContextualThinkingTool | null = null;
   private initialized = false;
 
-  constructor() {
+  /**
-    // Initialize tools only when requested to avoid circular dependencies
+   * Initialize all tools
-  }
+   */
+  async initialize(forceInit = false): Promise<void> {
+    if (this.initialized && !forceInit) {
+      return;
+    }
 
-  async initialize(aiServiceManager: LLMServiceInterface): Promise<void> {
     try {
-      if (this.initialized) {
+      log.info("Initializing agent tools");
-        return;
+
+      // Initialize the context service first
+      try {
+        await contextService.initialize();
+      } catch (error) {
+        log.error(`Error initializing context service: ${error}`);
+        // Continue anyway, some tools might work without the context service
       }
 
-      log.info("Initializing LLM agent tools...");
+      // Create tool instances
-
-      // Create tools
       this.vectorSearchTool = new VectorSearchTool();
       this.noteNavigatorTool = new NoteNavigatorTool();
       this.queryDecompositionTool = new QueryDecompositionTool();
       this.contextualThinkingTool = new ContextualThinkingTool();
 
       // Set context service in the vector search tool
-      this.vectorSearchTool.setContextService(contextService);
+      if (this.vectorSearchTool) {
+        this.vectorSearchTool.setContextService(contextService);
+      }
 
       this.initialized = true;
-      log.info("LLM agent tools initialized successfully");
+      log.info("Agent tools initialized successfully");
     } catch (error) {
       log.error(`Failed to initialize agent tools: ${error}`);
       throw error;
     }
   }
 
-  isInitialized(): boolean {
+  /**
-    return this.initialized;
+   * Get all available tools
+   */
+  getAllTools() {
+    return [
+      {
+        name: "vector_search",
+        description: "Searches your notes for semantically similar content",
+        function: this.vectorSearchTool?.search.bind(this.vectorSearchTool)
+      },
+      {
+        name: "navigate_to_note",
+        description: "Navigates to a specific note",
+        function: this.noteNavigatorTool?.getNoteInfo.bind(this.noteNavigatorTool)
+      },
+      {
+        name: "decompose_query",
+        description: "Breaks down a complex query into simpler sub-queries",
+        function: this.queryDecompositionTool?.decomposeQuery.bind(this.queryDecompositionTool)
+      },
+      {
+        name: "contextual_thinking",
+        description: "Provides structured thinking about a problem using available context",
+        function: this.contextualThinkingTool?.startThinking.bind(this.contextualThinkingTool)
+      }
+    ].filter(tool => tool.function !== undefined);
   }
 
   /**
-   * Get all available agent tools
+   * Get all tool objects (for direct access)
-   * @returns Object containing all initialized tools
    */
-  getAllTools() {
+  getTools() {
-    if (!this.initialized) {
-      throw new Error("Agent tools not initialized. Call initialize() first.");
-    }
-
     return {
       vectorSearch: this.vectorSearchTool as IVectorSearchTool,
       noteNavigator: this.noteNavigatorTool as INoteNavigatorTool,
@@ -84,53 +110,13 @@ export class AgentToolsManager implements IAgentToolsManager {
       contextualThinking: this.contextualThinkingTool as IContextualThinkingTool
     };
   }
-
-  /**
-   * Get the vector search tool
-   */
-  getVectorSearchTool(): IVectorSearchTool {
-    if (!this.initialized || !this.vectorSearchTool) {
-      throw new Error("Vector search tool not initialized");
-    }
-    return this.vectorSearchTool;
-  }
-
-  /**
-   * Get the note structure navigator tool
-   */
-  getNoteNavigatorTool(): INoteNavigatorTool {
-    if (!this.initialized || !this.noteNavigatorTool) {
-      throw new Error("Note navigator tool not initialized");
-    }
-    return this.noteNavigatorTool;
-  }
-
-  /**
-   * Get the query decomposition tool
-   */
-  getQueryDecompositionTool(): IQueryDecompositionTool {
-    if (!this.initialized || !this.queryDecompositionTool) {
-      throw new Error("Query decomposition tool not initialized");
-    }
-    return this.queryDecompositionTool;
-  }
-
-  /**
-   * Get the contextual thinking tool
-   */
-  getContextualThinkingTool(): IContextualThinkingTool {
-    if (!this.initialized || !this.contextualThinkingTool) {
-      throw new Error("Contextual thinking tool not initialized");
-    }
-    return this.contextualThinkingTool;
-  }
 }
 
-// Export a singleton instance
+// Create and export singleton instance
 const agentTools = new AgentToolsManager();
 export default agentTools;
 
-// Also export individual tool classes for direct use if needed
+// Export all tools for direct import if needed
 export {
   VectorSearchTool,
   NoteNavigatorTool,

src/services/llm/context_extractors/query_decomposition_tool.ts

@@ -1,534 +1,95 @@
 /**
- * Query Decomposition Tool
+ * Query Decomposition Tool - Compatibility Layer
  *
- * This tool helps the LLM agent break down complex user queries into
+ * This file provides backward compatibility with the new consolidated
- * sub-questions that can be answered individually and then synthesized
+ * query_processor.js implementation.
- * into a comprehensive response.
- *
- * Features:
- * - Analyze query complexity
- * - Extract multiple intents from a single question
- * - Create a multi-stage research plan
- * - Track progress through complex information gathering
- * 
- * Integration with pipeline architecture:
- * - Can use pipeline stages when available
- * - Falls back to direct methods when needed
  */
 
 import log from '../../log.js';
-import { AGENT_TOOL_PROMPTS } from '../constants/llm_prompt_constants.js';
+import queryProcessor from '../context/services/query_processor.js';
-import { QUERY_DECOMPOSITION_STRINGS } from '../constants/query_decomposition_constants.js';
+import type { SubQuery, DecomposedQuery } from '../context/services/query_processor.js';
-import aiServiceManager from '../ai_service_manager.js';
 
-export interface SubQuery {
+export type { SubQuery, DecomposedQuery };
-    id: string;
-    text: string;
-    reason: string;
-    isAnswered: boolean;
-    answer?: string;
-}
-
-export interface DecomposedQuery {
-    originalQuery: string;
-    subQueries: SubQuery[];
-    status: 'pending' | 'in_progress' | 'completed';
-    complexity: number;
-}
 
 export class QueryDecompositionTool {
-    private static queryCounter: number = 0;
+  /**
+   * Break down a complex query into smaller, more manageable sub-queries
+   *
+   * @param query The original user query
+   * @param context Optional context about the current note being viewed
+   * @returns A decomposed query object with sub-queries
+   */
+  decomposeQuery(query: string, context?: string): DecomposedQuery {
+    log.info('Using compatibility layer for QueryDecompositionTool.decomposeQuery');
+    return queryProcessor.decomposeQuery(query, context);
+  }
 
-    /**
+  /**
-     * Break down a complex query into smaller, more manageable sub-queries
+   * Update a sub-query with its answer
-     *
+   *
-     * @param query The original user query
+   * @param decomposedQuery The decomposed query object
-     * @param context Optional context about the current note being viewed
+   * @param subQueryId The ID of the sub-query to update
-     * @returns A decomposed query object with sub-queries
+   * @param answer The answer to the sub-query
-     */
+   * @returns The updated decomposed query
-    decomposeQuery(query: string, context?: string): DecomposedQuery {
+   */
-        try {
+  updateSubQueryAnswer(
-            // Log the decomposition attempt for tracking
+    decomposedQuery: DecomposedQuery,
-            log.info(QUERY_DECOMPOSITION_STRINGS.LOG_MESSAGES.DECOMPOSING_QUERY(query));
+    subQueryId: string,
+    answer: string
+  ): DecomposedQuery {
+    log.info('Using compatibility layer for QueryDecompositionTool.updateSubQueryAnswer');
+    return queryProcessor.updateSubQueryAnswer(decomposedQuery, subQueryId, answer);
+  }
 
-            if (!query || query.trim().length === 0) {
+  /**
-                log.info(QUERY_DECOMPOSITION_STRINGS.LOG_MESSAGES.EMPTY_QUERY);
+   * Synthesize all sub-query answers into a comprehensive response
-                return {
+   *
-                    originalQuery: query,
+   * @param decomposedQuery The decomposed query with all sub-queries answered
-                    subQueries: [],
+   * @returns A synthesized answer to the original query
-                    status: 'pending',
+   */
-                    complexity: 0
+  synthesizeAnswer(decomposedQuery: DecomposedQuery): string {
-                };
+    log.info('Using compatibility layer for QueryDecompositionTool.synthesizeAnswer');
-            }
+    return queryProcessor.synthesizeAnswer(decomposedQuery);
+  }
 
-            // Assess query complexity to determine if decomposition is needed
+  /**
-            const complexity = this.assessQueryComplexity(query);
+   * Generate a status report on the progress of answering a complex query
-            log.info(QUERY_DECOMPOSITION_STRINGS.LOG_MESSAGES.COMPLEXITY_ASSESSMENT(complexity));
+   *
+   * @param decomposedQuery The decomposed query
+   * @returns A status report string
+   */
+  getQueryStatus(decomposedQuery: DecomposedQuery): string {
+    log.info('Using compatibility layer for QueryDecompositionTool.getQueryStatus');
+    // This method doesn't exist directly in the new implementation
+    // We'll implement a simple fallback
 
-            // For simple queries, just return the original as a single sub-query
+    const answeredCount = decomposedQuery.subQueries.filter(sq => sq.isAnswered).length;
-            // Use a lower threshold (2 instead of 3) to decompose more queries
+    const totalCount = decomposedQuery.subQueries.length;
-            if (complexity < 2) {
-                log.info(QUERY_DECOMPOSITION_STRINGS.LOG_MESSAGES.SIMPLE_QUERY(complexity));
 
-                const mainSubQuery = {
+    let status = `Progress: ${answeredCount}/${totalCount} sub-queries answered\n\n`;
-                    id: this.generateSubQueryId(),
-                    text: query,
-                    reason: AGENT_TOOL_PROMPTS.QUERY_DECOMPOSITION.SUB_QUERY_DIRECT,
-                    isAnswered: false
-                };
 
-                // Still add a generic exploration query to get some related content
+    for (const sq of decomposedQuery.subQueries) {
-                const genericQuery = {
+      status += `${sq.isAnswered ? '✓' : '○'} ${sq.text}\n`;
-                    id: this.generateSubQueryId(),
+      if (sq.isAnswered && sq.answer) {
-                    text: QUERY_DECOMPOSITION_STRINGS.SUB_QUERY_TEMPLATES.INFORMATION_RELATED(query),
+        status += `Answer: ${sq.answer.substring(0, 100)}${sq.answer.length > 100 ? '...' : ''}\n`;
-                    reason: AGENT_TOOL_PROMPTS.QUERY_DECOMPOSITION.SUB_QUERY_GENERIC,
+      }
-                    isAnswered: false
+      status += '\n';
-                };
-
-                return {
-                    originalQuery: query,
-                    subQueries: [mainSubQuery, genericQuery],
-                    status: 'pending',
-                    complexity
-                };
-            }
-
-            // For complex queries, perform decomposition
-            const subQueries = this.createSubQueries(query, context);
-            log.info(QUERY_DECOMPOSITION_STRINGS.LOG_MESSAGES.DECOMPOSED_INTO(subQueries.length));
-
-            // Log the sub-queries for better visibility
-            subQueries.forEach((sq, index) => {
-                log.info(QUERY_DECOMPOSITION_STRINGS.LOG_MESSAGES.SUB_QUERY_LOG(index, sq.text, sq.reason));
-            });
-
-            return {
-                originalQuery: query,
-                subQueries,
-                status: 'pending',
-                complexity
-            };
-        } catch (error: any) {
-            log.error(QUERY_DECOMPOSITION_STRINGS.LOG_MESSAGES.ERROR_DECOMPOSING(error.message));
-
-            // Fallback to treating it as a simple query
-            return {
-                originalQuery: query,
-                subQueries: [{
-                    id: this.generateSubQueryId(),
-                    text: query,
-                    reason: AGENT_TOOL_PROMPTS.QUERY_DECOMPOSITION.SUB_QUERY_ERROR,
-                    isAnswered: false
-                }],
-                status: 'pending',
-                complexity: 1
-            };
-        }
     }
 
-    /**
+    return status;
-     * Update a sub-query with its answer
+  }
-     *
-     * @param decomposedQuery The decomposed query object
-     * @param subQueryId The ID of the sub-query to update
-     * @param answer The answer to the sub-query
-     * @returns The updated decomposed query
-     */
-    updateSubQueryAnswer(
-        decomposedQuery: DecomposedQuery,
-        subQueryId: string,
-        answer: string
-    ): DecomposedQuery {
-        const updatedSubQueries = decomposedQuery.subQueries.map(sq => {
-            if (sq.id === subQueryId) {
-                return {
-                    ...sq,
-                    answer,
-                    isAnswered: true
-                };
-            }
-            return sq;
-        });
 
-        // Check if all sub-queries are answered
+  /**
-        const allAnswered = updatedSubQueries.every(sq => sq.isAnswered);
+   * Assess the complexity of a query on a scale of 1-10
-
+   *
-        return {
+   * @param query The query to assess
-            ...decomposedQuery,
+   * @returns A complexity score from 1-10
-            subQueries: updatedSubQueries,
+   */
-            status: allAnswered ? 'completed' : 'in_progress'
+  assessQueryComplexity(query: string): number {
-        };
+    log.info('Using compatibility layer for QueryDecompositionTool.assessQueryComplexity');
-    }
+    return queryProcessor.assessQueryComplexity(query);
-
+  }
-    /**
-     * Synthesize all sub-query answers into a comprehensive response
-     *
-     * @param decomposedQuery The decomposed query with all sub-queries answered
-     * @returns A synthesized answer to the original query
-     */
-    synthesizeAnswer(decomposedQuery: DecomposedQuery): string {
-        try {
-            // Ensure all sub-queries are answered
-            if (!decomposedQuery.subQueries.every(sq => sq.isAnswered)) {
-                return QUERY_DECOMPOSITION_STRINGS.SYNTHESIS_TEMPLATES.CANNOT_SYNTHESIZE;
-            }
-
-            // For simple queries with just one sub-query, return the answer directly
-            if (decomposedQuery.subQueries.length === 1) {
-                return decomposedQuery.subQueries[0].answer || "";
-            }
-
-            // For complex queries, build a structured response that references each sub-answer
-            let synthesized = QUERY_DECOMPOSITION_STRINGS.SYNTHESIS_TEMPLATES.ANSWER_TO(decomposedQuery.originalQuery);
-
-            // Group by themes if there are many sub-queries
-            if (decomposedQuery.subQueries.length > 3) {
-                // Here we would ideally group related sub-queries, but for now we'll just present them in order
-                synthesized += QUERY_DECOMPOSITION_STRINGS.SYNTHESIS_TEMPLATES.BASED_ON_INFORMATION;
-
-                for (const sq of decomposedQuery.subQueries) {
-                    synthesized += `${sq.answer}\n\n`;
-                }
-            } else {
-                // For fewer sub-queries, present each one with its question
-                for (const sq of decomposedQuery.subQueries) {
-                    synthesized += `${sq.answer}\n\n`;
-                }
-            }
-
-            return synthesized.trim();
-        } catch (error: any) {
-            log.error(QUERY_DECOMPOSITION_STRINGS.LOG_MESSAGES.ERROR_SYNTHESIZING(error.message));
-            return QUERY_DECOMPOSITION_STRINGS.SYNTHESIS_TEMPLATES.ERROR_SYNTHESIZING;
-        }
-    }
-
-    /**
-     * Generate a status report on the progress of answering a complex query
-     *
-     * @param decomposedQuery The decomposed query
-     * @returns A status report string
-     */
-    getQueryStatus(decomposedQuery: DecomposedQuery): string {
-        const answeredCount = decomposedQuery.subQueries.filter(sq => sq.isAnswered).length;
-        const totalCount = decomposedQuery.subQueries.length;
-
-        let status = QUERY_DECOMPOSITION_STRINGS.STATUS_TEMPLATES.PROGRESS(answeredCount, totalCount);
-
-        for (const sq of decomposedQuery.subQueries) {
-            status += `${sq.isAnswered ? QUERY_DECOMPOSITION_STRINGS.STATUS_TEMPLATES.ANSWERED_MARKER : QUERY_DECOMPOSITION_STRINGS.STATUS_TEMPLATES.UNANSWERED_MARKER} ${sq.text}\n`;
-            if (sq.isAnswered) {
-                status += `${QUERY_DECOMPOSITION_STRINGS.STATUS_TEMPLATES.ANSWER_PREFIX}${this.truncateText(sq.answer || "", 100)}\n`;
-            }
-        }
-
-        return status;
-    }
-
-    /**
-     * Assess the complexity of a query on a scale of 1-10
-     * This helps determine how many sub-queries are needed
-     *
-     * @param query The query to assess
-     * @returns A complexity score from 1-10
-     */
-    assessQueryComplexity(query: string): number {
-        // Count the number of question marks as a basic indicator
-        const questionMarkCount = (query.match(/\?/g) || []).length;
-
-        // Count potential sub-questions based on question words
-        const questionWordMatches = QUERY_DECOMPOSITION_STRINGS.QUESTION_WORDS.map(word => {
-            const regex = new RegExp(`\\b${word}\\b`, 'gi');
-            return (query.match(regex) || []).length;
-        });
-
-        const questionWordCount = questionWordMatches.reduce((sum, count) => sum + count, 0);
-
-        // Look for conjunctions which might join multiple questions
-        const conjunctionPattern = new RegExp(`\\b(${QUERY_DECOMPOSITION_STRINGS.CONJUNCTIONS.join('|')})\\b`, 'gi');
-        const conjunctionCount = (query.match(conjunctionPattern) || []).length;
-
-        // Look for complex requirements
-        const comparisonPattern = new RegExp(`\\b(${QUERY_DECOMPOSITION_STRINGS.COMPARISON_TERMS.join('|')})\\b`, 'gi');
-        const comparisonCount = (query.match(comparisonPattern) || []).length;
-
-        const analysisPattern = new RegExp(`\\b(${QUERY_DECOMPOSITION_STRINGS.ANALYSIS_TERMS.join('|')})\\b`, 'gi');
-        const analysisCount = (query.match(analysisPattern) || []).length;
-
-        // Calculate base complexity
-        let complexity = 1;
-
-        // Add for multiple questions
-        complexity += Math.min(2, questionMarkCount);
-
-        // Add for question words beyond the first one
-        complexity += Math.min(2, Math.max(0, questionWordCount - 1));
-
-        // Add for conjunctions that might join questions
-        complexity += Math.min(2, conjunctionCount);
-
-        // Add for comparative/analytical requirements
-        complexity += Math.min(2, comparisonCount + analysisCount);
-
-        // Add for overall length/complexity
-        if (query.length > 100) complexity += 1;
-        if (query.length > 200) complexity += 1;
-
-        // Ensure we stay in the 1-10 range
-        return Math.max(1, Math.min(10, complexity));
-    }
-
-    /**
-     * Generate a unique ID for a sub-query
-     */
-    generateSubQueryId(): string {
-        return `sq_${Date.now()}_${Math.floor(Math.random() * 10000)}`;
-    }
-
-    /**
-     * Create sub-queries based on the original query
-     */
-    createSubQueries(query: string, context?: string): SubQuery[] {
-        // Simple rules to create sub-queries based on query content
-        const subQueries: SubQuery[] = [];
-
-        // Avoid creating subqueries that start with "Provide details about" or similar
-        // as these have been causing recursive loops
-        if (query.toLowerCase().includes(QUERY_DECOMPOSITION_STRINGS.QUERY_PATTERNS.PROVIDE_DETAILS_ABOUT) ||
-            query.toLowerCase().includes(QUERY_DECOMPOSITION_STRINGS.QUERY_PATTERNS.INFORMATION_RELATED_TO)) {
-            log.info(QUERY_DECOMPOSITION_STRINGS.LOG_MESSAGES.AVOIDING_RECURSIVE(query));
-            return [{
-                id: this.generateSubQueryId(),
-                text: query,
-                reason: AGENT_TOOL_PROMPTS.QUERY_DECOMPOSITION.SUB_QUERY_DIRECT_ANALYSIS,
-                isAnswered: false
-            }];
-        }
-
-        // First, add the original query as a sub-query (always)
-        subQueries.push({
-            id: this.generateSubQueryId(),
-            text: query,
-            reason: AGENT_TOOL_PROMPTS.QUERY_DECOMPOSITION.ORIGINAL_QUERY,
-            isAnswered: false
-        });
-
-        // Check for "compare", "difference", "versus" to identify comparison questions
-        if (
-            query.toLowerCase().includes(QUERY_DECOMPOSITION_STRINGS.QUERY_PATTERNS.COMPARE) ||
-            query.toLowerCase().includes(QUERY_DECOMPOSITION_STRINGS.QUERY_PATTERNS.DIFFERENCE_BETWEEN) ||
-            query.toLowerCase().includes(QUERY_DECOMPOSITION_STRINGS.QUERY_PATTERNS.VS) ||
-            query.toLowerCase().includes(QUERY_DECOMPOSITION_STRINGS.QUERY_PATTERNS.VERSUS)
-        ) {
-            // Extract entities to compare (simplified approach)
-            const entities = this.extractEntitiesForComparison(query);
-
-            if (entities.length >= 2) {
-                // Add sub-queries for each entity
-                entities.forEach(entity => {
-                    subQueries.push({
-                        id: this.generateSubQueryId(),
-                        text: QUERY_DECOMPOSITION_STRINGS.SUB_QUERY_TEMPLATES.KEY_CHARACTERISTICS(entity),
-                        reason: QUERY_DECOMPOSITION_STRINGS.SUB_QUERY_REASONS.GETTING_DETAILS(entity),
-                        isAnswered: false
-                    });
-                });
-
-                // Add explicit comparison sub-query
-                subQueries.push({
-                    id: this.generateSubQueryId(),
-                    text: QUERY_DECOMPOSITION_STRINGS.SUB_QUERY_TEMPLATES.COMPARISON_FEATURES(entities),
-                    reason: QUERY_DECOMPOSITION_STRINGS.SUB_QUERY_REASONS.DIRECT_COMPARISON,
-                    isAnswered: false
-                });
-            }
-        }
-        // Check for "how to" questions
-        else if (query.toLowerCase().includes(QUERY_DECOMPOSITION_STRINGS.QUERY_PATTERNS.HOW_TO)) {
-            const topic = query.replace(/how to /i, '').trim();
-
-            subQueries.push({
-                id: this.generateSubQueryId(),
-                text: QUERY_DECOMPOSITION_STRINGS.SUB_QUERY_TEMPLATES.STEPS_TO(topic),
-                reason: QUERY_DECOMPOSITION_STRINGS.SUB_QUERY_REASONS.FINDING_PROCEDURAL,
-                isAnswered: false
-            });
-
-            subQueries.push({
-                id: this.generateSubQueryId(),
-                text: QUERY_DECOMPOSITION_STRINGS.SUB_QUERY_TEMPLATES.CHALLENGES(topic),
-                reason: QUERY_DECOMPOSITION_STRINGS.SUB_QUERY_REASONS.IDENTIFYING_DIFFICULTIES,
-                isAnswered: false
-            });
-        }
-        // Check for "why" questions
-        else if (query.toLowerCase().startsWith(QUERY_DECOMPOSITION_STRINGS.QUERY_PATTERNS.WHY)) {
-            const topic = query.replace(/why /i, '').trim();
-
-            subQueries.push({
-                id: this.generateSubQueryId(),
-                text: QUERY_DECOMPOSITION_STRINGS.SUB_QUERY_TEMPLATES.CAUSES(topic),
-                reason: QUERY_DECOMPOSITION_STRINGS.SUB_QUERY_REASONS.IDENTIFYING_CAUSES,
-                isAnswered: false
-            });
-
-            subQueries.push({
-                id: this.generateSubQueryId(),
-                text: QUERY_DECOMPOSITION_STRINGS.SUB_QUERY_TEMPLATES.EVIDENCE(topic),
-                reason: QUERY_DECOMPOSITION_STRINGS.SUB_QUERY_REASONS.FINDING_EVIDENCE,
-                isAnswered: false
-            });
-        }
-        // Handle "what is" questions
-        else if (query.toLowerCase().startsWith(QUERY_DECOMPOSITION_STRINGS.QUERY_PATTERNS.WHAT_IS) ||
-                 query.toLowerCase().startsWith(QUERY_DECOMPOSITION_STRINGS.QUERY_PATTERNS.WHAT_ARE)) {
-            const topic = query.replace(/what (is|are) /i, '').trim().replace(/\?$/, '');
-
-            subQueries.push({
-                id: this.generateSubQueryId(),
-                text: QUERY_DECOMPOSITION_STRINGS.SUB_QUERY_TEMPLATES.DEFINITION(topic),
-                reason: QUERY_DECOMPOSITION_STRINGS.SUB_QUERY_REASONS.GETTING_DEFINITION,
-                isAnswered: false
-            });
-
-            subQueries.push({
-                id: this.generateSubQueryId(),
-                text: QUERY_DECOMPOSITION_STRINGS.SUB_QUERY_TEMPLATES.EXAMPLES(topic),
-                reason: QUERY_DECOMPOSITION_STRINGS.SUB_QUERY_REASONS.FINDING_EXAMPLES,
-                isAnswered: false
-            });
-        }
-
-        // If no specific sub-queries were added (beyond the original),
-        // generate generic exploratory sub-queries
-        if (subQueries.length <= 1) {
-            // Extract main entities/concepts from the query
-            const concepts = this.extractMainConcepts(query);
-
-            concepts.forEach(concept => {
-                // Don't create recursive or self-referential queries
-                if (!concept.toLowerCase().includes(QUERY_DECOMPOSITION_STRINGS.QUERY_PATTERNS.PROVIDE_DETAILS_ABOUT) &&
-                    !concept.toLowerCase().includes(QUERY_DECOMPOSITION_STRINGS.QUERY_PATTERNS.INFORMATION_RELATED_TO)) {
-                    subQueries.push({
-                        id: this.generateSubQueryId(),
-                        text: QUERY_DECOMPOSITION_STRINGS.SUB_QUERY_TEMPLATES.KEY_INFORMATION(concept),
-                        reason: QUERY_DECOMPOSITION_STRINGS.SUB_QUERY_REASONS.FINDING_INFORMATION(concept),
-                        isAnswered: false
-                    });
-                }
-            });
-        }
-
-        return subQueries;
-    }
-
-    /**
-     * Truncate text to a maximum length with ellipsis
-     */
-    private truncateText(text: string, maxLength: number): string {
-        if (text.length <= maxLength) return text;
-        return text.substring(0, maxLength - 3) + '...';
-    }
-
-    /**
-     * Extract entities for comparison from a query
-     *
-     * @param query The query to extract entities from
-     * @returns Array of entity strings
-     */
-    extractEntitiesForComparison(query: string): string[] {
-        // Try to match patterns like "compare X and Y" or "difference between X and Y"
-        const comparePattern = /\b(?:compare|difference between|similarities between)\s+([^,]+?)\s+(?:and|with|to)\s+([^,\?\.]+)/i;
-        const vsPattern = /\b([^,]+?)\s+(?:vs\.?|versus)\s+([^,\?\.]+)/i;
-
-        let match = query.match(comparePattern) || query.match(vsPattern);
-
-        if (match) {
-            return [match[1].trim(), match[2].trim()];
-        }
-
-        // If no pattern match, try to extract noun phrases
-        const words = query.split(/\s+/);
-        const potentialEntities = [];
-        let currentPhrase = '';
-
-        for (const word of words) {
-            // Skip common words that are unlikely to be part of entity names
-            const stopWordsPattern = new RegExp(`^(${QUERY_DECOMPOSITION_STRINGS.STOP_WORDS.join('|')})$`, 'i');
-            if (stopWordsPattern.test(word)) {
-                if (currentPhrase.trim()) {
-                    potentialEntities.push(currentPhrase.trim());
-                    currentPhrase = '';
-                }
-                continue;
-            }
-
-            currentPhrase += word + ' ';
-        }
-
-        if (currentPhrase.trim()) {
-            potentialEntities.push(currentPhrase.trim());
-        }
-
-        return potentialEntities.slice(0, 2); // Return at most 2 entities
-    }
-
-    /**
-     * Extract main concepts from a query
-     *
-     * @param query The query to extract concepts from
-     * @returns Array of concept strings
-     */
-    extractMainConcepts(query: string): string[] {
-        // Remove question words and common stop words
-        const stopWordsPattern = new RegExp(QUERY_DECOMPOSITION_STRINGS.STOP_WORDS.join('|'), 'gi');
-        const cleanedQuery = query.replace(stopWordsPattern, ' ');
-
-        // Split into words and filter out short words
-        const words = cleanedQuery.split(/\s+/).filter(word => word.length > 3);
-
-        // Count word frequency
-        const wordCounts: Record<string, number> = {};
-        for (const word of words) {
-            wordCounts[word.toLowerCase()] = (wordCounts[word.toLowerCase()] || 0) + 1;
-        }
-
-        // Sort by frequency
-        const sortedWords = Object.entries(wordCounts)
-            .sort((a, b) => b[1] - a[1])
-            .map(entry => entry[0]);
-
-        // Try to build meaningful phrases around top words
-        const conceptPhrases: string[] = [];
-
-        if (sortedWords.length === 0) {
-            // Fallback if no significant words found
-            return [query.trim()];
-        }
-
-        // Use the top 2-3 words to form concepts
-        for (let i = 0; i < Math.min(sortedWords.length, 3); i++) {
-            const word = sortedWords[i];
-
-            // Try to find the word in the original query and extract a small phrase around it
-            const wordIndex = query.toLowerCase().indexOf(word);
-            if (wordIndex >= 0) {
-                // Extract a window of text around the word (3 words before and after)
-                const start = Math.max(0, query.lastIndexOf(' ', wordIndex - 15) + 1);
-                const end = Math.min(query.length, query.indexOf(' ', wordIndex + word.length + 15));
-
-                if (end > start) {
-                    conceptPhrases.push(query.substring(start, end).trim());
-                } else {
-                    conceptPhrases.push(word);
-                }
-            } else {
-                conceptPhrases.push(word);
-            }
-        }
-
-        return conceptPhrases;
-    }
 }
 
-export default QueryDecompositionTool;
+// Export default instance for compatibility
+export default new QueryDecompositionTool();

src/services/llm/context_extractors/vector_search_tool.ts

@@ -7,14 +7,12 @@
  * - Extracting relevant sections from notes
  * - Providing relevant context for LLM to generate accurate responses
  *
- * The tool uses embeddings to find notes with similar semantic meaning,
+ * Updated to use the consolidated VectorSearchService
- * allowing the LLM to find relevant information even when exact keywords
- * are not present.
  */
 
 import log from '../../log.js';
-import { VectorSearchStage } from '../pipeline/stages/vector_search_stage.js';
+import type { ContextService } from '../context/services/context_service.js';
-import type { ContextService } from '../context/modules/context_service.js';
+import vectorSearchService from '../context/services/vector_search_service.js';
 
 export interface VectorSearchResult {
   noteId: string;
@@ -36,29 +34,19 @@ export interface SearchResultItem {
   dateModified?: string;
 }
 
-export interface ChunkSearchResultItem {
-  noteId: string;
-  noteTitle: string;
-  chunk: string;
-  similarity: number;
-  parentId?: string;
-}
-
 export interface VectorSearchOptions {
   limit?: number;
   threshold?: number;
   includeContent?: boolean;
+  summarize?: boolean;
 }
 
 export class VectorSearchTool {
   private contextService: any = null;
   private maxResults: number = 5;
-  private vectorSearchStage: VectorSearchStage;
 
   constructor() {
-    // Initialize the vector search stage
+    log.info('VectorSearchTool initialized using consolidated VectorSearchService');
-    this.vectorSearchStage = new VectorSearchStage();
-    log.info('VectorSearchTool initialized with VectorSearchStage pipeline component');
   }
 
   /**
@@ -82,46 +70,25 @@ export class VectorSearchTool {
       const options = {
         maxResults: searchOptions.limit || 15, // Increased from default
         threshold: searchOptions.threshold || 0.5, // Lower threshold to include more results
-        useEnhancedQueries: true, // Enable query enhancement by default
         includeContent: searchOptions.includeContent !== undefined ? searchOptions.includeContent : true,
+        summarizeContent: searchOptions.summarize || false,
         ...searchOptions
       };
 
       log.info(`Vector search: "${query.substring(0, 50)}..." with limit=${options.maxResults}, threshold=${options.threshold}`);
 
-      // Use the pipeline stage for vector search
+      // Use the consolidated vector search service
-      const result = await this.vectorSearchStage.execute({
+      const searchResults = await vectorSearchService.findRelevantNotes(
         query,
-        noteId: contextNoteId || null,
+        contextNoteId || null,
-        options: {
+        {
           maxResults: options.maxResults,
           threshold: options.threshold,
-          useEnhancedQueries: options.useEnhancedQueries
+          summarizeContent: options.summarizeContent
         }
-      });
+      );
 
-      const searchResults = result.searchResults;
+      log.info(`Vector search found ${searchResults.length} relevant notes`);
-      log.info(`Vector search found ${searchResults.length} relevant notes via pipeline`);
-
-      // If includeContent is true but we're missing content for some notes, fetch it
-      if (options.includeContent) {
-        for (let i = 0; i < searchResults.length; i++) {
-          const result = searchResults[i];
-          try {
-            // Get content if missing
-            if (!result.content) {
-              const noteContent = await import('../context/note_content.js');
-              const content = await noteContent.getNoteContent(result.noteId);
-              if (content) {
-                result.content = content.substring(0, 2000); // Limit to 2000 chars
-                log.info(`Added direct content for note ${result.noteId}, length: ${result.content.length} chars`);
-              }
-            }
-          } catch (error) {
-            log.error(`Error getting content for note ${result.noteId}: ${error}`);
-          }
-        }
-      }
 
       // Format results to match the expected VectorSearchResult interface
       return searchResults.map(note => ({
@@ -147,27 +114,29 @@ export class VectorSearchTool {
   async searchNotes(query: string, options: {
     parentNoteId?: string,
     maxResults?: number,
-    similarityThreshold?: number
+    similarityThreshold?: number,
+    summarize?: boolean
   } = {}): Promise<VectorSearchResult[]> {
     try {
       // Set defaults
       const maxResults = options.maxResults || this.maxResults;
       const threshold = options.similarityThreshold || 0.6;
       const parentNoteId = options.parentNoteId || null;
+      const summarize = options.summarize || false;
 
-      // Use the pipeline for consistent search behavior
+      // Use the consolidated vector search service
-      const result = await this.vectorSearchStage.execute({
+      const results = await vectorSearchService.findRelevantNotes(
         query,
-        noteId: parentNoteId,
+        parentNoteId,
-        options: {
+        {
           maxResults,
           threshold,
-          useEnhancedQueries: true
+          summarizeContent: summarize
         }
-      });
+      );
 
       // Format results to match the expected interface
-      return result.searchResults.map(result => ({
+      return results.map(result => ({
         noteId: result.noteId,
         title: result.title,
         contentPreview: result.content ?
@@ -190,7 +159,8 @@ export class VectorSearchTool {
   async searchContentChunks(query: string, options: {
     noteId?: string,
     maxResults?: number,
-    similarityThreshold?: number
+    similarityThreshold?: number,
+    summarize?: boolean
   } = {}): Promise<VectorSearchResult[]> {
     try {
       // For now, use the same implementation as searchNotes,
@@ -198,7 +168,8 @@ export class VectorSearchTool {
       return this.searchNotes(query, {
         parentNoteId: options.noteId,
         maxResults: options.maxResults,
-        similarityThreshold: options.similarityThreshold
+        similarityThreshold: options.similarityThreshold,
+        summarize: options.summarize
       });
     } catch (error) {
       log.error(`Error in vector chunk search: ${error}`);
@@ -231,4 +202,4 @@ export class VectorSearchTool {
   }
 }
 
-export default VectorSearchTool;
+export default new VectorSearchTool();

src/services/llm/context_service.ts

@@ -1,322 +0,0 @@
-/**
- * Trilium Notes Context Service
- *
- * Unified entry point for all context-related services
- * Provides intelligent context management for AI features
- */
-
-import log from '../log.js';
-import contextService from './context/modules/context_service.js';
-import { ContextExtractor } from './context/index.js';
-import type { NoteSearchResult } from './interfaces/context_interfaces.js';
-import type { Message } from './ai_interface.js';
-import type { LLMServiceInterface } from './interfaces/agent_tool_interfaces.js';
-import { MessageFormatterFactory } from './interfaces/message_formatter.js';
-
-/**
- * Main Context Service for Trilium Notes
- *
- * This service provides a unified interface for all context-related functionality:
- * - Processing user queries with semantic search
- * - Finding relevant notes using AI-enhanced query understanding
- * - Progressive context loading based on query complexity
- * - Semantic context extraction
- * - Context formatting for different LLM providers
- *
- * This implementation uses a modular approach with specialized services:
- * - Provider management
- * - Cache management
- * - Semantic search
- * - Query enhancement
- * - Context formatting
- */
-class TriliumContextService {
-    private contextExtractor: ContextExtractor;
-
-    constructor() {
-        this.contextExtractor = new ContextExtractor();
-        log.info('TriliumContextService created');
-    }
-
-    /**
-     * Initialize the context service
-     */
-    async initialize(): Promise<void> {
-        return contextService.initialize();
-    }
-
-    /**
-     * Process a user query to find relevant context in Trilium notes
-     *
-     * @param userQuestion - The user's query
-     * @param llmService - The LLM service to use for query enhancement
-     * @param contextNoteId - Optional note ID to restrict search to a branch
-     * @param showThinking - Whether to show the LLM's thinking process
-     * @returns Context information and relevant notes
-     */
-    async processQuery(
-        userQuestion: string,
-        llmService: any,
-        contextNoteId: string | null = null,
-        showThinking: boolean = false
-    ) {
-        return contextService.processQuery(userQuestion, llmService, contextNoteId, showThinking);
-    }
-
-    /**
-     * Get context enhanced with agent tools
-     *
-     * @param noteId - The current note ID
-     * @param query - The user's query
-     * @param showThinking - Whether to show thinking process
-     * @param relevantNotes - Optional pre-found relevant notes
-     * @returns Enhanced context string
-     */
-    async getAgentToolsContext(
-        noteId: string,
-        query: string,
-        showThinking: boolean = false,
-        relevantNotes: Array<any> = []
-    ): Promise<string> {
-        return contextService.getAgentToolsContext(noteId, query, showThinking, relevantNotes);
-    }
-
-    /**
-     * Build formatted context from notes
-     *
-     * @param sources - Array of notes or content sources
-     * @param query - The original user query
-     * @returns Formatted context string
-     */
-    async buildContextFromNotes(sources: NoteSearchResult[], query: string): Promise<string> {
-        const provider = await (await import('./context/modules/provider_manager.js')).default.getPreferredEmbeddingProvider();
-        const providerId = provider?.name || 'default';
-        return (await import('./context/modules/context_formatter.js')).default.buildContextFromNotes(sources, query, providerId);
-    }
-
-    /**
-     * Find relevant notes using multi-query approach
-     *
-     * @param queries - Array of search queries
-     * @param contextNoteId - Optional note ID to restrict search
-     * @param limit - Maximum notes to return
-     * @returns Array of relevant notes
-     */
-    async findRelevantNotesMultiQuery(
-        queries: string[],
-        contextNoteId: string | null = null,
-        limit = 10
-    ): Promise<any[]> {
-        try {
-            // Use the VectorSearchStage for all searches to ensure consistency
-            const VectorSearchStage = (await import('./pipeline/stages/vector_search_stage.js')).VectorSearchStage;
-            const vectorSearchStage = new VectorSearchStage();
-            
-            const allResults: Map<string, any> = new Map();
-            log.info(`Finding relevant notes for ${queries.length} queries in context ${contextNoteId || 'global'}`);
-
-            // Process each query in parallel using Promise.all for better performance
-            const searchPromises = queries.map(query => 
-                vectorSearchStage.execute({
-                    query,
-                    noteId: contextNoteId,
-                    options: {
-                        maxResults: Math.ceil(limit / queries.length), // Distribute limit among queries
-                        useEnhancedQueries: false, // Don't enhance the queries here, as they're already enhanced
-                        threshold: 0.5 // Lower threshold to get more diverse results
-                    }
-                })
-            );
-
-            const searchResults = await Promise.all(searchPromises);
-            
-            // Combine all results
-            for (let i = 0; i < searchResults.length; i++) {
-                const results = searchResults[i].searchResults;
-                log.info(`Query "${queries[i].substring(0, 30)}..." returned ${results.length} results`);
-                
-                // Combine results, avoiding duplicates
-                for (const result of results) {
-                    if (!allResults.has(result.noteId)) {
-                        allResults.set(result.noteId, result);
-                    } else {
-                        // If note already exists, update similarity to max of both values
-                        const existing = allResults.get(result.noteId);
-                        if (result.similarity > existing.similarity) {
-                            existing.similarity = result.similarity;
-                            allResults.set(result.noteId, existing);
-                        }
-                    }
-                }
-            }
-
-            // Convert map to array and limit to top results
-            const finalResults = Array.from(allResults.values())
-                .sort((a, b) => b.similarity - a.similarity)
-                .slice(0, limit);
-                
-            log.info(`Combined ${queries.length} queries into ${finalResults.length} final results`);
-            return finalResults;
-        } catch (error) {
-            log.error(`Error in findRelevantNotesMultiQuery: ${error}`);
-            // Fall back to legacy approach if the new approach fails
-            return this.findRelevantNotesMultiQueryLegacy(queries, contextNoteId, limit);
-        }
-    }
-    
-    /**
-     * Legacy implementation of multi-query search (for fallback)
-     * @private
-     */
-    private async findRelevantNotesMultiQueryLegacy(
-        queries: string[],
-        contextNoteId: string | null = null,
-        limit = 10
-    ): Promise<any[]> {
-        log.info(`Using legacy findRelevantNotesMultiQuery implementation for ${queries.length} queries`);
-        const allResults: Map<string, any> = new Map();
-
-        for (const query of queries) {
-            const results = await (await import('./context/modules/semantic_search.js')).default.findRelevantNotes(
-                query,
-                contextNoteId,
-                Math.ceil(limit / queries.length) // Distribute limit among queries
-            );
-
-            // Combine results, avoiding duplicates
-            for (const result of results) {
-                if (!allResults.has(result.noteId)) {
-                    allResults.set(result.noteId, result);
-                } else {
-                    // If note already exists, update similarity to max of both values
-                    const existing = allResults.get(result.noteId);
-                    if (result.similarity > existing.similarity) {
-                        existing.similarity = result.similarity;
-                        allResults.set(result.noteId, existing);
-                    }
-                }
-            }
-        }
-
-        // Convert map to array and limit to top results
-        return Array.from(allResults.values())
-            .sort((a, b) => b.similarity - a.similarity)
-            .slice(0, limit);
-    }
-
-    /**
-     * Generate search queries to find relevant information
-     *
-     * @param userQuestion - The user's question
-     * @param llmService - The LLM service to use for generating queries
-     * @returns Array of search queries
-     */
-    async generateSearchQueries(userQuestion: string, llmService: any): Promise<string[]> {
-        return (await import('./context/modules/query_enhancer.js')).default.generateSearchQueries(userQuestion, llmService);
-    }
-
-    /**
-     * Get semantic context for a note
-     *
-     * @param noteId - The note ID
-     * @param userQuery - The user's query
-     * @param maxResults - Maximum results to include
-     * @param messages - Optional conversation messages to adjust context size
-     * @returns Formatted context string
-     */
-    async getSemanticContext(noteId: string, userQuery: string, maxResults = 5, messages: Message[] = []): Promise<string> {
-        return contextService.getSemanticContext(noteId, userQuery, maxResults, messages);
-    }
-
-    /**
-     * Get progressive context based on depth level
-     *
-     * @param noteId - The note ID
-     * @param depth - Depth level (1-4)
-     * @returns Context string
-     */
-    async getProgressiveContext(noteId: string, depth = 1): Promise<string> {
-        return contextService.getProgressiveContext(noteId, depth);
-    }
-
-    /**
-     * Get smart context that adapts to query complexity
-     *
-     * @param noteId - The note ID
-     * @param userQuery - The user's query
-     * @returns Context string
-     */
-    async getSmartContext(noteId: string, userQuery: string): Promise<string> {
-        return contextService.getSmartContext(noteId, userQuery);
-    }
-
-    /**
-     * Clear all context caches
-     */
-    clearCaches(): void {
-        return contextService.clearCaches();
-    }
-
-    /**
-     * Builds messages with context for LLM service
-     * This takes a set of messages and adds context in the appropriate format for each LLM provider
-     *
-     * @param messages Array of messages to enhance with context
-     * @param context The context to add (built from relevant notes)
-     * @param llmService The LLM service to format messages for
-     * @returns Promise resolving to the messages array with context properly integrated
-     */
-    async buildMessagesWithContext(
-        messages: Message[],
-        context: string,
-        llmService: LLMServiceInterface
-    ): Promise<Message[]> {
-        try {
-            if (!messages || messages.length === 0) {
-                log.info('No messages provided to buildMessagesWithContext');
-                return [];
-            }
-
-            if (!context || context.trim() === '') {
-                log.info('No context provided to buildMessagesWithContext, returning original messages');
-                return messages;
-            }
-
-            // Get the provider name, handling service classes and raw provider names
-            let providerName: string;
-            if (typeof llmService === 'string') {
-                // If llmService is a string, assume it's the provider name
-                providerName = llmService;
-            } else if (llmService.constructor && llmService.constructor.name) {
-                // Extract provider name from service class name (e.g., OllamaService -> ollama)
-                providerName = llmService.constructor.name.replace('Service', '').toLowerCase();
-            } else {
-                // Fallback to default
-                providerName = 'default';
-            }
-
-            log.info(`Using formatter for provider: ${providerName}`);
-
-            // Get the appropriate formatter for this provider
-            const formatter = MessageFormatterFactory.getFormatter(providerName);
-
-            // Format messages with context using the provider-specific formatter
-            const formattedMessages = formatter.formatMessages(
-                messages,
-                undefined, // No system prompt override - use what's in the messages
-                context
-            );
-
-            log.info(`Formatted ${messages.length} messages into ${formattedMessages.length} messages for ${providerName}`);
-
-            return formattedMessages;
-        } catch (error) {
-            log.error(`Error building messages with context: ${error}`);
-            // Fallback to original messages in case of error
-            return messages;
-        }
-    }
-}
-
-// Export singleton instance
-export default new TriliumContextService();

src/services/llm/interfaces/context_interfaces.ts

@@ -32,11 +32,12 @@ export interface ICacheManager {
 export interface NoteSearchResult {
   noteId: string;
   title: string;
-  content?: string | null;
+  content: string | null;
-  type?: string;
-  mime?: string;
   similarity: number;
   parentId?: string;
+  parentPath?: string;
+  type?: string;
+  mime?: string;
   parentTitle?: string;
   dateCreated?: string;
   dateModified?: string;
@@ -58,17 +59,8 @@ export interface IContextFormatter {
  * Interface for query enhancer
  */
 export interface IQueryEnhancer {
-  generateSearchQueries(userQuestion: string, llmService: {
+  generateSearchQueries(question: string, llmService: any): Promise<string[]>;
-    generateChatCompletion: (messages: Array<{
+  estimateQueryComplexity(query: string): number;
-      role: 'user' | 'assistant' | 'system';
-      content: string;
-    }>, options?: {
-      temperature?: number;
-      maxTokens?: number;
-    }) => Promise<{
-      text: string;
-    }>;
-  }): Promise<string[]>;
 }
 
 /**
@@ -97,3 +89,25 @@ export interface IContentChunker {
   chunkContent(content: string, metadata?: Record<string, unknown>): ContentChunk[];
   chunkNoteContent(noteId: string, content: string, title: string): Promise<NoteChunk[]>;
 }
+
+/**
+ * Interface for context service
+ */
+export interface IContextService {
+  initialize(): Promise<void>;
+  processQuery(
+    userQuestion: string,
+    llmService: any,
+    contextNoteId?: string | null,
+    showThinking?: boolean
+  ): Promise<{ context: string; sources: NoteSearchResult[]; thinking?: string }>;
+  findRelevantNotes(
+    query: string,
+    contextNoteId?: string | null,
+    options?: {
+      maxResults?: number;
+      summarize?: boolean;
+      llmService?: any;
+    }
+  ): Promise<NoteSearchResult[]>;
+}

src/services/llm/pipeline/stages/agent_tools_context_stage.ts

@@ -3,30 +3,58 @@ import type { PipelineInput } from '../interfaces.js';
 import aiServiceManager from '../../ai_service_manager.js';
 import log from '../../../log.js';
 
-interface AgentToolsContextInput extends PipelineInput {
+export interface AgentToolsContextInput {
-    noteId: string;
+    noteId?: string;
-    query: string;
+    query?: string;
     showThinking?: boolean;
 }
 
+export interface AgentToolsContextOutput {
+    context: string;
+    noteId: string;
+    query: string;
+}
+
 /**
- * Pipeline stage for retrieving agent tools context
+ * Pipeline stage for adding LLM agent tools context
  */
-export class AgentToolsContextStage extends BasePipelineStage<AgentToolsContextInput, { context: string }> {
+export class AgentToolsContextStage {
     constructor() {
-        super('AgentToolsContext');
+        log.info('AgentToolsContextStage initialized');
     }
 
     /**
-     * Get enhanced context with agent tools
+     * Execute the agent tools context stage
      */
-    protected async process(input: AgentToolsContextInput): Promise<{ context: string }> {
+    async execute(input: AgentToolsContextInput): Promise<AgentToolsContextOutput> {
-        const { noteId, query, showThinking = false } = input;
+        return this.process(input);
-        log.info(`Getting agent tools context for note ${noteId}, query: ${query?.substring(0, 50)}..., showThinking: ${showThinking}`);
+    }
 
-        const contextService = aiServiceManager.getContextService();
+    /**
-        const context = await contextService.getAgentToolsContext(noteId, query, showThinking);
+     * Process the input and add agent tools context
+     */
+    protected async process(input: AgentToolsContextInput): Promise<AgentToolsContextOutput> {
+        const noteId = input.noteId || 'global';
+        const query = input.query || '';
+        const showThinking = !!input.showThinking;
 
-        return { context };
+        log.info(`AgentToolsContextStage: Getting agent tools context for noteId=${noteId}, query="${query.substring(0, 30)}...", showThinking=${showThinking}`);
+
+        try {
+            // Use the AI service manager to get agent tools context
+            const context = await aiServiceManager.getAgentToolsContext(noteId, query, showThinking);
+
+            log.info(`AgentToolsContextStage: Generated agent tools context (${context.length} chars)`);
+
+            return {
+                context,
+                noteId,
+                query
+            };
+        } catch (error: unknown) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            log.error(`AgentToolsContextStage: Error getting agent tools context: ${errorMessage}`);
+            throw error;
+        }
     }
 }

src/services/llm/pipeline/stages/context_extraction_stage.ts

@@ -3,31 +3,70 @@ import type { ContextExtractionInput } from '../interfaces.js';
 import aiServiceManager from '../../ai_service_manager.js';
 import log from '../../../log.js';
 
+/**
+ * Context Extraction Pipeline Stage
+ */
+
+export interface ContextExtractionOutput {
+    context: string;
+    noteId: string;
+    query: string;
+}
+
 /**
  * Pipeline stage for extracting context from notes
  */
-export class ContextExtractionStage extends BasePipelineStage<ContextExtractionInput, { context: string }> {
+export class ContextExtractionStage {
     constructor() {
-        super('ContextExtraction');
+        log.info('ContextExtractionStage initialized');
     }
 
     /**
-     * Extract context from a note
+     * Execute the context extraction stage
      */
-    protected async process(input: ContextExtractionInput): Promise<{ context: string }> {
+    async execute(input: ContextExtractionInput): Promise<ContextExtractionOutput> {
-        const { noteId, query, useSmartContext = true } = input;
+        return this.process(input);
-        log.info(`Extracting context from note ${noteId}, query: ${query?.substring(0, 50)}...`);
+    }
 
-        let context: string;
+    /**
+     * Process the input and extract context
+     */
+    protected async process(input: ContextExtractionInput): Promise<ContextExtractionOutput> {
+        const { useSmartContext = true } = input;
+        const noteId = input.noteId || 'global';
+        const query = input.query || '';
 
-        if (useSmartContext && query) {
+        log.info(`ContextExtractionStage: Extracting context for noteId=${noteId}, query="${query.substring(0, 30)}..."`);
-            // Use smart context that considers the query for better relevance
+
-            context = await aiServiceManager.getContextService().getSmartContext(noteId, query);
+        try {
-        } else {
+            let context = '';
-            // Fall back to full context if smart context is disabled or no query available
+
-            context = await aiServiceManager.getContextExtractor().getFullContext(noteId);
+            // Get enhanced context from the context service
+            const contextService = aiServiceManager.getContextService();
+            const llmService = aiServiceManager.getService();
+
+            if (contextService) {
+                // Use unified context service to get smart context
+                context = await contextService.processQuery(
+                    query,
+                    llmService,
+                    { contextNoteId: noteId }
+                ).then(result => result.context);
+
+                log.info(`ContextExtractionStage: Generated enhanced context (${context.length} chars)`);
+            } else {
+                log.info('ContextExtractionStage: Context service not available, using default context');
+            }
+
+            return {
+                context,
+                noteId,
+                query
+            };
+        } catch (error: unknown) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            log.error(`ContextExtractionStage: Error extracting context: ${errorMessage}`);
+            throw error;
         }
-
-        return { context };
     }
 }

src/services/llm/pipeline/stages/semantic_context_extraction_stage.ts

@@ -5,6 +5,8 @@ import log from '../../../log.js';
 import { VectorSearchStage } from './vector_search_stage.js';
 import contextFormatter from '../../context/modules/context_formatter.js';
 import providerManager from '../../context/modules/provider_manager.js';
+import type { NoteSearchResult } from '../../interfaces/context_interfaces.js';
+import type { Message } from '../../ai_interface.js';
 
 /**
  * Pipeline stage for extracting semantic context from notes

src/services/llm/pipeline/stages/vector_search_stage.ts

@@ -1,206 +1,87 @@
-import { BasePipelineStage } from '../pipeline_stage.js';
+/**
-import type { VectorSearchInput } from '../interfaces.js';
+ * Vector Search Stage
-import type { NoteSearchResult } from '../../interfaces/context_interfaces.js';
+ *
+ * Part of the chat pipeline that handles finding semantically relevant notes
+ * using vector similarity search.
+ */
+
 import log from '../../../log.js';
-import queryEnhancer from '../../context/modules/query_enhancer.js';
+import vectorSearchService from '../../context/services/vector_search_service.js';
-import semanticSearch from '../../context/modules/semantic_search.js';
+import type { NoteSearchResult } from '../../interfaces/context_interfaces.js';
-import aiServiceManager from '../../ai_service_manager.js';
+import type { LLMServiceInterface } from '../../interfaces/agent_tool_interfaces.js';
+
+export interface VectorSearchInput {
+  query: string;
+  noteId?: string;
+  options?: {
+    maxResults?: number;
+    threshold?: number;
+    useEnhancedQueries?: boolean;
+    llmService?: LLMServiceInterface;
+  };
+}
+
+export interface VectorSearchOutput {
+  searchResults: NoteSearchResult[];
+  originalQuery: string;
+  noteId: string;
+}
 
 /**
- * Pipeline stage for handling semantic vector search with query enhancement
+ * Pipeline stage for performing vector-based semantic search
- * This centralizes all semantic search operations into the pipeline
  */
-export class VectorSearchStage extends BasePipelineStage<VectorSearchInput, { 
+export class VectorSearchStage {
-    searchResults: NoteSearchResult[], 
+  constructor() {
-    enhancedQueries?: string[] 
+    log.info('VectorSearchStage initialized');
-}> {
+  }
-    constructor() {
-        super('VectorSearch');
-    }
 
-    /**
+  /**
-     * Execute semantic search with optional query enhancement
+   * Execute vector search to find relevant notes
-     */
+   */
-    protected async process(input: VectorSearchInput): Promise<{ 
+  async execute(input: VectorSearchInput): Promise<VectorSearchOutput> {
-        searchResults: NoteSearchResult[], 
+    const {
-        enhancedQueries?: string[] 
+      query,
-    }> {
+      noteId = 'global',
-        const { query, noteId, options = {} } = input;
+      options = {}
-        const { 
+    } = input;
-            maxResults = 10, 
-            useEnhancedQueries = true,
-            threshold = 0.6,
-            llmService = null
-        } = options;
 
-        log.info(`========== PIPELINE VECTOR SEARCH ==========`);
+    const {
-        log.info(`Query: "${query.substring(0, 100)}${query.length > 100 ? '...' : ''}"`);
+      maxResults = 10,
-        log.info(`Parameters: noteId=${noteId || 'global'}, maxResults=${maxResults}, useEnhancedQueries=${useEnhancedQueries}, threshold=${threshold}`);
+      threshold = 0.6,
-        log.info(`LLM Service provided: ${llmService ? 'yes' : 'no'}`);
+      useEnhancedQueries = false,
-        log.info(`Start timestamp: ${new Date().toISOString()}`);
+      llmService = undefined
+    } = options;
 
-        try {
+    log.info(`VectorSearchStage: Searching for "${query.substring(0, 50)}..."`);
-            // STEP 1: Generate enhanced search queries if requested
+    log.info(`Parameters: noteId=${noteId}, maxResults=${maxResults}, threshold=${threshold}`);
-            let searchQueries: string[] = [query];
 
-            if (useEnhancedQueries) {
+    try {
-                log.info(`PIPELINE VECTOR SEARCH: Generating enhanced queries for: "${query.substring(0, 50)}..."`);
+      // Find relevant notes using vector search service
-                
+      const searchResults = await vectorSearchService.findRelevantNotes(
-                try {
+        query,
-                    // Get the LLM service to use for query enhancement
+        noteId === 'global' ? null : noteId,
-                    let enhancementService = llmService;
+        {
-                    
+          maxResults,
-                    // If no service provided, use AI service manager to get the default service
+          threshold,
-                    if (!enhancementService) {
+          llmService
-                        log.info(`No LLM service provided, using default from AI service manager`);
-                        const manager = aiServiceManager.getInstance();
-                        const provider = manager.getPreferredProvider();
-                        enhancementService = manager.getService(provider);
-                        log.info(`Using preferred provider "${provider}" with service type ${enhancementService.constructor.name}`);
-                    }
-                    
-                    // Create a special service wrapper that prevents recursion
-                    const recursionPreventionService = {
-                        generateChatCompletion: async (messages: any, options: any) => {
-                            // Add flags to prevent recursive calls
-                            const safeOptions = {
-                                ...options,
-                                bypassFormatter: true,
-                                _bypassContextProcessing: true,
-                                bypassQueryEnhancement: true, // Critical flag
-                                directToolExecution: true,
-                                enableTools: false // Disable tools for query enhancement
-                            };
-                            
-                            // Use the actual service implementation but with safe options
-                            return enhancementService.generateChatCompletion(messages, safeOptions);
-                        }
-                    };
-                    
-                    // Call the query enhancer with the safe service
-                    searchQueries = await queryEnhancer.generateSearchQueries(query, recursionPreventionService);
-                    log.info(`PIPELINE VECTOR SEARCH: Generated ${searchQueries.length} enhanced queries`);
-                } catch (error) {
-                    log.error(`PIPELINE VECTOR SEARCH: Error generating search queries, using original: ${error}`);
-                    searchQueries = [query]; // Fall back to original query
-                }
-            } else {
-                log.info(`PIPELINE VECTOR SEARCH: Using direct query without enhancement: "${query}"`);
-            }
-            
-            // STEP 2: Find relevant notes for each query
-            const allResults = new Map<string, NoteSearchResult>();
-            log.info(`PIPELINE VECTOR SEARCH: Searching for ${searchQueries.length} queries`);
-            
-            for (const searchQuery of searchQueries) {
-                try {
-                    log.info(`PIPELINE VECTOR SEARCH: Processing query: "${searchQuery.substring(0, 50)}..."`);
-                    const results = await semanticSearch.findRelevantNotes(
-                        searchQuery,
-                        noteId || null,
-                        maxResults
-                    );
-                    
-                    log.info(`PIPELINE VECTOR SEARCH: Found ${results.length} results for query "${searchQuery.substring(0, 50)}..."`);
-                    
-                    // Combine results, avoiding duplicates and keeping the highest similarity score
-                    for (const result of results) {
-                        if (!allResults.has(result.noteId)) {
-                            allResults.set(result.noteId, result);
-                        } else {
-                            // If note already exists, update similarity to max of both values
-                            const existing = allResults.get(result.noteId);
-                            if (existing && result.similarity > existing.similarity) {
-                                existing.similarity = result.similarity;
-                                allResults.set(result.noteId, existing);
-                            }
-                        }
-                    }
-                } catch (error) {
-                    log.error(`PIPELINE VECTOR SEARCH: Error searching for query "${searchQuery}": ${error}`);
-                }
-            }
-            
-            // STEP 3: Convert to array, filter and sort
-            const filteredResults = Array.from(allResults.values())
-                .filter(note => {
-                    // Filter out notes with no content or very minimal content
-                    const hasContent = note.content && note.content.trim().length > 10;
-                    // Apply similarity threshold
-                    const meetsThreshold = note.similarity >= threshold;
-                    
-                    if (!hasContent) {
-                        log.info(`PIPELINE VECTOR SEARCH: Filtering out empty/minimal note: "${note.title}" (${note.noteId})`);
-                    }
-                    
-                    if (!meetsThreshold) {
-                        log.info(`PIPELINE VECTOR SEARCH: Filtering out low similarity note: "${note.title}" - ${Math.round(note.similarity * 100)}% < ${Math.round(threshold * 100)}%`);
-                    }
-                    
-                    return hasContent && meetsThreshold;
-                })
-                .sort((a, b) => b.similarity - a.similarity)
-                .slice(0, maxResults);
-            
-            log.info(`PIPELINE VECTOR SEARCH: Search complete, returning ${filteredResults.length} results after filtering`);
-            
-            // Log top results in detail
-            if (filteredResults.length > 0) {
-                log.info(`========== VECTOR SEARCH RESULTS ==========`);
-                log.info(`Found ${filteredResults.length} relevant notes after filtering`);
-                
-                const topResults = filteredResults.slice(0, 5); // Show top 5 for better diagnostics
-                topResults.forEach((result, idx) => {
-                    log.info(`Result ${idx+1}:`);
-                    log.info(`  Title: "${result.title}"`);
-                    log.info(`  NoteID: ${result.noteId}`);
-                    log.info(`  Similarity: ${Math.round(result.similarity * 100)}%`);
-                    
-                    if (result.content) {
-                        const contentPreview = result.content.length > 150 
-                            ? `${result.content.substring(0, 150)}...` 
-                            : result.content;
-                        log.info(`  Content preview: ${contentPreview}`);
-                        log.info(`  Content length: ${result.content.length} chars`);
-                    } else {
-                        log.info(`  Content: None or not loaded`);
-                    }
-                });
-                
-                if (filteredResults.length > 5) {
-                    log.info(`... and ${filteredResults.length - 5} more results not shown`);
-                }
-                
-                log.info(`========== END VECTOR SEARCH RESULTS ==========`);
-            } else {
-                log.info(`No results found that meet the similarity threshold of ${threshold}`);
-            }
-            
-            // Log final statistics
-            log.info(`Vector search statistics:`);
-            log.info(`  Original query: "${query.substring(0, 50)}${query.length > 50 ? '...' : ''}"`);
-            if (searchQueries.length > 1) {
-                log.info(`  Enhanced with ${searchQueries.length} search queries`);
-                searchQueries.forEach((q, i) => {
-                    if (i > 0) { // Skip the original query
-                        log.info(`    Query ${i}: "${q.substring(0, 50)}${q.length > 50 ? '...' : ''}"`);
-                    }
-                });
-            }
-            log.info(`  Final results: ${filteredResults.length} notes`);
-            log.info(`  End timestamp: ${new Date().toISOString()}`);
-            log.info(`========== END PIPELINE VECTOR SEARCH ==========`);
-            
-            return { 
-                searchResults: filteredResults,
-                enhancedQueries: useEnhancedQueries ? searchQueries : undefined
-            };
-        } catch (error: any) {
-            log.error(`PIPELINE VECTOR SEARCH: Error in vector search stage: ${error.message || String(error)}`);
-            return { 
-                searchResults: [],
-                enhancedQueries: undefined
-            };
         }
+      );
+
+      log.info(`VectorSearchStage: Found ${searchResults.length} relevant notes`);
+
+      return {
+        searchResults,
+        originalQuery: query,
+        noteId
+      };
+    } catch (error) {
+      log.error(`Error in vector search stage: ${error}`);
+      // Return empty results on error
+      return {
+        searchResults: [],
+        originalQuery: query,
+        noteId
+      };
     }
+  }
 }

src/services/llm/providers/ollama_service.ts

@@ -57,7 +57,14 @@ export class OllamaService extends BaseAIService {
         }
 
         const apiBase = options.getOption('ollamaBaseUrl');
-        const model = opts.model || options.getOption('ollamaDefaultModel') || 'llama3';
+
+        // Get the model name and strip the "ollama:" prefix if it exists
+        let model = opts.model || options.getOption('ollamaDefaultModel') || 'llama3';
+        if (model.startsWith('ollama:')) {
+            model = model.substring(7); // Remove the "ollama:" prefix
+            log.info(`Stripped 'ollama:' prefix from model name, using: ${model}`);
+        }
+
         const temperature = opts.temperature !== undefined
             ? opts.temperature
             : parseFloat(options.getOption('aiTemperature') || '0.7');

src/services/llm/rest_chat_service.ts

@@ -1,15 +1,18 @@
 import log from "../log.js";
 import type { Request, Response } from "express";
 import type { Message, ChatCompletionOptions } from "./ai_interface.js";
-import contextService from "./context_service.js";
+import contextService from "./context/services/context_service.js";
 import { LLM_CONSTANTS } from './constants/provider_constants.js';
 import { ERROR_PROMPTS } from './constants/llm_prompt_constants.js';
-import aiServiceManagerImport from "./ai_service_manager.js";
 import becca from "../../becca/becca.js";
 import vectorStore from "./embeddings/index.js";
 import providerManager from "./providers/providers.js";
 import options from "../../services/options.js";
 import { randomString } from "../utils.js";
+import type { LLMServiceInterface } from './interfaces/agent_tool_interfaces.js';
+import { AIServiceManager } from "./ai_service_manager.js";
+import { ChatPipeline } from "./pipeline/chat_pipeline.js";
+import type { ChatPipelineInput } from "./pipeline/interfaces.js";
 
 // Define interfaces for the REST API
 export interface NoteSource {
@@ -43,6 +46,37 @@ const sessions = new Map<string, ChatSession>();
 // Flag to track if cleanup timer has been initialized
 let cleanupInitialized = false;
 
+// For message formatting - simple implementation to avoid dependency
+const formatMessages = {
+    getFormatter(providerName: string) {
+        return {
+            formatMessages(messages: Message[], systemPrompt?: string, context?: string): Message[] {
+                // Simple implementation that works for most providers
+                const formattedMessages: Message[] = [];
+
+                // Add system message if context or systemPrompt is provided
+                if (context || systemPrompt) {
+                    formattedMessages.push({
+                        role: 'system',
+                        content: systemPrompt || (context ? `Use the following context to answer the query: ${context}` : '')
+                    });
+                }
+
+                // Add all other messages
+                for (const message of messages) {
+                    if (message.role === 'system' && formattedMessages.some(m => m.role === 'system')) {
+                        // Skip duplicate system messages
+                        continue;
+                    }
+                    formattedMessages.push(message);
+                }
+
+                return formattedMessages;
+            }
+        };
+    }
+};
+
 /**
  * Service to handle chat API interactions
  */
@@ -94,7 +128,8 @@ class RestChatService {
 
         // Try to access the manager - will create instance only if needed
         try {
-            const aiManager = aiServiceManagerImport.getInstance();
+            // Create local instance to avoid circular references
+            const aiManager = new AIServiceManager();
 
             if (!aiManager) {
                 log.info("AI check failed: AI manager module is not available");
@@ -278,12 +313,36 @@ class RestChatService {
                 throw new Error('Content cannot be empty');
             }
 
-            // Get session
+            // Check if session exists, create one if not
+            let session: ChatSession;
             if (!sessionId || !sessions.has(sessionId)) {
-                throw new Error('Session not found');
+                if (req.method === 'GET') {
+                    // For GET requests, we must have an existing session
+                    throw new Error('Session not found');
+                }
+
+                // For POST requests, we can create a new session automatically
+                log.info(`Session ${sessionId} not found, creating a new one automatically`);
+                const now = new Date();
+                session = {
+                    id: sessionId || randomString(16),
+                    title: 'Auto-created Session',
+                    messages: [],
+                    createdAt: now,
+                    lastActive: now,
+                    metadata: {
+                        temperature: 0.7,
+                        maxTokens: undefined,
+                        model: undefined,
+                        provider: undefined
+                    }
+                };
+                sessions.set(session.id, session);
+                log.info(`Created new session with ID: ${session.id}`);
+            } else {
+                session = sessions.get(sessionId)!;
             }
 
-            const session = sessions.get(sessionId)!;
             session.lastActive = new Date();
 
             // For POST requests, store the user message
@@ -315,7 +374,8 @@ class RestChatService {
                 log.info("AI services are not available - checking for specific issues");
 
                 try {
-                    const aiManager = aiServiceManagerImport.getInstance();
+                    // Create a direct instance to avoid circular references
+                    const aiManager = new AIServiceManager();
 
                     if (!aiManager) {
                         log.error("AI service manager is not initialized");
@@ -340,11 +400,11 @@ class RestChatService {
                 };
             }
 
-            // Get the AI service manager
+            // Create direct instance to avoid circular references
-            const aiServiceManager = aiServiceManagerImport.getInstance();
+            const aiManager = new AIServiceManager();
 
             // Get the default service - just use the first available one
-            const availableProviders = aiServiceManager.getAvailableProviders();
+            const availableProviders = aiManager.getAvailableProviders();
 
             if (availableProviders.length === 0) {
                 log.error("No AI providers are available after manager check");
@@ -360,7 +420,7 @@ class RestChatService {
             // We know the manager has a 'services' property from our code inspection,
             // but TypeScript doesn't know that from the interface.
             // This is a workaround to access it
-            const service = (aiServiceManager as any).services[providerName];
+            const service = (aiManager as any).services[providerName];
 
             if (!service) {
                 log.error(`AI service for provider ${providerName} not found`);
@@ -369,87 +429,69 @@ class RestChatService {
                 };
             }
 
-            // Information to return to the client
+            // Initialize tools
-            let aiResponse = '';
+            log.info("Initializing LLM agent tools...");
-            let sourceNotes: NoteSource[] = [];
+            // Ensure tools are initialized to prevent tool execution issues
+            await this.ensureToolsInitialized();
 
-            // Check if this is a streaming request
+            // Create and use the chat pipeline instead of direct processing
-            const isStreamingRequest = req.method === 'GET' && req.query.format === 'stream';
+            const pipeline = new ChatPipeline({
+                enableStreaming: req.method === 'GET',
+                enableMetrics: true,
+                maxToolCallIterations: 5
+            });
 
-            // For POST requests, we need to process the message
+            log.info("Executing chat pipeline...");
-            // For GET (streaming) requests, we use the latest user message from the session
-            if (req.method === 'POST' || isStreamingRequest) {
-                // Get the latest user message for context
-                const latestUserMessage = session.messages
-                    .filter(msg => msg.role === 'user')
-                    .pop();
 
-                if (!latestUserMessage && req.method === 'GET') {
+            // Prepare the pipeline input
-                    throw new Error('No user message found in session');
+            const pipelineInput: ChatPipelineInput = {
-                }
+                messages: session.messages.map(msg => ({
-
+                    role: msg.role as 'user' | 'assistant' | 'system',
-                // Use the latest message content for GET requests
+                    content: msg.content
-                const messageContent = req.method === 'POST' ? content : latestUserMessage!.content;
+                })),
-
+                query: content,
-                try {
+                noteId: session.noteContext ?? undefined,
-                    // If Advanced Context is enabled, we use the improved method
+                showThinking: showThinking,
-                    if (useAdvancedContext) {
+                options: {
-                        sourceNotes = await this.processAdvancedContext(
+                    useAdvancedContext: useAdvancedContext,
-                            messageContent,
+                    systemPrompt: session.messages.find(m => m.role === 'system')?.content,
-                            session,
+                    temperature: session.metadata.temperature,
-                            service,
+                    maxTokens: session.metadata.maxTokens,
-                            isStreamingRequest,
+                    model: session.metadata.model
-                            res,
+                },
-                            showThinking
+                streamCallback: req.method === 'GET' ? (data, done) => {
-                        );
+                    res.write(`data: ${JSON.stringify({ content: data, done })}\n\n`);
-                    } else {
+                    if (done) {
-                        sourceNotes = await this.processStandardContext(
+                        res.end();
-                            messageContent,
-                            session,
-                            service,
-                            isStreamingRequest,
-                            res
-                        );
                     }
-
+                } : undefined
-                    // For streaming requests we don't return anything as we've already sent the response
-                    if (isStreamingRequest) {
-                        return null;
-                    }
-
-                    // For POST requests, return the response
-                    if (req.method === 'POST') {
-                        // Get the latest assistant message for the response
-                        const latestAssistantMessage = session.messages
-                            .filter(msg => msg.role === 'assistant')
-                            .pop();
-
-                        return {
-                            content: latestAssistantMessage?.content || '',
-                            sources: sourceNotes.map(note => ({
-                                noteId: note.noteId,
-                                title: note.title,
-                                similarity: note.similarity
-                            }))
-                        };
-                    }
-                } catch (processingError: any) {
-                    log.error(`Error processing message: ${processingError}`);
-                    return {
-                        error: `Error processing your request: ${processingError.message}`
-                    };
-                }
-            }
-
-            // If it's not a POST or streaming GET request, return the session's message history
-            return {
-                id: session.id,
-                messages: session.messages
             };
-        } catch (error: any) {
+
-            log.error(`Error in LLM query processing: ${error}`);
+            // Execute the pipeline
+            const response = await pipeline.execute(pipelineInput);
+
+            // Handle the response
+            if (req.method === 'POST') {
+                // Add assistant message to session
+                session.messages.push({
+                    role: 'assistant',
+                    content: response.text || '',
+                    timestamp: new Date()
+                });
+
+                // Return the response
+                return {
+                    content: response.text || '',
+                    sources: (response as any).sources || []
+                };
+            } else {
+                // For streaming requests, we've already sent the response
+                return null;
+            }
+        } catch (processingError: any) {
+            log.error(`Error processing message: ${processingError}`);
             return {
-                error: ERROR_PROMPTS.USER_ERRORS.GENERAL_ERROR
+                error: `Error processing your request: ${processingError.message}`
             };
         }
     }
@@ -474,11 +516,14 @@ class RestChatService {
         // Log that we're calling contextService with the parameters
         log.info(`Using enhanced context with: noteId=${contextNoteId}, showThinking=${showThinking}`);
 
+        // Correct parameters for contextService.processQuery
         const results = await contextService.processQuery(
             messageContent,
             service,
-            contextNoteId,
+            {
-            showThinking
+                contextNoteId,
+                showThinking
+            }
         );
 
         // Get the generated context
@@ -492,7 +537,7 @@ class RestChatService {
         }));
 
         // Format messages for the LLM using the proper context
-        const aiMessages = await contextService.buildMessagesWithContext(
+        const aiMessages = await this.buildMessagesWithContext(
             session.messages.slice(-LLM_CONSTANTS.SESSION.MAX_SESSION_MESSAGES).map(msg => ({
                 role: msg.role,
                 content: msg.content
@@ -646,7 +691,7 @@ class RestChatService {
         const context = this.buildContextFromNotes(relevantNotes, messageContent);
 
         // Get messages with context properly formatted for the specific LLM provider
-        const aiMessages = await contextService.buildMessagesWithContext(
+        const aiMessages = await this.buildMessagesWithContext(
             session.messages.slice(-LLM_CONSTANTS.SESSION.MAX_SESSION_MESSAGES).map(msg => ({
                 role: msg.role,
                 content: msg.content
@@ -850,7 +895,8 @@ class RestChatService {
             try {
                 const toolInitializer = await import('./tools/tool_initializer.js');
                 await toolInitializer.default.initializeTools();
-                log.info(`Initialized ${toolRegistry.getAllTools().length} tools`);
+                const tools = toolRegistry.getAllTools();
+                log.info(`Successfully registered ${tools.length} LLM tools: ${tools.map(t => t.definition.function.name).join(', ')}`);
             } catch (error: unknown) {
                 const errorMessage = error instanceof Error ? error.message : String(error);
                 log.error(`Failed to initialize tools: ${errorMessage}`);
@@ -1155,29 +1201,89 @@ class RestChatService {
     }
 
     /**
-     * Ensure that LLM tools are properly initialized
+     * Ensure LLM tools are initialized
-     * This helps prevent issues with tool execution
      */
-    private async ensureToolsInitialized(): Promise<void> {
+    private async ensureToolsInitialized() {
         try {
-            log.info("Initializing LLM agent tools...");
+            log.info("Initializing LLM tools...");
 
-            // Initialize LLM tools without depending on aiServiceManager
+            // Import tool initializer and registry
-            const toolInitializer = await import('./tools/tool_initializer.js');
+            const toolInitializer = (await import('./tools/tool_initializer.js')).default;
-            await toolInitializer.default.initializeTools();
-
-            // Get the tool registry to check if tools were initialized
             const toolRegistry = (await import('./tools/tool_registry.js')).default;
-            const tools = toolRegistry.getAllTools();
-            log.info(`LLM tools initialized successfully: ${tools.length} tools available`);
 
-            // Log available tools
+            // Check if tools are already initialized
-            if (tools.length > 0) {
+            const registeredTools = toolRegistry.getAllTools();
-                log.info(`Available tools: ${tools.map(t => t.definition.function.name).join(', ')}`);
+
+            if (registeredTools.length === 0) {
+                // Initialize tools if none are registered
+                await toolInitializer.initializeTools();
+                const tools = toolRegistry.getAllTools();
+                log.info(`Successfully registered ${tools.length} LLM tools: ${tools.map(t => t.definition.function.name).join(', ')}`);
+            } else {
+                log.info(`LLM tools already initialized: ${registeredTools.length} tools available`);
             }
-        } catch (error: any) {
+
-            log.error(`Error initializing LLM tools: ${error.message}`);
+            // Get all available tools for logging
-            // Don't throw, just log the error to prevent breaking the pipeline
+            const availableTools = toolRegistry.getAllTools().map(t => t.definition.function.name);
+            log.info(`Available tools: ${availableTools.join(', ')}`);
+
+            log.info("LLM tools initialized successfully: " + availableTools.length + " tools available");
+            return true;
+        } catch (error) {
+            log.error(`Failed to initialize LLM tools: ${error}`);
+            return false;
+        }
+    }
+
+    // Function to build messages with context
+    private async buildMessagesWithContext(
+        messages: Message[],
+        context: string,
+        llmService: LLMServiceInterface
+    ): Promise<Message[]> {
+        try {
+            if (!messages || messages.length === 0) {
+                log.info('No messages provided to buildMessagesWithContext');
+                return [];
+            }
+
+            if (!context || context.trim() === '') {
+                log.info('No context provided to buildMessagesWithContext, returning original messages');
+                return messages;
+            }
+
+            // Get the provider name, handling service classes and raw provider names
+            let providerName: string;
+            if (typeof llmService === 'string') {
+                // If llmService is a string, assume it's the provider name
+                providerName = llmService;
+            } else if (llmService.constructor && llmService.constructor.name) {
+                // Extract provider name from service class name (e.g., OllamaService -> ollama)
+                providerName = llmService.constructor.name.replace('Service', '').toLowerCase();
+            } else {
+                // Fallback to default
+                providerName = 'default';
+            }
+
+            log.info(`Using formatter for provider: ${providerName}`);
+
+            // Get the appropriate formatter for this provider
+            const formatter = formatMessages.getFormatter(providerName);
+
+            // Format messages with context using the provider-specific formatter
+            const formattedMessages = formatter.formatMessages(
+                messages,
+                undefined, // No system prompt override - use what's in the messages
+                context
+            );
+
+            log.info(`Formatted ${messages.length} messages into ${formattedMessages.length} messages for ${providerName}`);
+
+            return formattedMessages;
+        } catch (error) {
+            log.error(`Error building messages with context: ${error}`);
+            // Fallback to original messages in case of error
+            return messages;
         }
     }
 }