diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 92187ff..7348f58 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -33,7 +33,6 @@ The system works through these main steps: - `lib/prompts`: Prompt templates for LLMs - `lib/chains`: LangChain chains for various operations - `lib/agents`: LangGraph agents for advanced processing - - `lib/tools`: LangGraph tools for use by agents - `lib/utils`: Utility functions and types including web content retrieval and processing ## Focus Modes @@ -77,7 +76,6 @@ When working on this codebase, you might need to: - Create new prompt templates in `/src/lib/prompts` - Build new chains in `/src/lib/chains` - Implement new LangGraph agents in `/src/lib/agents` -- Create new tools for LangGraph agents in `/src/lib/tools` ## AI Behavior diff --git a/.gitignore b/.gitignore index c95173d..f984aa5 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,6 @@ Thumbs.db # Db db.sqlite /searxng + +# AI stuff for planning and implementation +.ai/ \ No newline at end of file diff --git a/src/app/api/chat/route.ts b/src/app/api/chat/route.ts index b8650f6..a54e56c 100644 --- a/src/app/api/chat/route.ts +++ b/src/app/api/chat/route.ts @@ -110,6 +110,18 @@ const handleEmitterEvents = async ( sources = parsedData.data; } }); + + stream.on('agent_action', (data) => { + writer.write( + encoder.encode( + JSON.stringify({ + type: 'agent_action', + data: data.data, + messageId: userMessageId, + }) + '\n', + ), + ); + }); let modelStats: ModelStats = { modelName: '', }; diff --git a/src/components/AgentActionDisplay.tsx b/src/components/AgentActionDisplay.tsx new file mode 100644 index 0000000..510fc54 --- /dev/null +++ b/src/components/AgentActionDisplay.tsx @@ -0,0 +1,122 @@ +'use client'; + +import { useState } from 'react'; +import { cn } from '@/lib/utils'; +import { ChevronDown, ChevronUp, Bot } from 'lucide-react'; +import { AgentActionEvent } from './ChatWindow'; + +interface AgentActionDisplayProps { + events: AgentActionEvent[]; + messageId: string; +} + +const AgentActionDisplay = ({ events, messageId }: AgentActionDisplayProps) => { + const [isExpanded, setIsExpanded] = useState(false); + + // Get the most recent event for collapsed view + const latestEvent = events[events.length - 1]; + + // Common function to format action names + const formatActionName = (action: string) => { + return action.replace(/_/g, ' ').toLocaleLowerCase(); + }; + + if (!latestEvent) { + return null; + } + + return ( +
+ + + {isExpanded && ( +
+
+ {events.map((event, index) => ( +
+
+ + + {formatActionName(event.action)} + +
+ + {event.message && event.message.length > 0 && ( +

{event.message}

+ )} + + {/* Display relevant details based on event type */} + {event.details && Object.keys(event.details).length > 0 && ( +
+ {event.details.sourceUrl && ( + + )} + {event.details.skipReason && ( +
+ Reason: + {event.details.skipReason} +
+ )} + {event.details.searchQuery && event.details.searchQuery !== event.details.query && ( +
+ Search Query: + "{event.details.searchQuery}" +
+ )} + {event.details.sourcesFound !== undefined && ( +
+ Sources Found: + {event.details.sourcesFound} +
+ )} + {/* {(event.details.documentCount !== undefined && event.details.documentCount > 0) && ( +
+ Documents: + {event.details.documentCount} +
+ )} */} + {event.details.contentLength !== undefined && ( +
+ Content Length: + {event.details.contentLength} chars +
+ )} + {event.details.searchInstructions !== undefined && ( +
+ Search Instructions: + {event.details.searchInstructions} +
+ )} +
+ )} +
+ ))} +
+
+ )} +
+ ); +}; + +export default AgentActionDisplay; diff --git a/src/components/Chat.tsx b/src/components/Chat.tsx index 9970453..f7c5045 100644 --- a/src/components/Chat.tsx +++ b/src/components/Chat.tsx @@ -5,6 +5,7 @@ import { File, Message } from './ChatWindow'; import MessageBox from './MessageBox'; import MessageBoxLoading from './MessageBoxLoading'; import MessageInput from './MessageInput'; +import AgentActionDisplay from './AgentActionDisplay'; const Chat = ({ loading, @@ -224,6 +225,25 @@ const Chat = ({ sendMessage={sendMessage} handleEditMessage={handleEditMessage} /> + {/* Show agent actions after user messages - either completed or in progress */} + {msg.role === 'user' && ( + <> + {/* Show agent actions if they exist */} + {msg.agentActions && msg.agentActions.length > 0 && ( + + )} + {/* Show empty agent action display if this is the last user message and we're loading */} + {loading && isLast && (!msg.agentActions || msg.agentActions.length === 0) && ( + + )} + + )} {!isLast && msg.role === 'assistant' && (
)} diff --git a/src/components/ChatWindow.tsx b/src/components/ChatWindow.tsx index 26e5a32..1fdccf8 100644 --- a/src/components/ChatWindow.tsx +++ b/src/components/ChatWindow.tsx @@ -18,6 +18,13 @@ export type ModelStats = { responseTime?: number; }; +export type AgentActionEvent = { + action: string; + message: string; + details: Record; + timestamp: Date; +}; + export type Message = { messageId: string; chatId: string; @@ -29,6 +36,7 @@ export type Message = { modelStats?: ModelStats; searchQuery?: string; searchUrl?: string; + agentActions?: AgentActionEvent[]; progress?: { message: string; current: number; @@ -423,6 +431,30 @@ const ChatWindow = ({ id }: { id?: string }) => { return; } + if (data.type === 'agent_action') { + const agentActionEvent: AgentActionEvent = { + action: data.data.action, + message: data.data.message, + details: data.data.details || {}, + timestamp: new Date(), + }; + + // Update the user message with agent actions + setMessages((prev) => + prev.map((message) => { + if (message.messageId === data.messageId && message.role === 'user') { + const updatedActions = [ + ...(message.agentActions || []), + agentActionEvent, + ]; + return { ...message, agentActions: updatedActions }; + } + return message; + }), + ); + return; + } + if (data.type === 'sources') { sources = data.data; if (!added) { diff --git a/src/components/MessageInputActions/Focus.tsx b/src/components/MessageInputActions/Focus.tsx index a11e339..d2a47e9 100644 --- a/src/components/MessageInputActions/Focus.tsx +++ b/src/components/MessageInputActions/Focus.tsx @@ -24,12 +24,12 @@ const focusModes = [ description: 'Searches across all of the internet', icon: , }, - { - key: 'academicSearch', - title: 'Academic', - description: 'Search in published academic papers', - icon: , - }, + // { + // key: 'academicSearch', + // title: 'Academic', + // description: 'Search in published academic papers', + // icon: , + // }, { key: 'chat', title: 'Chat', @@ -42,24 +42,24 @@ const focusModes = [ description: 'Research and interact with local files with citations', icon: , }, - { - key: 'redditSearch', - title: 'Reddit', - description: 'Search for discussions and opinions', - icon: , - }, - { - key: 'wolframAlphaSearch', - title: 'Wolfram Alpha', - description: 'Computational knowledge engine', - icon: , - }, - { - key: 'youtubeSearch', - title: 'Youtube', - description: 'Search and watch videos', - icon: , - }, + // { + // key: 'redditSearch', + // title: 'Reddit', + // description: 'Search for discussions and opinions', + // icon: , + // }, + // { + // key: 'wolframAlphaSearch', + // title: 'Wolfram Alpha', + // description: 'Computational knowledge engine', + // icon: , + // }, + // { + // key: 'youtubeSearch', + // title: 'Youtube', + // description: 'Search and watch videos', + // icon: , + // }, ]; const Focus = ({ diff --git a/src/lib/search/agentSearch.ts b/src/lib/search/agentSearch.ts index f5128bd..8292ff5 100644 --- a/src/lib/search/agentSearch.ts +++ b/src/lib/search/agentSearch.ts @@ -22,7 +22,7 @@ import { webSearchRetrieverAgentPrompt } from '../prompts/webSearch'; import { searchSearxng } from '../searxng'; import { formatDateForLLM } from '../utils'; import { getModelName } from '../utils/modelUtils'; -import { summarizeWebContent } from '../utils/summarizeWebContent'; +import { summarizeWebContent, SummarizeResult } from '../utils/summarizeWebContent'; /** * State interface for the agent supervisor workflow @@ -97,6 +97,21 @@ export class AgentSearch { private async webSearchAgent( state: typeof AgentState.State, ): Promise { + // Emit preparing web search event + this.emitter.emit('agent_action', { + type: 'agent_action', + data: { + action: 'PREPARING_SEARCH_QUERY', + // message: `Preparing search query`, + details: { + query: state.query, + searchInstructions: state.searchInstructions || state.query, + documentCount: state.relevantDocuments.length, + searchIterations: state.searchInstructionHistory.length + } + } + }); + const template = PromptTemplate.fromTemplate(webSearchRetrieverAgentPrompt); const prompt = await template.format({ systemInstructions: this.systemInstructions, @@ -118,11 +133,43 @@ export class AgentSearch { try { console.log(`Performing web search for query: "${searchQuery}"`); + + // Emit executing web search event + this.emitter.emit('agent_action', { + type: 'agent_action', + data: { + action: 'EXECUTING_WEB_SEARCH', + // message: `Searching the web for: '${searchQuery}'`, + details: { + query: state.query, + searchQuery: searchQuery, + documentCount: state.relevantDocuments.length, + searchIterations: state.searchInstructionHistory.length + } + } + }); + const searchResults = await searchSearxng(searchQuery, { language: 'en', engines: [], }); + // Emit web sources identified event + this.emitter.emit('agent_action', { + type: 'agent_action', + data: { + action: 'WEB_SOURCES_IDENTIFIED', + message: `Found ${searchResults.results.length} potential web sources`, + details: { + query: state.query, + searchQuery: searchQuery, + sourcesFound: searchResults.results.length, + documentCount: state.relevantDocuments.length, + searchIterations: state.searchInstructionHistory.length + } + } + }); + let bannedUrls = state.bannedUrls || []; let attemptedUrlCount = 0; // Summarize the top 2 search results @@ -130,6 +177,8 @@ export class AgentSearch { for (const result of searchResults.results) { if (bannedUrls.includes(result.url)) { console.log(`Skipping banned URL: ${result.url}`); + // Note: We don't emit an agent_action event for banned URLs as this is an internal + // optimization that should be transparent to the user continue; // Skip banned URLs } if (attemptedUrlCount >= 5) { @@ -146,20 +195,72 @@ export class AgentSearch { break; // Limit to top 1 document } - const summary = await summarizeWebContent( + // Emit analyzing source event + this.emitter.emit('agent_action', { + type: 'agent_action', + data: { + action: 'ANALYZING_SOURCE', + message: `Analyzing content from: ${result.title || result.url}`, + details: { + query: state.query, + sourceUrl: result.url, + sourceTitle: result.title || 'Untitled', + documentCount: state.relevantDocuments.length, + searchIterations: state.searchInstructionHistory.length + } + } + }); + + const summaryResult = await summarizeWebContent( result.url, state.query, this.llm, this.systemInstructions, this.signal, ); - if (summary) { - documents.push(summary); + + if (summaryResult.document) { + documents.push(summaryResult.document); + + // Emit context updated event + this.emitter.emit('agent_action', { + type: 'agent_action', + data: { + action: 'CONTEXT_UPDATED', + message: `Added information from ${summaryResult.document.metadata.title || result.url} to context`, + details: { + query: state.query, + sourceUrl: result.url, + sourceTitle: summaryResult.document.metadata.title || 'Untitled', + contentLength: summaryResult.document.pageContent.length, + documentCount: state.relevantDocuments.length + documents.length, + searchIterations: state.searchInstructionHistory.length + } + } + }); + console.log( - `Summarized content from ${result.url} to ${summary.pageContent.length} characters. Content: ${summary.pageContent}`, + `Summarized content from ${result.url} to ${summaryResult.document.pageContent.length} characters. Content: ${summaryResult.document.pageContent}`, ); } else { console.warn(`No relevant content found for URL: ${result.url}`); + + // Emit skipping irrelevant source event for non-relevant content + this.emitter.emit('agent_action', { + type: 'agent_action', + data: { + action: 'SKIPPING_IRRELEVANT_SOURCE', + message: `Source ${result.title || result.url} was not relevant - trying next`, + details: { + query: state.query, + sourceUrl: result.url, + sourceTitle: result.title || 'Untitled', + skipReason: summaryResult.notRelevantReason || 'Content was not relevant to the query', + documentCount: state.relevantDocuments.length + documents.length, + searchIterations: state.searchInstructionHistory.length + } + } + }); } } @@ -200,6 +301,20 @@ export class AgentSearch { private async analyzer(state: typeof AgentState.State): Promise { try { + // Emit initial analysis event + this.emitter.emit('agent_action', { + type: 'agent_action', + data: { + action: 'ANALYZING_CONTEXT', + message: 'Analyzing the context to see if we have enough information to answer the query', + details: { + documentCount: state.relevantDocuments.length, + query: state.query, + searchIterations: state.searchInstructionHistory.length + } + } + }); + console.log( `Analyzing ${state.relevantDocuments.length} documents for relevance...`, ); @@ -282,6 +397,22 @@ Today's date is ${formatDateForLLM(new Date())} console.log('Reason for insufficiency:', reason); if (analysisResult.startsWith('need_more_info')) { + // Emit reanalyzing event when we need more information + this.emitter.emit('agent_action', { + type: 'agent_action', + data: { + action: 'MORE_DATA_NEEDED', + message: 'Current context is insufficient - gathering more information', + details: { + reason: reason, + nextSearchQuery: moreInfoQuestion, + documentCount: state.relevantDocuments.length, + searchIterations: state.searchInstructionHistory.length, + query: state.query + } + } + }); + return new Command({ goto: 'web_search', update: { @@ -296,6 +427,20 @@ Today's date is ${formatDateForLLM(new Date())} }); } + // Emit information gathering complete event when we have sufficient information + this.emitter.emit('agent_action', { + type: 'agent_action', + data: { + action: 'INFORMATION_GATHERING_COMPLETE', + message: 'Sufficient information gathered - ready to synthesize response', + details: { + documentCount: state.relevantDocuments.length, + searchIterations: state.searchInstructionHistory.length, + query: state.query + } + } + }); + return new Command({ goto: 'synthesizer', update: { @@ -328,6 +473,20 @@ Today's date is ${formatDateForLLM(new Date())} state: typeof AgentState.State, ): Promise { try { + // Emit synthesizing response event + this.emitter.emit('agent_action', { + type: 'agent_action', + data: { + action: 'SYNTHESIZING_RESPONSE', + message: 'Synthesizing final answer...', + details: { + query: state.query, + documentCount: state.relevantDocuments.length, + searchIterations: state.searchInstructionHistory.length + } + } + }); + const synthesisPrompt = `You are an expert information synthesizer. Based on the search results and analysis provided, create a comprehensive, well-structured answer to the user's query. ## Response Instructions diff --git a/src/lib/utils/summarizeWebContent.ts b/src/lib/utils/summarizeWebContent.ts index 1a99b7a..c423f4d 100644 --- a/src/lib/utils/summarizeWebContent.ts +++ b/src/lib/utils/summarizeWebContent.ts @@ -4,18 +4,23 @@ import LineOutputParser from '../outputParsers/lineOutputParser'; import { formatDateForLLM } from '../utils'; import { getWebContent } from './documents'; +export type SummarizeResult = { + document: Document | null; + notRelevantReason?: string; +}; + export const summarizeWebContent = async ( url: string, query: string, llm: BaseChatModel, systemInstructions: string, signal: AbortSignal, -): Promise => { +): Promise => { try { // Helper function to summarize content and check relevance const summarizeContent = async ( content: Document, - ): Promise => { + ): Promise => { const systemPrompt = systemInstructions ? `${systemInstructions}\n\n` : ''; @@ -49,7 +54,7 @@ Here is the query you need to answer: ${query} Here is the content to summarize: ${i === 0 ? content.metadata.html : content.pageContent}, - `, + `, { signal }, ); break; @@ -63,7 +68,7 @@ ${i === 0 ? content.metadata.html : content.pageContent}, if (!summary || !summary.content) { console.error(`No summary content returned for URL: ${url}`); - return null; + return { document: null, notRelevantReason: 'No summary content returned from LLM' }; } const summaryParser = new LineOutputParser({ key: 'summary' }); @@ -79,16 +84,27 @@ ${i === 0 ? content.metadata.html : content.pageContent}, `LLM response for URL "${url}" indicates it's not needed or is empty:`, summarizedContent, ); - return null; + + // Extract the reason from the "not_needed" response + const reason = summarizedContent.startsWith('not_needed') + ? summarizedContent.substring('not_needed:'.length).trim() + : summarizedContent.trim().length === 0 + ? 'Source content was empty or could not be processed' + : 'Source content was not relevant to the query'; + + return { document: null, notRelevantReason: reason }; } - return new Document({ - pageContent: summarizedContent, - metadata: { - ...content.metadata, - url: url, - }, - }); + return { + document: new Document({ + pageContent: summarizedContent, + metadata: { + ...content.metadata, + url: url, + }, + }), + notRelevantReason: undefined + }; }; // // First try the lite approach @@ -121,9 +137,10 @@ ${i === 0 ? content.metadata.html : content.pageContent}, return await summarizeContent(webContent); } else { console.log(`No valid content found for URL: ${url}`); + return { document: null, notRelevantReason: 'No valid content found at the URL' }; } } catch (error) { console.error(`Error processing URL ${url}:`, error); + return { document: null, notRelevantReason: `Error processing URL: ${error instanceof Error ? error.message : 'Unknown error'}` }; } - return null; };