feat(agent): Display agent actions on the UI

.github/copilot-instructions.md

@@ -33,7 +33,6 @@ The system works through these main steps:
   - `lib/prompts`: Prompt templates for LLMs
   - `lib/chains`: LangChain chains for various operations
   - `lib/agents`: LangGraph agents for advanced processing
-  - `lib/tools`: LangGraph tools for use by agents
   - `lib/utils`: Utility functions and types including web content retrieval and processing
 
 ## Focus Modes
@@ -77,7 +76,6 @@ When working on this codebase, you might need to:
 - Create new prompt templates in `/src/lib/prompts`
 - Build new chains in `/src/lib/chains`
 - Implement new LangGraph agents in `/src/lib/agents`
-- Create new tools for LangGraph agents in `/src/lib/tools`
 
 ## AI Behavior
 

.gitignore

@@ -37,3 +37,6 @@ Thumbs.db
 # Db
 db.sqlite
 /searxng
+
+# AI stuff for planning and implementation
+.ai/

src/app/api/chat/route.ts

@@ -110,6 +110,18 @@ const handleEmitterEvents = async (
       sources = parsedData.data;
     }
   });
+
+  stream.on('agent_action', (data) => {
+    writer.write(
+      encoder.encode(
+        JSON.stringify({
+          type: 'agent_action',
+          data: data.data,
+          messageId: userMessageId,
+        }) + '\n',
+      ),
+    );
+  });
   let modelStats: ModelStats = {
     modelName: '',
   };

src/components/AgentActionDisplay.tsx

@@ -0,0 +1,122 @@
+'use client';
+
+import { useState } from 'react';
+import { cn } from '@/lib/utils';
+import { ChevronDown, ChevronUp, Bot } from 'lucide-react';
+import { AgentActionEvent } from './ChatWindow';
+
+interface AgentActionDisplayProps {
+  events: AgentActionEvent[];
+  messageId: string;
+}
+
+const AgentActionDisplay = ({ events, messageId }: AgentActionDisplayProps) => {
+  const [isExpanded, setIsExpanded] = useState(false);
+
+  // Get the most recent event for collapsed view
+  const latestEvent = events[events.length - 1];
+
+  // Common function to format action names
+  const formatActionName = (action: string) => {
+    return action.replace(/_/g, ' ').toLocaleLowerCase();
+  };
+
+  if (!latestEvent) {
+    return null;
+  }
+
+  return (
+    <div className="my-4 bg-light-secondary/50 dark:bg-dark-secondary/50 rounded-xl border border-light-200 dark:border-dark-200 overflow-hidden">
+      <button
+        onClick={() => setIsExpanded(!isExpanded)}
+        className="w-full flex items-center justify-between px-4 py-3 text-black/90 dark:text-white/90 hover:bg-light-200 dark:hover:bg-dark-200 transition duration-200"
+      >
+        <div className="flex items-center space-x-2">
+          <Bot size={20} className="text-[#9C27B0]" />
+          <span className="font-medium text-base text-black/70 dark:text-white/70 tracking-wide capitalize">
+            {latestEvent.action === 'SYNTHESIZING_RESPONSE' ? 'Agent Log' : formatActionName(latestEvent.action)}
+          </span>
+        </div>
+        {isExpanded ? (
+          <ChevronUp size={18} className="text-black/70 dark:text-white/70" />
+        ) : (
+          <ChevronDown size={18} className="text-black/70 dark:text-white/70" />
+        )}
+      </button>
+
+      {isExpanded && (
+        <div className="px-4 py-3 text-black/80 dark:text-white/80 text-base border-t border-light-200 dark:border-dark-200 bg-light-100/50 dark:bg-dark-100/50">
+          <div className="space-y-3">
+            {events.map((event, index) => (
+              <div
+                key={`${messageId}-${index}-${event.action}`}
+                className="flex flex-col space-y-1 p-3 bg-white/50 dark:bg-black/20 rounded-lg border border-light-200/50 dark:border-dark-200/50"
+              >
+                <div className="flex items-center space-x-2">
+                  <Bot size={16} className="text-[#9C27B0]" />
+                  <span className="font-medium text-sm text-black/70 dark:text-white/70 capitalize tracking-wide">
+                    {formatActionName(event.action)}
+                  </span>
+                </div>
+
+                {event.message && event.message.length > 0 && (
+                  <p className="text-base">{event.message}</p>
+                )}
+
+                {/* Display relevant details based on event type */}
+                {event.details && Object.keys(event.details).length > 0 && (
+                  <div className="mt-2 text-sm text-black/60 dark:text-white/60">
+                    {event.details.sourceUrl && (
+                      <div className="flex space-x-1">
+                        <span className="font-bold">Source:</span>
+                        <span className="truncate"><a href={event.details.sourceUrl} target='_blank'>{event.details.sourceUrl}</a></span>
+                      </div>
+                    )}
+                    {event.details.skipReason && (
+                      <div className="flex space-x-1">
+                        <span className="font-bold">Reason:</span>
+                        <span>{event.details.skipReason}</span>
+                      </div>
+                    )}
+                    {event.details.searchQuery && event.details.searchQuery !== event.details.query && (
+                      <div className="flex space-x-1">
+                        <span className="font-bold">Search Query:</span>
+                        <span className="italic">"{event.details.searchQuery}"</span>
+                      </div>
+                    )}
+                    {event.details.sourcesFound !== undefined && (
+                      <div className="flex space-x-1">
+                        <span className="font-bold">Sources Found:</span>
+                        <span>{event.details.sourcesFound}</span>
+                      </div>
+                    )}
+                    {/* {(event.details.documentCount !== undefined && event.details.documentCount > 0) && (
+                      <div className="flex space-x-1">
+                        <span className="font-bold">Documents:</span>
+                        <span>{event.details.documentCount}</span>
+                      </div>
+                    )} */}
+                    {event.details.contentLength !== undefined && (
+                      <div className="flex space-x-1">
+                        <span className="font-bold">Content Length:</span>
+                        <span>{event.details.contentLength} chars</span>
+                      </div>
+                    )}
+                    {event.details.searchInstructions !== undefined && (
+                      <div className="flex space-x-1">
+                        <span className="font-bold">Search Instructions:</span>
+                        <span>{event.details.searchInstructions}</span>
+                      </div>
+                    )}
+                  </div>
+                )}
+              </div>
+            ))}
+          </div>
+        </div>
+      )}
+    </div>
+  );
+};
+
+export default AgentActionDisplay;

src/components/Chat.tsx

@@ -5,6 +5,7 @@ import { File, Message } from './ChatWindow';
 import MessageBox from './MessageBox';
 import MessageBoxLoading from './MessageBoxLoading';
 import MessageInput from './MessageInput';
+import AgentActionDisplay from './AgentActionDisplay';
 
 const Chat = ({
   loading,
@@ -224,6 +225,25 @@ const Chat = ({
               sendMessage={sendMessage}
               handleEditMessage={handleEditMessage}
             />
+            {/* Show agent actions after user messages - either completed or in progress */}
+            {msg.role === 'user' && (
+              <>
+                {/* Show agent actions if they exist */}
+                {msg.agentActions && msg.agentActions.length > 0 && (
+                  <AgentActionDisplay
+                    messageId={msg.messageId}
+                    events={msg.agentActions}
+                  />
+                )}
+                {/* Show empty agent action display if this is the last user message and we're loading */}
+                {loading && isLast && (!msg.agentActions || msg.agentActions.length === 0) && (
+                  <AgentActionDisplay
+                    messageId={msg.messageId}
+                    events={[]}
+                  />
+                )}
+              </>
+            )}
             {!isLast && msg.role === 'assistant' && (
               <div className="h-px w-full bg-light-secondary dark:bg-dark-secondary" />
             )}

src/components/ChatWindow.tsx

@@ -18,6 +18,13 @@ export type ModelStats = {
   responseTime?: number;
 };
 
+export type AgentActionEvent = {
+  action: string;
+  message: string;
+  details: Record<string, any>;
+  timestamp: Date;
+};
+
 export type Message = {
   messageId: string;
   chatId: string;
@@ -29,6 +36,7 @@ export type Message = {
   modelStats?: ModelStats;
   searchQuery?: string;
   searchUrl?: string;
+  agentActions?: AgentActionEvent[];
   progress?: {
     message: string;
     current: number;
@@ -423,6 +431,30 @@ const ChatWindow = ({ id }: { id?: string }) => {
         return;
       }
 
+      if (data.type === 'agent_action') {
+        const agentActionEvent: AgentActionEvent = {
+          action: data.data.action,
+          message: data.data.message,
+          details: data.data.details || {},
+          timestamp: new Date(),
+        };
+
+        // Update the user message with agent actions
+        setMessages((prev) =>
+          prev.map((message) => {
+            if (message.messageId === data.messageId && message.role === 'user') {
+              const updatedActions = [
+                ...(message.agentActions || []),
+                agentActionEvent,
+              ];
+              return { ...message, agentActions: updatedActions };
+            }
+            return message;
+          }),
+        );
+        return;
+      }
+
       if (data.type === 'sources') {
         sources = data.data;
         if (!added) {

src/components/MessageInputActions/Focus.tsx

@@ -24,12 +24,12 @@ const focusModes = [
     description: 'Searches across all of the internet',
     icon: <Globe size={20} />,
   },
-  {
-    key: 'academicSearch',
-    title: 'Academic',
-    description: 'Search in published academic papers',
-    icon: <SwatchBook size={20} />,
-  },
+  // {
+  //   key: 'academicSearch',
+  //   title: 'Academic',
+  //   description: 'Search in published academic papers',
+  //   icon: <SwatchBook size={20} />,
+  // },
   {
     key: 'chat',
     title: 'Chat',
@@ -42,24 +42,24 @@ const focusModes = [
     description: 'Research and interact with local files with citations',
     icon: <Pencil size={16} />,
   },
-  {
-    key: 'redditSearch',
-    title: 'Reddit',
-    description: 'Search for discussions and opinions',
-    icon: <SiReddit className="h-5 w-auto mr-0.5" />,
-  },
-  {
-    key: 'wolframAlphaSearch',
-    title: 'Wolfram Alpha',
-    description: 'Computational knowledge engine',
-    icon: <BadgePercent size={20} />,
-  },
-  {
-    key: 'youtubeSearch',
-    title: 'Youtube',
-    description: 'Search and watch videos',
-    icon: <SiYoutube className="h-5 w-auto mr-0.5" />,
-  },
+  // {
+  //   key: 'redditSearch',
+  //   title: 'Reddit',
+  //   description: 'Search for discussions and opinions',
+  //   icon: <SiReddit className="h-5 w-auto mr-0.5" />,
+  // },
+  // {
+  //   key: 'wolframAlphaSearch',
+  //   title: 'Wolfram Alpha',
+  //   description: 'Computational knowledge engine',
+  //   icon: <BadgePercent size={20} />,
+  // },
+  // {
+  //   key: 'youtubeSearch',
+  //   title: 'Youtube',
+  //   description: 'Search and watch videos',
+  //   icon: <SiYoutube className="h-5 w-auto mr-0.5" />,
+  // },
 ];
 
 const Focus = ({

src/lib/search/agentSearch.ts

@@ -22,7 +22,7 @@ import { webSearchRetrieverAgentPrompt } from '../prompts/webSearch';
 import { searchSearxng } from '../searxng';
 import { formatDateForLLM } from '../utils';
 import { getModelName } from '../utils/modelUtils';
-import { summarizeWebContent } from '../utils/summarizeWebContent';
+import { summarizeWebContent, SummarizeResult } from '../utils/summarizeWebContent';
 
 /**
  * State interface for the agent supervisor workflow
@@ -97,6 +97,21 @@ export class AgentSearch {
   private async webSearchAgent(
     state: typeof AgentState.State,
   ): Promise<Command> {
+    // Emit preparing web search event
+    this.emitter.emit('agent_action', {
+      type: 'agent_action',
+      data: {
+        action: 'PREPARING_SEARCH_QUERY',
+        // message: `Preparing search query`,
+        details: {
+          query: state.query,
+          searchInstructions: state.searchInstructions || state.query,
+          documentCount: state.relevantDocuments.length,
+          searchIterations: state.searchInstructionHistory.length
+        }
+      }
+    });
+
     const template = PromptTemplate.fromTemplate(webSearchRetrieverAgentPrompt);
     const prompt = await template.format({
       systemInstructions: this.systemInstructions,
@@ -118,11 +133,43 @@ export class AgentSearch {
 
     try {
       console.log(`Performing web search for query: "${searchQuery}"`);
+      
+      // Emit executing web search event
+      this.emitter.emit('agent_action', {
+        type: 'agent_action',
+        data: {
+          action: 'EXECUTING_WEB_SEARCH',
+          // message: `Searching the web for: '${searchQuery}'`,
+          details: {
+            query: state.query,
+            searchQuery: searchQuery,
+            documentCount: state.relevantDocuments.length,
+            searchIterations: state.searchInstructionHistory.length
+          }
+        }
+      });
+
       const searchResults = await searchSearxng(searchQuery, {
         language: 'en',
         engines: [],
       });
 
+      // Emit web sources identified event
+      this.emitter.emit('agent_action', {
+        type: 'agent_action',
+        data: {
+          action: 'WEB_SOURCES_IDENTIFIED',
+          message: `Found ${searchResults.results.length} potential web sources`,
+          details: {
+            query: state.query,
+            searchQuery: searchQuery,
+            sourcesFound: searchResults.results.length,
+            documentCount: state.relevantDocuments.length,
+            searchIterations: state.searchInstructionHistory.length
+          }
+        }
+      });
+
       let bannedUrls = state.bannedUrls || [];
       let attemptedUrlCount = 0;
       // Summarize the top 2 search results
@@ -130,6 +177,8 @@ export class AgentSearch {
       for (const result of searchResults.results) {
         if (bannedUrls.includes(result.url)) {
           console.log(`Skipping banned URL: ${result.url}`);
+          // Note: We don't emit an agent_action event for banned URLs as this is an internal
+          // optimization that should be transparent to the user
           continue; // Skip banned URLs
         }
         if (attemptedUrlCount >= 5) {
@@ -146,20 +195,72 @@ export class AgentSearch {
           break; // Limit to top 1 document
         }
 
-        const summary = await summarizeWebContent(
+        // Emit analyzing source event
+        this.emitter.emit('agent_action', {
+          type: 'agent_action',
+          data: {
+            action: 'ANALYZING_SOURCE',
+            message: `Analyzing content from: ${result.title || result.url}`,
+            details: {
+              query: state.query,
+              sourceUrl: result.url,
+              sourceTitle: result.title || 'Untitled',
+              documentCount: state.relevantDocuments.length,
+              searchIterations: state.searchInstructionHistory.length
+            }
+          }
+        });
+
+        const summaryResult = await summarizeWebContent(
           result.url,
           state.query,
           this.llm,
           this.systemInstructions,
           this.signal,
         );
-        if (summary) {
-          documents.push(summary);
+        
+        if (summaryResult.document) {
+          documents.push(summaryResult.document);
+          
+          // Emit context updated event
+          this.emitter.emit('agent_action', {
+            type: 'agent_action',
+            data: {
+              action: 'CONTEXT_UPDATED',
+              message: `Added information from ${summaryResult.document.metadata.title || result.url} to context`,
+              details: {
+                query: state.query,
+                sourceUrl: result.url,
+                sourceTitle: summaryResult.document.metadata.title || 'Untitled',
+                contentLength: summaryResult.document.pageContent.length,
+                documentCount: state.relevantDocuments.length + documents.length,
+                searchIterations: state.searchInstructionHistory.length
+              }
+            }
+          });
+          
           console.log(
-            `Summarized content from ${result.url} to ${summary.pageContent.length} characters. Content: ${summary.pageContent}`,
+            `Summarized content from ${result.url} to ${summaryResult.document.pageContent.length} characters. Content: ${summaryResult.document.pageContent}`,
           );
         } else {
           console.warn(`No relevant content found for URL: ${result.url}`);
+          
+          // Emit skipping irrelevant source event for non-relevant content
+          this.emitter.emit('agent_action', {
+            type: 'agent_action',
+            data: {
+              action: 'SKIPPING_IRRELEVANT_SOURCE',
+              message: `Source ${result.title || result.url} was not relevant - trying next`,
+              details: {
+                query: state.query,
+                sourceUrl: result.url,
+                sourceTitle: result.title || 'Untitled',
+                skipReason: summaryResult.notRelevantReason || 'Content was not relevant to the query',
+                documentCount: state.relevantDocuments.length + documents.length,
+                searchIterations: state.searchInstructionHistory.length
+              }
+            }
+          });
         }
       }
 
@@ -200,6 +301,20 @@ export class AgentSearch {
 
   private async analyzer(state: typeof AgentState.State): Promise<Command> {
     try {
+      // Emit initial analysis event
+      this.emitter.emit('agent_action', {
+        type: 'agent_action',
+        data: {
+          action: 'ANALYZING_CONTEXT',
+          message: 'Analyzing the context to see if we have enough information to answer the query',
+          details: {
+            documentCount: state.relevantDocuments.length,
+            query: state.query,
+            searchIterations: state.searchInstructionHistory.length
+          }
+        }
+      });
+
       console.log(
         `Analyzing ${state.relevantDocuments.length} documents for relevance...`,
       );
@@ -282,6 +397,22 @@ Today's date is ${formatDateForLLM(new Date())}
       console.log('Reason for insufficiency:', reason);
 
       if (analysisResult.startsWith('need_more_info')) {
+        // Emit reanalyzing event when we need more information
+        this.emitter.emit('agent_action', {
+          type: 'agent_action',
+          data: {
+            action: 'MORE_DATA_NEEDED',
+            message: 'Current context is insufficient - gathering more information',
+            details: {
+              reason: reason,
+              nextSearchQuery: moreInfoQuestion,
+              documentCount: state.relevantDocuments.length,
+              searchIterations: state.searchInstructionHistory.length,
+              query: state.query
+            }
+          }
+        });
+
         return new Command({
           goto: 'web_search',
           update: {
@@ -296,6 +427,20 @@ Today's date is ${formatDateForLLM(new Date())}
         });
       }
 
+      // Emit information gathering complete event when we have sufficient information
+      this.emitter.emit('agent_action', {
+        type: 'agent_action',
+        data: {
+          action: 'INFORMATION_GATHERING_COMPLETE',
+          message: 'Sufficient information gathered - ready to synthesize response',
+          details: {
+            documentCount: state.relevantDocuments.length,
+            searchIterations: state.searchInstructionHistory.length,
+            query: state.query
+          }
+        }
+      });
+
       return new Command({
         goto: 'synthesizer',
         update: {
@@ -328,6 +473,20 @@ Today's date is ${formatDateForLLM(new Date())}
     state: typeof AgentState.State,
   ): Promise<Command> {
     try {
+      // Emit synthesizing response event
+      this.emitter.emit('agent_action', {
+        type: 'agent_action',
+        data: {
+          action: 'SYNTHESIZING_RESPONSE',
+          message: 'Synthesizing final answer...',
+          details: {
+            query: state.query,
+            documentCount: state.relevantDocuments.length,
+            searchIterations: state.searchInstructionHistory.length
+          }
+        }
+      });
+
       const synthesisPrompt = `You are an expert information synthesizer. Based on the search results and analysis provided, create a comprehensive, well-structured answer to the user's query.
 
 ## Response Instructions

src/lib/utils/summarizeWebContent.ts

@@ -4,18 +4,23 @@ import LineOutputParser from '../outputParsers/lineOutputParser';
 import { formatDateForLLM } from '../utils';
 import { getWebContent } from './documents';
 
+export type SummarizeResult = {
+  document: Document | null;
+  notRelevantReason?: string;
+};
+
 export const summarizeWebContent = async (
   url: string,
   query: string,
   llm: BaseChatModel,
   systemInstructions: string,
   signal: AbortSignal,
-): Promise<Document | null> => {
+): Promise<SummarizeResult> => {
   try {
     // Helper function to summarize content and check relevance
     const summarizeContent = async (
       content: Document,
-    ): Promise<Document | null> => {
+    ): Promise<SummarizeResult> => {
       const systemPrompt = systemInstructions
         ? `${systemInstructions}\n\n`
         : '';
@@ -49,7 +54,7 @@ Here is the query you need to answer: ${query}
 
 Here is the content to summarize:
 ${i === 0 ? content.metadata.html : content.pageContent},
-        `,
+            `,
             { signal },
           );
           break;
@@ -63,7 +68,7 @@ ${i === 0 ? content.metadata.html : content.pageContent},
 
       if (!summary || !summary.content) {
         console.error(`No summary content returned for URL: ${url}`);
-        return null;
+        return { document: null, notRelevantReason: 'No summary content returned from LLM' };
       }
 
       const summaryParser = new LineOutputParser({ key: 'summary' });
@@ -79,16 +84,27 @@ ${i === 0 ? content.metadata.html : content.pageContent},
           `LLM response for URL "${url}" indicates it's not needed or is empty:`,
           summarizedContent,
         );
-        return null;
+        
+        // Extract the reason from the "not_needed" response
+        const reason = summarizedContent.startsWith('not_needed') 
+          ? summarizedContent.substring('not_needed:'.length).trim()
+          : summarizedContent.trim().length === 0 
+            ? 'Source content was empty or could not be processed'
+            : 'Source content was not relevant to the query';
+            
+        return { document: null, notRelevantReason: reason };
       }
 
-      return new Document({
-        pageContent: summarizedContent,
-        metadata: {
-          ...content.metadata,
-          url: url,
-        },
-      });
+      return { 
+        document: new Document({
+          pageContent: summarizedContent,
+          metadata: {
+            ...content.metadata,
+            url: url,
+          },
+        }),
+        notRelevantReason: undefined
+      };
     };
 
     // // First try the lite approach
@@ -121,9 +137,10 @@ ${i === 0 ? content.metadata.html : content.pageContent},
       return await summarizeContent(webContent);
     } else {
       console.log(`No valid content found for URL: ${url}`);
+      return { document: null, notRelevantReason: 'No valid content found at the URL' };
     }
   } catch (error) {
     console.error(`Error processing URL ${url}:`, error);
+    return { document: null, notRelevantReason: `Error processing URL: ${error instanceof Error ? error.message : 'Unknown error'}` };
   }
-  return null;
 };