sk/Perplexica
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(agent): Improve code formatting and readability across multiple components

Browse source
This commit is contained in:
Willie Zutz 2025-06-17 00:20:05 -06:00
parent 74c3934aa5
commit 72c2ddc3a0
11 changed files with 302 additions and 223 deletions
Download patch file Download diff file Expand all files Collapse all files

38
src/components/AgentActionDisplay.tsx
View file

@ -2,7 +2,14 @@
import { useState } from 'react';
import { cn } from '@/lib/utils';
import { ChevronDown, ChevronUp, Bot, Search, Zap, Microscope } from 'lucide-react';
import {
ChevronDown,
ChevronUp,
Bot,
Search,
Zap,
Microscope,
} from 'lucide-react';
import { AgentActionEvent } from './ChatWindow';
interface AgentActionDisplayProps {
@ -48,7 +55,9 @@ const AgentActionDisplay = ({ events, messageId }: AgentActionDisplayProps) => {
<div className="flex items-center space-x-2">
{getActionIcon(latestEvent.action)}
<span className="font-medium text-base text-black/70 dark:text-white/70 tracking-wide capitalize">
{latestEvent.action === 'SYNTHESIZING_RESPONSE' ? 'Agent Log' : formatActionName(latestEvent.action)}
{latestEvent.action === 'SYNTHESIZING_RESPONSE'
? 'Agent Log'
: formatActionName(latestEvent.action)}
</span>
</div>
{isExpanded ? (
@ -83,7 +92,11 @@ const AgentActionDisplay = ({ events, messageId }: AgentActionDisplayProps) => {
{event.details.sourceUrl && (
<div className="flex space-x-1">
<span className="font-bold">Source:</span>
<span className="truncate"><a href={event.details.sourceUrl} target='_blank'>{event.details.sourceUrl}</a></span>
<span className="truncate">
<a href={event.details.sourceUrl} target="_blank">
{event.details.sourceUrl}
</a>
</span>
</div>
)}
{event.details.skipReason && (
@ -92,12 +105,15 @@ const AgentActionDisplay = ({ events, messageId }: AgentActionDisplayProps) => {
<span>{event.details.skipReason}</span>
</div>
)}
{event.details.searchQuery && event.details.searchQuery !== event.details.query && (
<div className="flex space-x-1">
<span className="font-bold">Search Query:</span>
<span className="italic">&quot;{event.details.searchQuery}&quot;</span>
</div>
)}
{event.details.searchQuery &&
event.details.searchQuery !== event.details.query && (
<div className="flex space-x-1">
<span className="font-bold">Search Query:</span>
<span className="italic">
&quot;{event.details.searchQuery}&quot;
</span>
</div>
)}
{event.details.sourcesFound !== undefined && (
<div className="flex space-x-1">
<span className="font-bold">Sources Found:</span>
@ -131,7 +147,9 @@ const AgentActionDisplay = ({ events, messageId }: AgentActionDisplayProps) => {
{event.details.processingType && (
<div className="flex space-x-1">
<span className="font-bold">Processing Type:</span>
<span className="capitalize">{event.details.processingType.replace('-', ' ')}</span>
<span className="capitalize">
{event.details.processingType.replace('-', ' ')}
</span>
</div>
)}
{event.details.insufficiencyReason && (

11
src/components/Chat.tsx
View file

@ -236,12 +236,11 @@ const Chat = ({
/>
)}
{/* Show empty agent action display if this is the last user message and we're loading */}
{loading && isLast && (!msg.agentActions || msg.agentActions.length === 0) && (
<AgentActionDisplay
messageId={msg.messageId}
events={[]}
/>
)}
{loading &&
isLast &&
(!msg.agentActions || msg.agentActions.length === 0) && (
<AgentActionDisplay messageId={msg.messageId} events={[]} />
)}
</>
)}
{!isLast && msg.role === 'assistant' && (

5
src/components/ChatWindow.tsx
View file

@ -442,7 +442,10 @@ const ChatWindow = ({ id }: { id?: string }) => {
// Update the user message with agent actions
setMessages((prev) =>
prev.map((message) => {
if (message.messageId === data.messageId && message.role === 'user') {
if (
message.messageId === data.messageId &&
message.role === 'user'
) {
const updatedActions = [
...(message.agentActions || []),
agentActionEvent,

10
src/components/MessageSources.tsx
View file

@ -40,12 +40,18 @@ const MessageSources = ({ sources }: { sources: Document[] }) => {
{/* Processing type indicator */}
{source.metadata.processingType === 'preview-only' && (
<span title="Partial content analyzed" className="inline-flex">
<Zap size={14} className="text-black/40 dark:text-white/40 ml-1" />
<Zap
size={14}
className="text-black/40 dark:text-white/40 ml-1"
/>
</span>
)}
{source.metadata.processingType === 'full-content' && (
<span title="Full content analyzed" className="inline-flex">
<Microscope size={14} className="text-black/40 dark:text-white/40 ml-1" />
<Microscope
size={14}
className="text-black/40 dark:text-white/40 ml-1"
/>
</span>
)}
</div>

36
src/lib/agents/analyzerAgent.ts
View file

@ -32,13 +32,14 @@ export class AnalyzerAgent {
type: 'agent_action',
data: {
action: 'ANALYZING_CONTEXT',
message: 'Analyzing the context to see if we have enough information to answer the query',
message:
'Analyzing the context to see if we have enough information to answer the query',
details: {
documentCount: state.relevantDocuments.length,
query: state.query,
searchIterations: state.searchInstructionHistory.length
}
}
searchIterations: state.searchInstructionHistory.length,
},
},
});
console.log(
@ -94,7 +95,10 @@ Today's date is ${formatDateForLLM(new Date())}
).format({
systemInstructions: this.systemInstructions,
context: state.relevantDocuments
.map((doc, index) => `<source${index + 1}>${doc?.metadata?.title ? `<title>${doc?.metadata?.title}</title>` : ''}<content>${doc.pageContent}</content></source${index + 1}>`)
.map(
(doc, index) =>
`<source${index + 1}>${doc?.metadata?.title ? `<title>${doc?.metadata?.title}</title>` : ''}<content>${doc.pageContent}</content></source${index + 1}>`,
)
.join('\n\n'),
});
@ -115,9 +119,7 @@ Today's date is ${formatDateForLLM(new Date())}
const moreInfoQuestion = await moreInfoOutputParser.parse(
response.content as string,
);
const reason = await reasonOutputParser.parse(
response.content as string,
);
const reason = await reasonOutputParser.parse(response.content as string);
console.log('Analysis result:', analysisResult);
console.log('More info question:', moreInfoQuestion);
@ -129,15 +131,16 @@ Today's date is ${formatDateForLLM(new Date())}
type: 'agent_action',
data: {
action: 'MORE_DATA_NEEDED',
message: 'Current context is insufficient - gathering more information',
message:
'Current context is insufficient - gathering more information',
details: {
reason: reason,
nextSearchQuery: moreInfoQuestion,
documentCount: state.relevantDocuments.length,
searchIterations: state.searchInstructionHistory.length,
query: state.query
}
}
query: state.query,
},
},
});
return new Command({
@ -159,13 +162,14 @@ Today's date is ${formatDateForLLM(new Date())}
type: 'agent_action',
data: {
action: 'INFORMATION_GATHERING_COMPLETE',
message: 'Sufficient information gathered - ready to synthesize response',
message:
'Sufficient information gathered - ready to synthesize response',
details: {
documentCount: state.relevantDocuments.length,
searchIterations: state.searchInstructionHistory.length,
query: state.query
}
}
query: state.query,
},
},
});
return new Command({

6
src/lib/agents/synthesizerAgent.ts
View file

@ -37,9 +37,9 @@ export class SynthesizerAgent {
details: {
query: state.query,
documentCount: state.relevantDocuments.length,
searchIterations: state.searchInstructionHistory.length
}
}
searchIterations: state.searchInstructionHistory.length,
},
},
});
const synthesisPrompt = `You are an expert information synthesizer. Based on the search results and analysis provided, create a comprehensive, well-structured answer to the user's query.

270
src/lib/agents/webSearchAgent.ts
View file

@ -9,7 +9,10 @@ import { webSearchRetrieverAgentPrompt } from '../prompts/webSearch';
import { searchSearxng } from '../searxng';
import { formatDateForLLM } from '../utils';
import { summarizeWebContent } from '../utils/summarizeWebContent';
import { analyzePreviewContent, PreviewContent } from '../utils/analyzePreviewContent';
import {
analyzePreviewContent,
PreviewContent,
} from '../utils/analyzePreviewContent';
import { AgentState } from './agentState';
export class WebSearchAgent {
@ -44,9 +47,9 @@ export class WebSearchAgent {
query: state.query,
searchInstructions: state.searchInstructions || state.query,
documentCount: state.relevantDocuments.length,
searchIterations: state.searchInstructionHistory.length
}
}
searchIterations: state.searchInstructionHistory.length,
},
},
});
const template = PromptTemplate.fromTemplate(webSearchRetrieverAgentPrompt);
@ -70,7 +73,7 @@ export class WebSearchAgent {
try {
console.log(`Performing web search for query: "${searchQuery}"`);
// Emit executing web search event
this.emitter.emit('agent_action', {
type: 'agent_action',
@ -81,9 +84,9 @@ export class WebSearchAgent {
query: state.query,
searchQuery: searchQuery,
documentCount: state.relevantDocuments.length,
searchIterations: state.searchInstructionHistory.length
}
}
searchIterations: state.searchInstructionHistory.length,
},
},
});
const searchResults = await searchSearxng(searchQuery, {
@ -102,30 +105,34 @@ export class WebSearchAgent {
searchQuery: searchQuery,
sourcesFound: searchResults.results.length,
documentCount: state.relevantDocuments.length,
searchIterations: state.searchInstructionHistory.length
}
}
searchIterations: state.searchInstructionHistory.length,
},
},
});
let bannedUrls = state.bannedUrls || [];
// Extract preview content from top 8 search results for analysis
const previewContents: PreviewContent[] = searchResults.results
.filter(result => !bannedUrls.includes(result.url)) // Filter out banned URLs first
.filter((result) => !bannedUrls.includes(result.url)) // Filter out banned URLs first
.slice(0, 8) // Then take top 8 results
.map(result => ({
.map((result) => ({
title: result.title || 'Untitled',
snippet: result.content || '',
url: result.url
url: result.url,
}));
console.log(`Extracted preview content from ${previewContents.length} search results for analysis`);
console.log(
`Extracted preview content from ${previewContents.length} search results for analysis`,
);
// Perform preview analysis to determine if full content retrieval is needed
let previewAnalysisResult = null;
if (previewContents.length > 0) {
console.log('Starting preview content analysis to determine if full processing is needed');
console.log(
'Starting preview content analysis to determine if full processing is needed',
);
// Emit preview analysis event
this.emitter.emit('agent_action', {
type: 'agent_action',
@ -136,9 +143,9 @@ export class WebSearchAgent {
query: state.query,
previewCount: previewContents.length,
documentCount: state.relevantDocuments.length,
searchIterations: state.searchInstructionHistory.length
}
}
searchIterations: state.searchInstructionHistory.length,
},
},
});
previewAnalysisResult = await analyzePreviewContent(
@ -147,10 +154,12 @@ export class WebSearchAgent {
state.messages,
this.llm,
this.systemInstructions,
this.signal
this.signal,
);
console.log(`Preview analysis result: ${previewAnalysisResult.isSufficient ? 'SUFFICIENT' : 'INSUFFICIENT'}${previewAnalysisResult.reason ? ` - ${previewAnalysisResult.reason}` : ''}`);
console.log(
`Preview analysis result: ${previewAnalysisResult.isSufficient ? 'SUFFICIENT' : 'INSUFFICIENT'}${previewAnalysisResult.reason ? ` - ${previewAnalysisResult.reason}` : ''}`,
);
}
let documents: Document[] = [];
@ -159,8 +168,10 @@ export class WebSearchAgent {
// Conditional workflow based on preview analysis result
if (previewAnalysisResult && previewAnalysisResult.isSufficient) {
// Preview content is sufficient - create documents from preview content
console.log('Preview content determined sufficient - skipping full content retrieval');
console.log(
'Preview content determined sufficient - skipping full content retrieval',
);
// Emit preview processing event
this.emitter.emit('agent_action', {
type: 'agent_action',
@ -172,30 +183,38 @@ export class WebSearchAgent {
previewCount: previewContents.length,
documentCount: state.relevantDocuments.length,
searchIterations: state.searchInstructionHistory.length,
processingType: 'preview-only'
}
}
processingType: 'preview-only',
},
},
});
// Create documents from preview content
documents = previewContents.map((content, index) => new Document({
pageContent: `# ${content.title}\n\n${content.snippet}`,
metadata: {
title: content.title,
url: content.url,
source: content.url,
processingType: 'preview-only',
snippet: content.snippet
}
}));
documents = previewContents.map(
(content, index) =>
new Document({
pageContent: `# ${content.title}\n\n${content.snippet}`,
metadata: {
title: content.title,
url: content.url,
source: content.url,
processingType: 'preview-only',
snippet: content.snippet,
},
}),
);
console.log(`Created ${documents.length} documents from preview content`);
console.log(
`Created ${documents.length} documents from preview content`,
);
} else {
// Preview content is insufficient - proceed with full content processing
const insufficiencyReason = previewAnalysisResult?.reason || 'Preview content not available or insufficient';
console.log(`Preview content insufficient: ${insufficiencyReason} - proceeding with full content retrieval`);
const insufficiencyReason =
previewAnalysisResult?.reason ||
'Preview content not available or insufficient';
console.log(
`Preview content insufficient: ${insufficiencyReason} - proceeding with full content retrieval`,
);
// Emit full processing event
this.emitter.emit('agent_action', {
type: 'agent_action',
@ -207,101 +226,106 @@ export class WebSearchAgent {
insufficiencyReason: insufficiencyReason,
documentCount: state.relevantDocuments.length,
searchIterations: state.searchInstructionHistory.length,
processingType: 'full-content'
}
}
processingType: 'full-content',
},
},
});
// Summarize the top 2 search results
for (const result of searchResults.results) {
if (bannedUrls.includes(result.url)) {
console.log(`Skipping banned URL: ${result.url}`);
// Note: We don't emit an agent_action event for banned URLs as this is an internal
// optimization that should be transparent to the user
continue; // Skip banned URLs
}
if (attemptedUrlCount >= 5) {
console.warn(
'Too many attempts to summarize URLs, stopping further attempts.',
);
break; // Limit the number of attempts to summarize URLs
}
attemptedUrlCount++;
bannedUrls.push(result.url); // Add to banned URLs to avoid duplicates
if (documents.length >= 1) {
break; // Limit to top 1 document
}
// Emit analyzing source event
this.emitter.emit('agent_action', {
type: 'agent_action',
data: {
action: 'ANALYZING_SOURCE',
message: `Analyzing content from: ${result.title || result.url}`,
details: {
query: state.query,
sourceUrl: result.url,
sourceTitle: result.title || 'Untitled',
documentCount: state.relevantDocuments.length,
searchIterations: state.searchInstructionHistory.length
}
if (bannedUrls.includes(result.url)) {
console.log(`Skipping banned URL: ${result.url}`);
// Note: We don't emit an agent_action event for banned URLs as this is an internal
// optimization that should be transparent to the user
continue; // Skip banned URLs
}
});
if (attemptedUrlCount >= 5) {
console.warn(
'Too many attempts to summarize URLs, stopping further attempts.',
);
break; // Limit the number of attempts to summarize URLs
}
attemptedUrlCount++;
const summaryResult = await summarizeWebContent(
result.url,
state.query,
this.llm,
this.systemInstructions,
this.signal,
);
if (summaryResult.document) {
documents.push(summaryResult.document);
// Emit context updated event
bannedUrls.push(result.url); // Add to banned URLs to avoid duplicates
if (documents.length >= 1) {
break; // Limit to top 1 document
}
// Emit analyzing source event
this.emitter.emit('agent_action', {
type: 'agent_action',
data: {
action: 'CONTEXT_UPDATED',
message: `Added information from ${summaryResult.document.metadata.title || result.url} to context`,
details: {
query: state.query,
sourceUrl: result.url,
sourceTitle: summaryResult.document.metadata.title || 'Untitled',
contentLength: summaryResult.document.pageContent.length,
documentCount: state.relevantDocuments.length + documents.length,
searchIterations: state.searchInstructionHistory.length
}
}
});
console.log(
`Summarized content from ${result.url} to ${summaryResult.document.pageContent.length} characters. Content: ${summaryResult.document.pageContent}`,
);
} else {
console.warn(`No relevant content found for URL: ${result.url}`);
// Emit skipping irrelevant source event for non-relevant content
this.emitter.emit('agent_action', {
type: 'agent_action',
data: {
action: 'SKIPPING_IRRELEVANT_SOURCE',
message: `Source ${result.title || result.url} was not relevant - trying next`,
action: 'ANALYZING_SOURCE',
message: `Analyzing content from: ${result.title || result.url}`,
details: {
query: state.query,
sourceUrl: result.url,
sourceTitle: result.title || 'Untitled',
skipReason: summaryResult.notRelevantReason || 'Content was not relevant to the query',
documentCount: state.relevantDocuments.length + documents.length,
searchIterations: state.searchInstructionHistory.length
}
}
documentCount: state.relevantDocuments.length,
searchIterations: state.searchInstructionHistory.length,
},
},
});
const summaryResult = await summarizeWebContent(
result.url,
state.query,
this.llm,
this.systemInstructions,
this.signal,
);
if (summaryResult.document) {
documents.push(summaryResult.document);
// Emit context updated event
this.emitter.emit('agent_action', {
type: 'agent_action',
data: {
action: 'CONTEXT_UPDATED',
message: `Added information from ${summaryResult.document.metadata.title || result.url} to context`,
details: {
query: state.query,
sourceUrl: result.url,
sourceTitle:
summaryResult.document.metadata.title || 'Untitled',
contentLength: summaryResult.document.pageContent.length,
documentCount:
state.relevantDocuments.length + documents.length,
searchIterations: state.searchInstructionHistory.length,
},
},
});
console.log(
`Summarized content from ${result.url} to ${summaryResult.document.pageContent.length} characters. Content: ${summaryResult.document.pageContent}`,
);
} else {
console.warn(`No relevant content found for URL: ${result.url}`);
// Emit skipping irrelevant source event for non-relevant content
this.emitter.emit('agent_action', {
type: 'agent_action',
data: {
action: 'SKIPPING_IRRELEVANT_SOURCE',
message: `Source ${result.title || result.url} was not relevant - trying next`,
details: {
query: state.query,
sourceUrl: result.url,
sourceTitle: result.title || 'Untitled',
skipReason:
summaryResult.notRelevantReason ||
'Content was not relevant to the query',
documentCount:
state.relevantDocuments.length + documents.length,
searchIterations: state.searchInstructionHistory.length,
},
},
});
}
}
}
} // Close the else block for full content processing
if (documents.length === 0) {

47
src/lib/search/agentSearch.ts
View file

@ -5,18 +5,13 @@ import {
HumanMessage,
SystemMessage,
} from '@langchain/core/messages';
import {
END,
MemorySaver,
START,
StateGraph,
} from '@langchain/langgraph';
import { END, MemorySaver, START, StateGraph } from '@langchain/langgraph';
import { EventEmitter } from 'events';
import {
import {
AgentState,
WebSearchAgent,
AnalyzerAgent,
SynthesizerAgent
SynthesizerAgent,
} from '../agents';
/**
@ -49,19 +44,19 @@ export class AgentSearch {
llm,
emitter,
systemInstructions,
signal
signal,
);
this.analyzerAgent = new AnalyzerAgent(
llm,
emitter,
systemInstructions,
signal
signal,
);
this.synthesizerAgent = new SynthesizerAgent(
llm,
emitter,
personaInstructions,
signal
signal,
);
}
@ -70,15 +65,27 @@ export class AgentSearch {
*/
private createWorkflow() {
const workflow = new StateGraph(AgentState)
.addNode('web_search', this.webSearchAgent.execute.bind(this.webSearchAgent), {
ends: ['analyzer'],
})
.addNode('analyzer', this.analyzerAgent.execute.bind(this.analyzerAgent), {
ends: ['web_search', 'synthesizer'],
})
.addNode('synthesizer', this.synthesizerAgent.execute.bind(this.synthesizerAgent), {
ends: [END],
})
.addNode(
'web_search',
this.webSearchAgent.execute.bind(this.webSearchAgent),
{
ends: ['analyzer'],
},
)
.addNode(
'analyzer',
this.analyzerAgent.execute.bind(this.analyzerAgent),
{
ends: ['web_search', 'synthesizer'],
},
)
.addNode(
'synthesizer',
this.synthesizerAgent.execute.bind(this.synthesizerAgent),
{
ends: [END],
},
)
.addEdge(START, 'analyzer');
return workflow.compile({ checkpointer: this.checkpointer });

8
src/lib/search/metaSearchAgent.ts
View file

@ -180,8 +180,8 @@ class MetaSearchAgent implements MetaSearchAgentType {
? `${systemInstructions}\n\n`
: '';
const res =
await llm.invoke(`${systemPrompt}You are a web search summarizer, tasked with summarizing a piece of text retrieved from a web search. Your job is to summarize the
const res = await llm.invoke(
`${systemPrompt}You are a web search summarizer, tasked with summarizing a piece of text retrieved from a web search. Your job is to summarize the
text into a detailed, 2-4 paragraph explanation that captures the main ideas and provides a comprehensive answer to the query.
If the query is \"summarize\", you should provide a detailed summary of the text. If the query is a specific question, you should answer it in the summary.
@ -239,7 +239,9 @@ class MetaSearchAgent implements MetaSearchAgentType {
</text>
Make sure to answer the query in the summary.
`, { signal });
`,
{ signal },
);
const document = new Document({
pageContent: res.content as string,

67
src/lib/utils/analyzePreviewContent.ts
View file

@ -24,37 +24,37 @@ export const analyzePreviewContent = async (
): Promise<PreviewAnalysisResult> => {
try {
console.log(`Analyzing preview content for query: "${query}"`);
console.log(`Preview content being analyzed:`, previewContents.map(content => ({
title: content.title,
snippet: content.snippet.substring(0, 100) + '...',
url: content.url
})));
console.log(
`Preview content being analyzed:`,
previewContents.map((content) => ({
title: content.title,
snippet: content.snippet.substring(0, 100) + '...',
url: content.url,
})),
);
// Format preview content for analysis
const formattedPreviewContent = previewContents
.map((content, index) =>
`Source ${index + 1}:
.map(
(content, index) =>
`Source ${index + 1}:
Title: ${content.title}
Snippet: ${content.snippet}
URL: ${content.url}
---`
---`,
)
.join('\n\n');
// Format chat history for context
const formattedChatHistory = chatHistory
.slice(-10) // Only include last 10 messages for context
.map((message, index) =>
`${message._getType()}: ${message.content}`
)
.map((message, index) => `${message._getType()}: ${message.content}`)
.join('\n');
const systemPrompt = systemInstructions
? `${systemInstructions}\n\n`
: '';
const systemPrompt = systemInstructions ? `${systemInstructions}\n\n` : '';
console.log(`Invoking LLM for preview content analysis`);
const analysisResponse = await llm.invoke(
`${systemPrompt}You are a preview content analyzer, tasked with determining if search result snippets contain sufficient information to answer a user's query.
@ -85,9 +85,10 @@ ${formattedPreviewContent}
if (!analysisResponse || !analysisResponse.content) {
console.error('No analysis response returned from LLM');
return {
isSufficient: false,
reason: 'No analysis response returned from LLM - falling back to full content processing'
return {
isSufficient: false,
reason:
'No analysis response returned from LLM - falling back to full content processing',
};
}
@ -99,30 +100,36 @@ ${formattedPreviewContent}
console.log(`LLM decision response:`, decision);
if (decision.toLowerCase().trim() === 'sufficient') {
console.log('Preview content determined to be sufficient for answering the query');
console.log(
'Preview content determined to be sufficient for answering the query',
);
return { isSufficient: true };
} else if (decision.toLowerCase().startsWith('not_needed')) {
// Extract the reason from the "not_needed" response
const reason = decision.startsWith('not_needed')
const reason = decision.startsWith('not_needed')
? decision.substring('not_needed:'.length).trim()
: 'Preview content insufficient for complete answer';
console.log(`Preview content determined to be insufficient. Reason: ${reason}`);
console.log(
`Preview content determined to be insufficient. Reason: ${reason}`,
);
return { isSufficient: false, reason };
} else {
// Default to not sufficient if unclear response
console.log(`Unclear LLM response, defaulting to insufficient: ${decision}`);
return {
isSufficient: false,
reason: 'Unclear analysis response - falling back to full content processing'
console.log(
`Unclear LLM response, defaulting to insufficient: ${decision}`,
);
return {
isSufficient: false,
reason:
'Unclear analysis response - falling back to full content processing',
};
}
} catch (error) {
console.error('Error analyzing preview content:', error);
return {
isSufficient: false,
reason: `Error during preview analysis: ${error instanceof Error ? error.message : 'Unknown error'} - falling back to full content processing`
return {
isSufficient: false,
reason: `Error during preview analysis: ${error instanceof Error ? error.message : 'Unknown error'} - falling back to full content processing`,
};
}
};

27
src/lib/utils/summarizeWebContent.ts
View file

@ -68,7 +68,10 @@ ${i === 0 ? content.metadata.html : content.pageContent},
if (!summary || !summary.content) {
console.error(`No summary content returned for URL: ${url}`);
return { document: null, notRelevantReason: 'No summary content returned from LLM' };
return {
document: null,
notRelevantReason: 'No summary content returned from LLM',
};
}
const summaryParser = new LineOutputParser({ key: 'summary' });
@ -84,18 +87,18 @@ ${i === 0 ? content.metadata.html : content.pageContent},
`LLM response for URL "${url}" indicates it's not needed or is empty:`,
summarizedContent,
);
// Extract the reason from the "not_needed" response
const reason = summarizedContent.startsWith('not_needed')
const reason = summarizedContent.startsWith('not_needed')
? summarizedContent.substring('not_needed:'.length).trim()
: summarizedContent.trim().length === 0
: summarizedContent.trim().length === 0
? 'Source content was empty or could not be processed'
: 'Source content was not relevant to the query';
return { document: null, notRelevantReason: reason };
}
return {
return {
document: new Document({
pageContent: summarizedContent,
metadata: {
@ -104,7 +107,7 @@ ${i === 0 ? content.metadata.html : content.pageContent},
processingType: 'full-content',
},
}),
notRelevantReason: undefined
notRelevantReason: undefined,
};
};
@ -138,10 +141,16 @@ ${i === 0 ? content.metadata.html : content.pageContent},
return await summarizeContent(webContent);
} else {
console.log(`No valid content found for URL: ${url}`);
return { document: null, notRelevantReason: 'No valid content found at the URL' };
return {
document: null,
notRelevantReason: 'No valid content found at the URL',
};
}
} catch (error) {
console.error(`Error processing URL ${url}:`, error);
return { document: null, notRelevantReason: `Error processing URL: ${error instanceof Error ? error.message : 'Unknown error'}` };
return {
document: null,
notRelevantReason: `Error processing URL: ${error instanceof Error ? error.message : 'Unknown error'}`,
};
}
};