From b9d4a4e779c2dd1a90f589de3d8da24ed549a053 Mon Sep 17 00:00:00 2001 From: Willie Zutz Date: Sun, 22 Jun 2025 23:59:29 -0600 Subject: [PATCH] feat(agent): Implement structured output using Zod schemas for Analyzer, Task Manager, and Web Search agents --- src/lib/agents/analyzerAgent.ts | 99 +++++++++++++++------------- src/lib/agents/taskManagerAgent.ts | 29 +++++--- src/lib/agents/webSearchAgent.ts | 23 +++++-- src/lib/prompts/analyzer.ts | 15 ++--- src/lib/prompts/taskBreakdown.ts | 56 ++++++++-------- src/lib/prompts/webSearch.ts | 71 +++++++++----------- src/lib/utils/summarizeWebContent.ts | 85 +++++++++++++----------- 7 files changed, 205 insertions(+), 173 deletions(-) diff --git a/src/lib/agents/analyzerAgent.ts b/src/lib/agents/analyzerAgent.ts index b35ea37..1eea51e 100644 --- a/src/lib/agents/analyzerAgent.ts +++ b/src/lib/agents/analyzerAgent.ts @@ -7,6 +7,7 @@ import { import { ChatPromptTemplate } from '@langchain/core/prompts'; import { Command, END } from '@langchain/langgraph'; import { EventEmitter } from 'events'; +import { z } from 'zod'; import LineOutputParser from '../outputParsers/lineOutputParser'; import { formatDateForLLM } from '../utils'; import { AgentState } from './agentState'; @@ -21,6 +22,22 @@ import { removeThinkingBlocksFromMessages, } from '../utils/contentUtils'; +// Define Zod schemas for structured output +const NextActionSchema = z.object({ + action: z.enum(['good_content', 'need_user_info', 'need_more_info']).describe('The next action to take based on content analysis'), + reasoning: z.string().describe('Brief explanation of why this action was chosen') +}); + +const UserInfoRequestSchema = z.object({ + question: z.string().describe('A detailed question to ask the user for additional information'), + reasoning: z.string().describe('Explanation of why this information is needed') +}); + +const SearchRefinementSchema = z.object({ + question: z.string().describe('A refined search question to gather more specific information'), + reasoning: z.string().describe('Explanation of what information is missing and why this search will help') +}); + export class AnalyzerAgent { private llm: BaseChatModel; private emitter: EventEmitter; @@ -48,7 +65,6 @@ export class AnalyzerAgent { state.originalQuery = state.query; } - let nextActionContent = 'need_more_info'; // Skip full analysis if this is the first run. //if (state.fullAnalysisAttempts > 0) { // Emit initial analysis event @@ -91,20 +107,20 @@ export class AnalyzerAgent { state.messages, ); - const nextActionResponse = await this.llm.invoke( + // Use structured output for next action decision + const structuredLlm = this.llm.withStructuredOutput(NextActionSchema, { + name: 'analyze_content', + }); + + const nextActionResponse = await structuredLlm.invoke( [...thinkingBlocksRemovedMessages, new HumanMessage(nextActionPrompt)], { signal: this.signal }, ); - nextActionContent = removeThinkingBlocks( - nextActionResponse.content as string, - ); - - console.log('Next action response:', nextActionContent); + console.log('Next action response:', nextActionResponse); if ( - !nextActionContent.startsWith('good_content') && - !nextActionContent.startsWith('`good_content`') + nextActionResponse.action !== 'good_content' ) { // If we don't have enough information, but we still have available tasks, proceed with the next task @@ -119,9 +135,13 @@ export class AnalyzerAgent { } if ( - nextActionContent.startsWith('need_user_info') || - nextActionContent.startsWith('`need_user_info`') + nextActionResponse.action === 'need_user_info' ) { + // Use structured output for user info request + const userInfoLlm = this.llm.withStructuredOutput(UserInfoRequestSchema, { + name: 'request_user_info', + }); + const moreUserInfoPrompt = await ChatPromptTemplate.fromTemplate( additionalUserInputPrompt, ).format({ @@ -139,39 +159,27 @@ export class AnalyzerAgent { query: state.originalQuery || state.query, // Use original query for user info context }); - const stream = await this.llm.stream( + const userInfoRequest = await userInfoLlm.invoke( [ ...removeThinkingBlocksFromMessages(state.messages), - new SystemMessage(moreUserInfoPrompt), + new HumanMessage(moreUserInfoPrompt), ], { signal: this.signal }, ); - let fullResponse = ''; - for await (const chunk of stream) { - if (this.signal.aborted) { - break; - } - - const content = chunk.content; - if (typeof content === 'string' && content.length > 0) { - fullResponse += content; - - // Emit each chunk as a data response in real-time - this.emitter.emit( - 'data', - JSON.stringify({ - type: 'response', - data: content, - }), - ); - } - } + // Emit the complete question to the user + this.emitter.emit( + 'data', + JSON.stringify({ + type: 'response', + data: userInfoRequest.question, + }), + ); this.emitter.emit('end'); // Create the final response message with the complete content - const response = new SystemMessage(fullResponse); + const response = new SystemMessage(userInfoRequest.question); return new Command({ goto: END, @@ -182,6 +190,11 @@ export class AnalyzerAgent { } // If we need more information from the LLM, generate a more specific search query + // Use structured output for search refinement + const searchRefinementLlm = this.llm.withStructuredOutput(SearchRefinementSchema, { + name: 'refine_search', + }); + const moreInfoPrompt = await ChatPromptTemplate.fromTemplate( additionalWebSearchPrompt, ).format({ @@ -199,7 +212,7 @@ export class AnalyzerAgent { query: state.originalQuery || state.query, // Use original query for more info context }); - const moreInfoResponse = await this.llm.invoke( + const searchRefinement = await searchRefinementLlm.invoke( [ ...removeThinkingBlocksFromMessages(state.messages), new HumanMessage(moreInfoPrompt), @@ -207,10 +220,6 @@ export class AnalyzerAgent { { signal: this.signal }, ); - const moreInfoQuestion = removeThinkingBlocks( - moreInfoResponse.content as string, - ); - // Emit reanalyzing event when we need more information this.emitter.emit('agent_action', { type: 'agent_action', @@ -219,11 +228,11 @@ export class AnalyzerAgent { message: 'Current context is insufficient - analyzing search requirements', details: { - nextSearchQuery: moreInfoQuestion, + nextSearchQuery: searchRefinement.question, documentCount: state.relevantDocuments.length, searchIterations: state.searchInstructionHistory.length, query: state.originalQuery || state.query, // Show original query in details - currentSearchFocus: moreInfoQuestion, + currentSearchFocus: searchRefinement.question, }, }, }); @@ -233,14 +242,14 @@ export class AnalyzerAgent { update: { messages: [ new AIMessage( - `The following question can help refine the search: ${moreInfoQuestion}`, + `The following question can help refine the search: ${searchRefinement.question}`, ), ], - query: moreInfoQuestion, // Use the refined question for TaskManager to analyze - searchInstructions: moreInfoQuestion, + query: searchRefinement.question, // Use the refined question for TaskManager to analyze + searchInstructions: searchRefinement.question, searchInstructionHistory: [ ...(state.searchInstructionHistory || []), - moreInfoQuestion, + searchRefinement.question, ], fullAnalysisAttempts: 1, originalQuery: state.originalQuery || state.query, // Preserve the original user query diff --git a/src/lib/agents/taskManagerAgent.ts b/src/lib/agents/taskManagerAgent.ts index 3d1cb87..08f0bf5 100644 --- a/src/lib/agents/taskManagerAgent.ts +++ b/src/lib/agents/taskManagerAgent.ts @@ -3,10 +3,19 @@ import { AIMessage } from '@langchain/core/messages'; import { PromptTemplate } from '@langchain/core/prompts'; import { Command } from '@langchain/langgraph'; import { EventEmitter } from 'events'; +import { z } from 'zod'; import { taskBreakdownPrompt } from '../prompts/taskBreakdown'; import { AgentState } from './agentState'; import { setTemperature } from '../utils/modelUtils'; +// Define Zod schema for structured task breakdown output +const TaskBreakdownSchema = z.object({ + tasks: z.array(z.string()).describe('Array of specific, focused tasks broken down from the original query'), + reasoning: z.string().describe('Explanation of how and why the query was broken down into these tasks') +}); + +type TaskBreakdown = z.infer; + export class TaskManagerAgent { private llm: BaseChatModel; private emitter: EventEmitter; @@ -115,19 +124,19 @@ export class TaskManagerAgent { query: state.query, }); - const taskBreakdownResult = await this.llm.invoke([prompt], { + // Use structured output for task breakdown + const structuredLlm = this.llm.withStructuredOutput(TaskBreakdownSchema, { + name: 'break_down_tasks', + }); + + const taskBreakdownResult = await structuredLlm.invoke([prompt], { signal: this.signal, }); - // Parse the response to extract tasks - const responseContent = taskBreakdownResult.content as string; + console.log('Task breakdown response:', taskBreakdownResult); - console.log('Task breakdown response:', responseContent); - const taskLines = responseContent - .split('\n') - .filter((line) => line.trim().startsWith('TASK:')) - .map((line) => line.replace('TASK:', '').trim()) - .filter((task) => task.length > 0); + // Extract tasks from structured response + const taskLines = taskBreakdownResult.tasks.filter((task) => task.trim().length > 0); if (taskLines.length === 0) { // Fallback: if no tasks found, use the original query @@ -137,6 +146,7 @@ export class TaskManagerAgent { console.log( `Task breakdown completed: ${taskLines.length} tasks identified`, ); + console.log('Reasoning:', taskBreakdownResult.reasoning); taskLines.forEach((task, index) => { console.log(`Task ${index + 1}: ${task}`); }); @@ -151,6 +161,7 @@ export class TaskManagerAgent { query: state.query, taskCount: taskLines.length, tasks: taskLines, + reasoning: taskBreakdownResult.reasoning, }, }, }); diff --git a/src/lib/agents/webSearchAgent.ts b/src/lib/agents/webSearchAgent.ts index a2c2e41..f2320e3 100644 --- a/src/lib/agents/webSearchAgent.ts +++ b/src/lib/agents/webSearchAgent.ts @@ -4,6 +4,7 @@ import { PromptTemplate } from '@langchain/core/prompts'; import { Command, END } from '@langchain/langgraph'; import { EventEmitter } from 'events'; import { Document } from 'langchain/document'; +import { z } from 'zod'; import LineOutputParser from '../outputParsers/lineOutputParser'; import { webSearchRetrieverAgentPrompt } from '../prompts/webSearch'; import { searchSearxng } from '../searxng'; @@ -19,6 +20,14 @@ import { Embeddings } from '@langchain/core/embeddings'; import { removeThinkingBlocksFromMessages } from '../utils/contentUtils'; import computeSimilarity from '../utils/computeSimilarity'; +// Define Zod schema for structured search query output +const SearchQuerySchema = z.object({ + searchQuery: z.string().describe('The optimized search query to use for web search'), + reasoning: z.string().describe('Explanation of how the search query was optimized for better results') +}); + +type SearchQuery = z.infer; + export class WebSearchAgent { private llm: BaseChatModel; private emitter: EventEmitter; @@ -85,18 +94,20 @@ export class WebSearchAgent { supervisor: state.searchInstructions, }); - const searchQueryResult = await this.llm.invoke( + // Use structured output for search query generation + const structuredLlm = this.llm.withStructuredOutput(SearchQuerySchema, { + name: 'generate_search_query', + }); + + const searchQueryResult = await structuredLlm.invoke( [...removeThinkingBlocksFromMessages(state.messages), prompt], { signal: this.signal }, ); - // Parse the response to extract the search query with the lineoutputparser - const lineOutputParser = new LineOutputParser({ key: 'answer' }); - const searchQuery = await lineOutputParser.parse( - searchQueryResult.content as string, - ); + const searchQuery = searchQueryResult.searchQuery; console.log(`Performing web search for query: "${searchQuery}"`); + console.log('Search query reasoning:', searchQueryResult.reasoning); // Emit executing web search event this.emitter.emit('agent_action', { diff --git a/src/lib/prompts/analyzer.ts b/src/lib/prompts/analyzer.ts index aec9fc0..2084162 100644 --- a/src/lib/prompts/analyzer.ts +++ b/src/lib/prompts/analyzer.ts @@ -63,11 +63,6 @@ Ask yourself: "Could this missing information reasonably be found through a web - If it's factual and searchable → \`need_more_info\` - If the context is complete or the user wants to use the existing context → \`good_content\` -The only output in your response should be one of the following: -- \`good_content\` -- \`need_user_info\` -- \`need_more_info\` - # System Instructions {systemInstructions} @@ -81,7 +76,11 @@ Today's date is {date} {context} -`; + +# Search Instruction History +{searchInstructionHistory} + +Provide your response as a JSON object with "action" and "reasoning" fields where action is one of: good_content, need_user_info, or need_more_info.`; export const additionalUserInputPrompt = `You are an expert content analyzer. Your task is to analyze the provided context and user query to determine what additional information is needed to fully answer the user's query. @@ -105,7 +104,7 @@ Today's date is {date} # Instructions -Respond with a detailed question that will be directed to the user to gather more specific information that can help refine the search. +Respond with a JSON object containing "question" and "reasoning" fields. The question should be detailed and directed to the user to gather more specific information that can help refine the search. The reasoning should explain why this information is needed. `; export const additionalWebSearchPrompt = ` @@ -135,5 +134,5 @@ Today's date is {date} {context} -Respond with a detailed question that will be directed to an LLM to gather more specific information that can help refine the search. +Respond with a JSON object containing "question" and "reasoning" fields. The question should be detailed and directed to an LLM to gather more specific information that can help refine the search. The reasoning should explain what information is missing and why this search will help. `; diff --git a/src/lib/prompts/taskBreakdown.ts b/src/lib/prompts/taskBreakdown.ts index 2d811b4..9561906 100644 --- a/src/lib/prompts/taskBreakdown.ts +++ b/src/lib/prompts/taskBreakdown.ts @@ -30,47 +30,49 @@ export const taskBreakdownPrompt = `You are a task breakdown specialist. Your jo **Input**: "What's the capital of New York, California, and France?" **Analysis**: Multiple distinct geographical subjects **Output**: -TASK: What's the capital of New York? -TASK: What's the capital of California? -TASK: What's the capital of France? +{{ + "tasks": [ + "What's the capital of New York?", + "What's the capital of California?", + "What's the capital of France?" + ], + "reasoning": "The question asks about capitals of three distinct geographical entities that can each be answered independently." +}} **Input**: "How many calories are in my meal of: One chicken breast, one apple, three oreo cookies, two cups of peanut butter" **Analysis**: Multiple food items requiring separate calorie calculations **Output**: -TASK: How many calories are in one chicken breast? -TASK: How many calories are in one apple? -TASK: How many calories are in one oreo cookie? -TASK: How many calories are in one cup of peanut butter? +{{ + "tasks": [ + "How many calories are in one chicken breast?", + "How many calories are in one apple?", + "How many calories are in one oreo cookie?", + "How many calories are in one cup of peanut butter?" + ], + "reasoning": "The question involves calculating calories for multiple distinct food items that can be researched separately and then combined." +}} **Input**: "What is the capital of France?" **Analysis**: Single focused question, no breakdown needed **Output**: -TASK: What is the capital of France? +{{ + "tasks": ["What is the capital of France?"], + "reasoning": "This is already a single, focused question that doesn't require breaking down into smaller parts." +}} **Input**: "Compare the economies of Japan and Germany" **Analysis**: Comparative question requiring detailed data about each economy separately **Output**: -TASK: What is the current state of Japan's economy? -TASK: What is the current state of Germany's economy? - -**Input**: "What are the side effects of aspirin, ibuprofen, and acetaminophen?" -**Analysis**: Multiple distinct medications -**Output**: -TASK: What are the side effects of aspirin? -TASK: What are the side effects of ibuprofen? -TASK: What are the side effects of acetaminophen? - -**Input**: "What day is New Year's Day this year?" -**Analysis**: Single focused question, no breakdown needed -**Output**: -TASK: What day is New Year's Day this year? +{{ + "tasks": [ + "What is the current state of Japan's economy?", + "What is the current state of Germany's economy?" + ], + "reasoning": "To compare two economies, we need detailed information about each country's economic situation separately, which can then be compared." +}} ## Your Task: Analyze this user question: "{query}" -Provide your response in the following format: -- Each sub-question on a new line starting with "TASK:" -- If the question is already focused, provide it as a single task - -Your response:`; +Provide your response as a JSON object with "tasks" (array of task strings) and "reasoning" (explanation of your analysis) fields.`; diff --git a/src/lib/prompts/webSearch.ts b/src/lib/prompts/webSearch.ts index d947d75..a377e59 100644 --- a/src/lib/prompts/webSearch.ts +++ b/src/lib/prompts/webSearch.ts @@ -181,8 +181,6 @@ export const webSearchRetrieverAgentPrompt = ` - Condense the question to its essence and remove any unnecessary details - Search queries should be short and to the point, focusing on the main topic or question - Ensure the question is grammatically correct and free of spelling errors -- If it is a simple writing task or a greeting (unless the greeting contains a question after it) like Hi, Hello, How are you, etc. instead of a question then you need to return \`not_needed\` as the response in the XML block -- If you are a thinking or reasoning AI, do not use and or and tags in your thinking. Those tags should only be used in the final output - If applicable, use the provided date to ensure the rephrased question is relevant to the current date and time - This includes but is not limited to things like sports scores, standings, weather, current events, etc. - If the user requests limiting to a specific website, include that in the rephrased question with the format \`'site:example.com'\`, be sure to include the quotes. Only do this if the limiting is explicitly mentioned in the question @@ -191,9 +189,10 @@ export const webSearchRetrieverAgentPrompt = ` # Data - The user question is contained in the tag after the below -- You must always return the rephrased question inside an XML block, if there are no links in the follow-up question then don't insert a XML block in your response +- You must return your response as a JSON object with "searchQuery" and "reasoning" fields +- The searchQuery should contain the optimized search query +- The reasoning should explain how you optimized the query for better search results - Current date is: {date} -- Do not include any other text in your answer # System Instructions - These instructions are provided by the user in the tag @@ -214,9 +213,10 @@ There are several examples attached for your reference inside the below examples - - Run Windows games on macOS with Apple Silicon - + {{ + "searchQuery": "Run Windows games on macOS with Apple Silicon", + "reasoning": "Simplified the query to focus on the core topic of running Windows games on Apple Silicon Macs, removing the requirement for source count as that's handled by the search system." + }} @@ -229,9 +229,10 @@ There are several examples attached for your reference inside the below examples - - F1 Monaco Grand Prix highlights - + {{ + "searchQuery": "F1 Monaco Grand Prix highlights", + "reasoning": "Added specific context from supervisor instructions to identify this as an F1 Monaco Grand Prix query, making the search more targeted." + }} @@ -242,22 +243,10 @@ There are several examples attached for your reference inside the below examples - - Capital of France - - - - - - - - Hi, how are you? - - - - - not_needed - + {{ + "searchQuery": "Capital of France", + "reasoning": "Simplified the query to essential keywords for better search engine optimization while maintaining the core meaning." + }} @@ -268,9 +257,10 @@ There are several examples attached for your reference inside the below examples - - Weather in Albany, New York {date} 'site:weather.com' - + {{ + "searchQuery": "Weather in Albany, New York {date} 'site:weather.com'", + "reasoning": "Added location context and current date for weather relevance, included site restriction as requested by user." + }} @@ -281,10 +271,10 @@ There are several examples attached for your reference inside the below examples - ## Example 6 output - - {date} F1 constructor standings - + {{ + "searchQuery": "{date} F1 constructor standings", + "reasoning": "Added current date to ensure we get the most recent F1 constructor standings information." + }} @@ -295,10 +285,10 @@ There are several examples attached for your reference inside the below examples - ## Example 7 output - - Top 10 restaurants in New York on {date} 'site:yelp.com' - + {{ + "searchQuery": "Top 10 restaurants in New York on {date} 'site:yelp.com'", + "reasoning": "Focused on the core query about top restaurants, added current date for relevance, and included the site restriction to yelp.com as requested." + }} @@ -312,9 +302,10 @@ There are several examples attached for your reference inside the below examples - - Top 10 restaurants in New York on {date} - + {{ + "searchQuery": "Top 10 restaurants in New York on {date}", + "reasoning": "Following supervisor instructions to focus specifically on New York restaurants, ignoring Chicago and Boston for this search iteration." + }} diff --git a/src/lib/utils/summarizeWebContent.ts b/src/lib/utils/summarizeWebContent.ts index 40746f1..f78287a 100644 --- a/src/lib/utils/summarizeWebContent.ts +++ b/src/lib/utils/summarizeWebContent.ts @@ -1,6 +1,6 @@ import { Document } from '@langchain/core/documents'; import { BaseChatModel } from '@langchain/core/language_models/chat_models'; -import LineOutputParser from '../outputParsers/lineOutputParser'; +import { z } from 'zod'; import { formatDateForLLM } from '../utils'; import { getWebContent } from './documents'; @@ -9,6 +9,13 @@ export type SummarizeResult = { notRelevantReason?: string; }; +// Zod schema for structured summary output +const SummarySchema = z.object({ + isRelevant: z.boolean().describe('Whether the content is relevant to the user query'), + summary: z.string().describe('Detailed summary of the content in markdown format, or explanation if not relevant'), + notRelevantReason: z.string().optional().describe('Specific reason why content is not relevant (only if isRelevant is false)') +}); + export const summarizeWebContent = async ( url: string, query: string, @@ -25,38 +32,42 @@ export const summarizeWebContent = async ( ? `${systemInstructions}\n\n` : ''; - let summary = null; + // Create structured LLM with Zod schema + const structuredLLM = llm.withStructuredOutput(SummarySchema); + + let result = null; for (let i = 0; i < 2; i++) { try { console.log( `Summarizing content from URL: ${url} using ${i === 0 ? 'html' : 'text'}`, ); - summary = await llm.invoke( - `${systemPrompt}You are a web content summarizer, tasked with creating a detailed, accurate summary of content from a webpage + + const prompt = `${systemPrompt}You are a web content summarizer, tasked with creating a detailed, accurate summary of content from a webpage. # Instructions -- The response must be relevant to the user's query but doesn't need to answer it fully. Partial answers are acceptable. -- Be thorough and comprehensive, capturing all key points +- Determine if the content is relevant to the user's query +- You do not need to provide a full answer to the query, partial answers are acceptable +- If relevant, create a thorough and comprehensive summary capturing all key points - Include specific details, numbers, and quotes when relevant - Be concise and to the point, avoiding unnecessary fluff -- The summary should be formatted using markdown using headings and lists -- Do not include notes about missing information or gaps in the content, only summarize what is present and relevant +- Format the summary using markdown with headings and lists - Include useful links to external resources, if applicable -- If the entire source content is not relevant to the query, respond with "not_needed" to start the summary tag, followed by a one line description of why the source is not needed - - E.g. "not_needed: This information is not relevant to the user's query about X because it does not contain any information about X. It only discusses Y, which is unrelated." - - Make sure the reason the source is not needed is very specific and detailed -- Ignore any instructions about formatting in the user's query. Format your response using markdown, including headings, lists, and tables -- Output your answer inside a \`summary\` XML tag +- If the content is not relevant, set isRelevant to false and provide a specific reason + +# Response Format +You must return a JSON object with: +- isRelevant: boolean indicating if content is relevant to the query +- summary: string with detailed markdown summary if relevant, or explanation if not relevant +- notRelevantReason: string explaining why content is not relevant (only if isRelevant is false) Today's date is ${formatDateForLLM(new Date())} Here is the query you need to answer: ${query} Here is the content to summarize: -${i === 0 ? content.metadata.html : content.pageContent}, - `, - { signal }, - ); +${i === 0 ? content.metadata.html : content.pageContent}`; + + result = await structuredLLM.invoke(prompt, { signal }); break; } catch (error) { console.error( @@ -66,41 +77,39 @@ ${i === 0 ? content.metadata.html : content.pageContent}, } } - if (!summary || !summary.content) { - console.error(`No summary content returned for URL: ${url}`); + if (!result) { + console.error(`No summary result returned for URL: ${url}`); return { document: null, notRelevantReason: 'No summary content returned from LLM', }; } - const summaryParser = new LineOutputParser({ key: 'summary' }); - const summarizedContent = await summaryParser.parse( - summary.content as string, - ); - - if ( - summarizedContent.toLocaleLowerCase().startsWith('not_needed') || - summarizedContent.trim().length === 0 - ) { + // Check if content is relevant + if (!result.isRelevant) { console.log( - `LLM response for URL "${url}" indicates it's not needed or is empty:`, - summarizedContent, + `LLM response for URL "${url}" indicates it's not relevant:`, + result.notRelevantReason || result.summary, ); - // Extract the reason from the "not_needed" response - const reason = summarizedContent.startsWith('not_needed') - ? summarizedContent.substring('not_needed:'.length).trim() - : summarizedContent.trim().length === 0 - ? 'Source content was empty or could not be processed' - : 'Source content was not relevant to the query'; + return { + document: null, + notRelevantReason: result.notRelevantReason || result.summary + }; + } - return { document: null, notRelevantReason: reason }; + // Content is relevant, create document with summary + if (!result.summary || result.summary.trim().length === 0) { + console.error(`No summary content in relevant response for URL: ${url}`); + return { + document: null, + notRelevantReason: 'Summary content was empty', + }; } return { document: new Document({ - pageContent: summarizedContent, + pageContent: result.summary, metadata: { ...content.metadata, url: url,