feat(app): Introduce quality mode. Improve functionality of balanced mode using readability to get page content and pull relevant excerpts

feat(UI): Show progress during inferrence
feat(security): Don't show API keys in the UI any more
feat(models): Support Claude 4 Anthropic models

README.md

@@ -1,6 +1,6 @@
 # 🚀 Perplexica - An AI-powered search engine 🔎 <!-- omit in toc -->
 
-*This is a fork of [ItzCrazyKns/Perplexica](https://github.com/ItzCrazyKns/Perplexica) with additional features and improvements.*
+_This is a fork of [ItzCrazyKns/Perplexica](https://github.com/ItzCrazyKns/Perplexica) with additional features and improvements._
 
 ![preview](.assets/perplexica-screenshot.png?)
 
@@ -200,6 +200,7 @@ This ensures that OpenSearch descriptions, browser integrations, and all URLs wo
 This fork adds several enhancements to the original Perplexica project:
 
 ### UI Improvements
+
 - ✅ Tabbed interface for message results
 - ✅ Added message editing capability
 - ✅ Ability to select AI models directly while chatting without opening settings
@@ -209,28 +210,35 @@ This fork adds several enhancements to the original Perplexica project:
 - ✅ Display search query with the response
 - ✅ Improved styling for all screen sizes
 - ✅ Added model statistics showing model name and response time
+- ✅ Shows progress during processing
+- ✅ Secures API keys by not showing them in the UI
 
 ### Search and Integration Enhancements
+
 - ✅ OpenSearch support with dynamic XML generation
   - Added BASE_URL config to support reverse proxy deployments
   - Added autocomplete functionality proxied to SearxNG
 - ✅ Enhanced Reddit focus mode to work around SearxNG limitations
+- ✅ Adds Quality mode that uses the full content of web pages to answer queries
+  - Enhances Balanced mode which uses relevant excerpts of web content to answer queries
 
 ### AI Functionality
+
 - ✅ True chat mode implementation (moved writing mode to local research mode)
 - ✅ Enhanced system prompts for more reliable and relevant results
 - ✅ Better parsing for reasoning models
 - ✅ User customizable context window for Ollama models
 - ✅ Toggle for automatic suggestions
+- ✅ Added support for latest Anthropic models
 
 ### Bug Fixes
+
 - ✅ Improved history rewriting
 
 ## Support Us
 
 If you find Perplexica useful, consider giving us a star on GitHub. This helps more people discover Perplexica and supports the development of new features. Your support is greatly appreciated.
 
-
 ## Contribution
 
 Perplexica is built on the idea that AI and large language models should be easy for everyone to use. If you find bugs or have ideas, please share them in via GitHub Issues. For more information on contributing to Perplexica you can read the [CONTRIBUTING.md](CONTRIBUTING.md) file to learn more about Perplexica and how you can contribute to it.

package.json

@@ -22,6 +22,7 @@
     "@langchain/ollama": "^0.2.0",
     "@langchain/openai": "^0.0.25",
     "@langchain/textsplitters": "^0.1.0",
+    "@mozilla/readability": "^0.6.0",
     "@tailwindcss/typography": "^0.5.12",
     "@types/react-syntax-highlighter": "^15.5.13",
     "@xenova/transformers": "^2.17.2",
@@ -32,6 +33,7 @@
     "compute-dot": "^1.1.0",
     "drizzle-orm": "^0.40.1",
     "html-to-text": "^9.0.5",
+    "jsdom": "^26.1.0",
     "langchain": "^0.3.26",
     "lucide-react": "^0.363.0",
     "markdown-to-jsx": "^7.7.2",
@@ -52,6 +54,7 @@
   "devDependencies": {
     "@types/better-sqlite3": "^7.6.12",
     "@types/html-to-text": "^9.0.4",
+    "@types/jsdom": "^21.1.7",
     "@types/node": "^20",
     "@types/pdf-parse": "^1.1.4",
     "@types/react": "^18",

src/app/api/chat/route.ts

@@ -18,10 +18,7 @@ import { ChatOpenAI } from '@langchain/openai';
 import crypto from 'crypto';
 import { and, eq, gt } from 'drizzle-orm';
 import { EventEmitter } from 'stream';
-import {
-  registerCancelToken,
-  cleanupCancelToken,
-} from '@/lib/cancel-tokens';
+import { registerCancelToken, cleanupCancelToken } from '@/lib/cancel-tokens';
 
 export const runtime = 'nodejs';
 export const dynamic = 'force-dynamic';
@@ -115,6 +112,21 @@ const handleEmitterEvents = async (
     modelName: '',
   };
 
+  stream.on('progress', (data) => {
+    const parsedData = JSON.parse(data);
+    if (parsedData.type === 'progress') {
+      writer.write(
+        encoder.encode(
+          JSON.stringify({
+            type: 'progress',
+            data: parsedData.data,
+            messageId: aiMessageId,
+          }) + '\n',
+        ),
+      );
+    }
+  });
+
   stream.on('stats', (data) => {
     const parsedData = JSON.parse(data);
     if (parsedData.type === 'modelStats') {

src/app/api/config/route.ts

@@ -53,7 +53,7 @@ export const GET = async (req: Request) => {
 
     // Helper function to obfuscate API keys
     const protectApiKey = (key: string | null | undefined) => {
-      return key ? "protected" : key;
+      return key ? 'protected' : key;
     };
 
     // Obfuscate all API keys in the response
@@ -85,39 +85,57 @@ export const POST = async (req: Request) => {
   try {
     const config = await req.json();
 
-    const getUpdatedProtectedValue = (newValue: string, currentConfig: string) => {
+    const getUpdatedProtectedValue = (
+      newValue: string,
+      currentConfig: string,
+    ) => {
       if (newValue === 'protected') {
         return currentConfig;
       }
       return newValue;
-    }
+    };
 
     const updatedConfig = {
       MODELS: {
         OPENAI: {
-          API_KEY: getUpdatedProtectedValue(config.openaiApiKey, getOpenaiApiKey()),
+          API_KEY: getUpdatedProtectedValue(
+            config.openaiApiKey,
+            getOpenaiApiKey(),
+          ),
         },
         GROQ: {
           API_KEY: getUpdatedProtectedValue(config.groqApiKey, getGroqApiKey()),
         },
         ANTHROPIC: {
-          API_KEY: getUpdatedProtectedValue(config.anthropicApiKey, getAnthropicApiKey()),
+          API_KEY: getUpdatedProtectedValue(
+            config.anthropicApiKey,
+            getAnthropicApiKey(),
+          ),
         },
         GEMINI: {
-          API_KEY: getUpdatedProtectedValue(config.geminiApiKey, getGeminiApiKey()),
+          API_KEY: getUpdatedProtectedValue(
+            config.geminiApiKey,
+            getGeminiApiKey(),
+          ),
         },
         OLLAMA: {
           API_URL: config.ollamaApiUrl,
         },
         DEEPSEEK: {
-          API_KEY: getUpdatedProtectedValue(config.deepseekApiKey, getDeepseekApiKey()),
+          API_KEY: getUpdatedProtectedValue(
+            config.deepseekApiKey,
+            getDeepseekApiKey(),
+          ),
         },
         LM_STUDIO: {
           API_URL: config.lmStudioApiUrl,
         },
         CUSTOM_OPENAI: {
           API_URL: config.customOpenaiApiUrl,
-          API_KEY: getUpdatedProtectedValue(config.customOpenaiApiKey, getCustomOpenaiApiKey()),
+          API_KEY: getUpdatedProtectedValue(
+            config.customOpenaiApiKey,
+            getCustomOpenaiApiKey(),
+          ),
           MODEL_NAME: config.customOpenaiModelName,
         },
       },

src/app/settings/page.tsx

@@ -1,6 +1,11 @@
 'use client';
 
-import { Settings as SettingsIcon, ArrowLeft, Loader2, Info } from 'lucide-react';
+import {
+  Settings as SettingsIcon,
+  ArrowLeft,
+  Loader2,
+  Info,
+} from 'lucide-react';
 import { useEffect, useState } from 'react';
 import { cn } from '@/lib/utils';
 import { Switch } from '@headlessui/react';
@@ -128,7 +133,10 @@ const SettingsSection = ({
       <h2 className="text-black/90 dark:text-white/90 font-medium">{title}</h2>
       {tooltip && (
         <div className="relative group">
-          <Info size={16} className="text-black/70 dark:text-white/70 cursor-help" />
+          <Info
+            size={16}
+            className="text-black/70 dark:text-white/70 cursor-help"
+          />
           <div className="absolute left-1/2 -translate-x-1/2 bottom-full mb-2 px-3 py-2 bg-black/90 dark:bg-white/90 text-white dark:text-black text-xs rounded-lg opacity-0 group-hover:opacity-100 whitespace-nowrap transition-opacity">
             {tooltip}
           </div>
@@ -238,7 +246,7 @@ const Page = () => {
     fetchConfig();
   }, []);
 
-  const saveConfig = async (key: string, value: any) => {    
+  const saveConfig = async (key: string, value: any) => {
     setSavingStates((prev) => ({ ...prev, [key]: true }));
 
     try {
@@ -798,8 +806,8 @@ const Page = () => {
               )}
             </SettingsSection>
 
-            <SettingsSection 
-              title="API Keys" 
+            <SettingsSection
+              title="API Keys"
               tooltip="API Key values can be viewed in the config.toml file"
             >
               <div className="flex flex-col space-y-4">

src/components/Chat.tsx

@@ -21,6 +21,7 @@ const Chat = ({
   focusMode,
   setFocusMode,
   handleEditMessage,
+  analysisProgress,
 }: {
   messages: Message[];
   sendMessage: (
@@ -43,6 +44,11 @@ const Chat = ({
   focusMode: string;
   setFocusMode: (mode: string) => void;
   handleEditMessage: (messageId: string, content: string) => void;
+  analysisProgress: {
+    message: string;
+    current: number;
+    total: number;
+  } | null;
 }) => {
   const [isAtBottom, setIsAtBottom] = useState(true);
   const [manuallyScrolledUp, setManuallyScrolledUp] = useState(false);
@@ -220,7 +226,7 @@ const Chat = ({
           </Fragment>
         );
       })}
-      {loading && <MessageBoxLoading />}
+      {loading && <MessageBoxLoading progress={analysisProgress} />}
       <div className="fixed bottom-24 lg:bottom-10 z-40" style={inputStyle}>
         {/* Scroll to bottom button - appears above the MessageInput when user has scrolled up */}
         {manuallyScrolledUp && !isAtBottom && (

src/components/ChatWindow.tsx

@@ -29,6 +29,11 @@ export type Message = {
   modelStats?: ModelStats;
   searchQuery?: string;
   searchUrl?: string;
+  progress?: {
+    message: string;
+    current: number;
+    total: number;
+  };
 };
 
 export interface File {
@@ -270,6 +275,11 @@ const ChatWindow = ({ id }: { id?: string }) => {
 
   const [loading, setLoading] = useState(false);
   const [scrollTrigger, setScrollTrigger] = useState(0);
+  const [analysisProgress, setAnalysisProgress] = useState<{
+    message: string;
+    current: number;
+    total: number;
+  } | null>(null);
 
   const [chatHistory, setChatHistory] = useState<[string, string][]>([]);
   const [messages, setMessages] = useState<Message[]>([]);
@@ -405,6 +415,11 @@ const ChatWindow = ({ id }: { id?: string }) => {
         return;
       }
 
+      if (data.type === 'progress') {
+        setAnalysisProgress(data.data);
+        return;
+      }
+
       if (data.type === 'sources') {
         sources = data.data;
         if (!added) {
@@ -460,6 +475,9 @@ const ChatWindow = ({ id }: { id?: string }) => {
       }
 
       if (data.type === 'messageEnd') {
+        // Clear analysis progress
+        setAnalysisProgress(null);
+
         setChatHistory((prevHistory) => [
           ...prevHistory,
           ['human', message],
@@ -656,6 +674,7 @@ const ChatWindow = ({ id }: { id?: string }) => {
               focusMode={focusMode}
               setFocusMode={setFocusMode}
               handleEditMessage={handleEditMessage}
+              analysisProgress={analysisProgress}
             />
           </>
         ) : (

src/components/MessageBoxLoading.tsx

@@ -1,9 +1,37 @@
-const MessageBoxLoading = () => {
+interface MessageBoxLoadingProps {
+  progress: {
+    message: string;
+    current: number;
+    total: number;
+  } | null;
+}
+
+const MessageBoxLoading = ({ progress }: MessageBoxLoadingProps) => {
   return (
-    <div className="flex flex-col space-y-2 w-full lg:w-9/12 bg-light-primary dark:bg-dark-primary animate-pulse rounded-lg py-3">
-      <div className="h-2 rounded-full w-full bg-light-secondary dark:bg-dark-secondary" />
-      <div className="h-2 rounded-full w-9/12 bg-light-secondary dark:bg-dark-secondary" />
-      <div className="h-2 rounded-full w-10/12 bg-light-secondary dark:bg-dark-secondary" />
+    <div className="flex flex-col space-y-4 w-full lg:w-9/12">
+      {progress && progress.current !== progress.total ? (
+        <div className="bg-light-primary dark:bg-dark-primary rounded-lg p-4">
+          <div className="flex flex-col space-y-3">
+            <p className="text-sm text-black/70 dark:text-white/70">
+              {progress.message}
+            </p>
+            <div className="w-full bg-light-secondary dark:bg-dark-secondary rounded-full h-2 overflow-hidden">
+              <div
+                className="h-full bg-[#24A0ED] transition-all duration-300 ease-in-out"
+                style={{
+                  width: `${(progress.current / progress.total) * 100}%`,
+                }}
+              />
+            </div>
+          </div>
+        </div>
+      ) : (
+        <div className="bg-light-primary dark:bg-dark-primary animate-pulse rounded-lg py-3">
+          <div className="h-2 rounded-full w-full bg-light-secondary dark:bg-dark-secondary" />
+          <div className="h-2 mt-2 rounded-full w-9/12 bg-light-secondary dark:bg-dark-secondary" />
+          <div className="h-2 mt-2 rounded-full w-10/12 bg-light-secondary dark:bg-dark-secondary" />
+        </div>
+      )}
     </div>
   );
 };

src/components/MessageInputActions/Optimization.tsx

@@ -22,7 +22,7 @@ const OptimizationModes = [
   },
   {
     key: 'quality',
-    title: 'Quality (Soon)',
+    title: 'Quality',
     description: 'Get the most thorough and accurate answer',
     icon: (
       <Star
@@ -80,13 +80,11 @@ const Optimization = ({
               <PopoverButton
                 onClick={() => handleOptimizationChange(mode.key)}
                 key={i}
-                disabled={mode.key === 'quality'}
                 className={cn(
                   'p-2 rounded-lg flex flex-col items-start justify-start text-start space-y-1 duration-200 cursor-pointer transition',
                   optimizationMode === mode.key
                     ? 'bg-light-secondary dark:bg-dark-secondary'
                     : 'hover:bg-light-secondary dark:hover:bg-dark-secondary',
-                  mode.key === 'quality' && 'opacity-50 cursor-not-allowed',
                 )}
               >
                 <div className="flex flex-row items-center space-x-1 text-black dark:text-white">

src/lib/prompts/webSearch.ts

@@ -3,7 +3,8 @@ export const webSearchRetrieverPrompt = `
 - You are an AI question rephraser
 - You will be given a conversation and a user question
 - Rephrase the question so it is appropriate for web search
-- Only add additional information or change the meaning of the question if it is necessary for clarity or relevance to the conversation
+- Only add additional information or change the meaning of the question if it is necessary for clarity or relevance to the conversation such as adding a date or time for current events, or using historical content to augment the question with relevant context
+- Do not make up any new information like links or URLs
 - Condense the question to its essence and remove any unnecessary details
 - Ensure the question is grammatically correct and free of spelling errors
 - If it is a simple writing task or a greeting (unless the greeting contains a question after it) like Hi, Hello, How are you, etc. instead of a question then you need to return \`not_needed\` as the response in the <answer> XML block

src/lib/providers/anthropic.ts

@@ -9,6 +9,14 @@ export const PROVIDER_INFO = {
 import { BaseChatModel } from '@langchain/core/language_models/chat_models';
 
 const anthropicChatModels: Record<string, string>[] = [
+  {
+    displayName: 'Claude 4 Opus',
+    key: 'claude-opus-4-20250514',
+  },
+  {
+    displayName: 'Claude 4 Sonnet',
+    key: 'claude-sonnet-4-20250514',
+  },
   {
     displayName: 'Claude 3.7 Sonnet',
     key: 'claude-3-7-sonnet-20250219',
@@ -29,10 +37,6 @@ const anthropicChatModels: Record<string, string>[] = [
     displayName: 'Claude 3 Opus',
     key: 'claude-3-opus-20240229',
   },
-  {
-    displayName: 'Claude 3 Sonnet',
-    key: 'claude-3-sonnet-20240229',
-  },
   {
     displayName: 'Claude 3 Haiku',
     key: 'claude-3-haiku-20240307',

src/lib/search/index.ts

@@ -64,6 +64,6 @@ export const searchHandlers: Record<string, MetaSearchAgent> = {
     rerankThreshold: 0.3,
     searchWeb: true,
     summarizer: false,
-    additionalSearchCriteria: '\'site:reddit.com\'',
+    additionalSearchCriteria: "'site:reddit.com'",
   }),
 };

src/lib/search/metaSearchAgent.ts

@@ -22,8 +22,9 @@ import LineOutputParser from '../outputParsers/lineOutputParser';
 import LineListOutputParser from '../outputParsers/listLineOutputParser';
 import { searchSearxng } from '../searxng';
 import computeSimilarity from '../utils/computeSimilarity';
-import { getDocumentsFromLinks } from '../utils/documents';
+import { getDocumentsFromLinks, getWebContent } from '../utils/documents';
 import formatChatHistoryAsString from '../utils/formatHistory';
+import { getModelName } from '../utils/modelUtils';
 
 export interface MetaSearchAgentType {
   searchAndAnswer: (
@@ -64,9 +65,35 @@ class MetaSearchAgent implements MetaSearchAgentType {
     this.config = config;
   }
 
-  private async createSearchRetrieverChain(llm: BaseChatModel) {
+  /**
+   * Emit a progress event with the given percentage and message
+   */
+  private emitProgress(
+    emitter: eventEmitter,
+    percentage: number,
+    message: string,
+  ) {
+    emitter.emit(
+      'progress',
+      JSON.stringify({
+        type: 'progress',
+        data: {
+          message,
+          current: percentage,
+          total: 100,
+        },
+      }),
+    );
+  }
+
+  private async createSearchRetrieverChain(
+    llm: BaseChatModel,
+    emitter: eventEmitter,
+  ) {
     (llm as unknown as ChatOpenAI).temperature = 0;
 
+    this.emitProgress(emitter, 10, `Building search query`);
+
     return RunnableSequence.from([
       PromptTemplate.fromTemplate(this.config.queryGeneratorPrompt),
       llm,
@@ -131,6 +158,8 @@ class MetaSearchAgent implements MetaSearchAgentType {
             }
           });
 
+          this.emitProgress(emitter, 20, `Summarizing content`);
+
           await Promise.all(
             docGroups.map(async (doc) => {
               const res = await llm.invoke(`
@@ -208,6 +237,7 @@ class MetaSearchAgent implements MetaSearchAgentType {
 
           return { query: question, docs: docs };
         } else {
+          this.emitProgress(emitter, 20, `Searching the web`);
           if (this.config.additionalSearchCriteria) {
             question = `${question} ${this.config.additionalSearchCriteria}`;
           }
@@ -249,6 +279,7 @@ class MetaSearchAgent implements MetaSearchAgentType {
     optimizationMode: 'speed' | 'balanced' | 'quality',
     systemInstructions: string,
     signal: AbortSignal,
+    emitter: eventEmitter,
   ) {
     return RunnableSequence.from([
       RunnableMap.from({
@@ -276,7 +307,7 @@ class MetaSearchAgent implements MetaSearchAgentType {
 
             if (this.config.searchWeb) {
               const searchRetrieverChain =
-                await this.createSearchRetrieverChain(llm);
+                await this.createSearchRetrieverChain(llm, emitter);
               var date = new Date().toISOString();
 
               const searchRetrieverResult = await searchRetrieverChain.invoke(
@@ -303,8 +334,14 @@ class MetaSearchAgent implements MetaSearchAgentType {
               fileIds,
               embeddings,
               optimizationMode,
+              llm,
+              emitter,
+              signal,
             );
 
+            console.log('Ranked docs:', sortedDocs);
+
+            this.emitProgress(emitter, 100, `Done`);
             return sortedDocs;
           },
         )
@@ -331,11 +368,18 @@ class MetaSearchAgent implements MetaSearchAgentType {
     fileIds: string[],
     embeddings: Embeddings,
     optimizationMode: 'speed' | 'balanced' | 'quality',
-  ) {
+    llm: BaseChatModel,
+    emitter: eventEmitter,
+    signal: AbortSignal,
+  ): Promise<Document[]> {
     if (docs.length === 0 && fileIds.length === 0) {
       return docs;
     }
 
+    if (query.toLocaleLowerCase() === 'summarize') {
+      return docs.slice(0, 15);
+    }
+
     const filesData = fileIds
       .map((file) => {
         const filePath = path.join(process.cwd(), 'uploads', file);
@@ -360,107 +404,216 @@ class MetaSearchAgent implements MetaSearchAgentType {
       })
       .flat();
 
-    if (query.toLocaleLowerCase() === 'summarize') {
-      return docs.slice(0, 15);
-    }
-
-    const docsWithContent = docs.filter(
+    let docsWithContent = docs.filter(
       (doc) => doc.pageContent && doc.pageContent.length > 0,
     );
 
-    if (optimizationMode === 'speed' || this.config.rerank === false) {
-      if (filesData.length > 0) {
-        const [queryEmbedding] = await Promise.all([
-          embeddings.embedQuery(query),
-        ]);
+    const queryEmbedding = await embeddings.embedQuery(query);
 
+    const getRankedDocs = async (
+      queryEmbedding: number[],
+      includeFiles: boolean,
+      includeNonFileDocs: boolean,
+      maxDocs: number,
+    ) => {
+      let docsToRank = includeNonFileDocs ? docsWithContent : [];
+
+      if (includeFiles) {
+        // Add file documents to the ranking
         const fileDocs = filesData.map((fileData) => {
           return new Document({
             pageContent: fileData.content,
             metadata: {
               title: fileData.fileName,
               url: `File`,
+              embeddings: fileData.embeddings,
             },
           });
         });
+        docsToRank.push(...fileDocs);
+      }
 
-        const similarity = filesData.map((fileData, i) => {
-          const sim = computeSimilarity(queryEmbedding, fileData.embeddings);
-
+      const similarity = await Promise.all(
+        docsToRank.map(async (doc, i) => {
+          const sim = computeSimilarity(
+            queryEmbedding,
+            doc.metadata?.embeddings
+              ? doc.metadata?.embeddings
+              : (await embeddings.embedDocuments([doc.pageContent]))[0],
+          );
           return {
             index: i,
             similarity: sim,
           };
-        });
+        }),
+      );
 
-        let sortedDocs = similarity
-          .filter(
-            (sim) => sim.similarity > (this.config.rerankThreshold ?? 0.3),
-          )
-          .sort((a, b) => b.similarity - a.similarity)
-          .slice(0, 15)
-          .map((sim) => fileDocs[sim.index]);
+      let rankedDocs = similarity
+        .filter((sim) => sim.similarity > (this.config.rerankThreshold ?? 0.3))
+        .sort((a, b) => b.similarity - a.similarity)
+        .map((sim) => docsToRank[sim.index]);
 
-        sortedDocs =
-          docsWithContent.length > 0 ? sortedDocs.slice(0, 8) : sortedDocs;
+      rankedDocs =
+        docsToRank.length > 0 ? rankedDocs.slice(0, maxDocs) : rankedDocs;
+      return rankedDocs;
+    };
+
+    if (optimizationMode === 'speed' || this.config.rerank === false) {
+      this.emitProgress(emitter, 50, `Ranking sources`);
+      if (filesData.length > 0) {
+        const sortedFiles = await getRankedDocs(queryEmbedding, true, false, 8);
 
         return [
-          ...sortedDocs,
-          ...docsWithContent.slice(0, 15 - sortedDocs.length),
+          ...sortedFiles,
+          ...docsWithContent.slice(0, 15 - sortedFiles.length),
         ];
       } else {
         return docsWithContent.slice(0, 15);
       }
     } else if (optimizationMode === 'balanced') {
-      const [docEmbeddings, queryEmbedding] = await Promise.all([
-        embeddings.embedDocuments(
-          docsWithContent.map((doc) => doc.pageContent),
-        ),
-        embeddings.embedQuery(query),
-      ]);
+      this.emitProgress(emitter, 40, `Ranking sources`);
+      let sortedDocs = await getRankedDocs(queryEmbedding, true, true, 10);
 
-      docsWithContent.push(
-        ...filesData.map((fileData) => {
-          return new Document({
-            pageContent: fileData.content,
-            metadata: {
-              title: fileData.fileName,
-              url: `File`,
-            },
+      this.emitProgress(emitter, 60, `Enriching sources`);
+      sortedDocs = await Promise.all(
+        sortedDocs.map(async (doc) => {
+          const webContent = await getWebContent(doc.metadata.url);
+          const chunks =
+            webContent?.pageContent
+              .match(/.{1,500}/g)
+              ?.map((chunk) => chunk.trim()) || [];
+          const chunkEmbeddings = await embeddings.embedDocuments(chunks);
+          const similarities = chunkEmbeddings.map((chunkEmbedding) => {
+            return computeSimilarity(queryEmbedding, chunkEmbedding);
           });
+
+          const topChunks = similarities
+            .map((similarity, index) => ({ similarity, index }))
+            .sort((a, b) => b.similarity - a.similarity)
+            .slice(0, 5)
+            .map((chunk) => chunks[chunk.index]);
+          const excerpt = topChunks.join('\n\n');
+
+          let newDoc = {
+            ...doc,
+            pageContent: excerpt
+              ? `${excerpt}\n\n${doc.pageContent}`
+              : doc.pageContent,
+          };
+          return newDoc;
         }),
       );
 
-      docEmbeddings.push(...filesData.map((fileData) => fileData.embeddings));
+      return sortedDocs;
+    } else if (optimizationMode === 'quality') {
+      this.emitProgress(emitter, 30, 'Ranking sources...');
 
-      const similarity = docEmbeddings.map((docEmbedding, i) => {
-        const sim = computeSimilarity(queryEmbedding, docEmbedding);
+      // Get the top ranked web results for detailed analysis based off their preview embeddings
+      const topWebResults = await getRankedDocs(
+        queryEmbedding,
+        false,
+        true,
+        30,
+      );
 
-        return {
-          index: i,
-          similarity: sim,
-        };
+      const summaryParser = new LineOutputParser({
+        key: 'summary',
       });
 
-      const sortedDocs = similarity
-        .filter((sim) => sim.similarity > (this.config.rerankThreshold ?? 0.3))
-        .sort((a, b) => b.similarity - a.similarity)
-        .slice(0, 15)
-        .map((sim) => docsWithContent[sim.index]);
+      // Get full content and generate detailed summaries for top results sequentially
+      const enhancedDocs: Document[] = [];
+      const maxEnhancedDocs = 5;
+      for (let i = 0; i < topWebResults.length; i++) {
+        if (signal.aborted) {
+          return [];
+        }
+        if (enhancedDocs.length >= maxEnhancedDocs) {
+          break; // Limit to 5 documents
+        }
+        const result = topWebResults[i];
 
-      return sortedDocs;
+        this.emitProgress(
+          emitter,
+          enhancedDocs.length * 10 + 40,
+          `Deep analyzing sources: ${enhancedDocs.length + 1}/${maxEnhancedDocs}`,
+        );
+
+        try {
+          const url = result.metadata.url;
+          const webContent = await getWebContent(url, true);
+
+          if (webContent) {
+            // Generate a detailed summary using the LLM
+            const summary = await llm.invoke(`
+              You are a web content summarizer, tasked with creating a detailed, accurate summary of content from a webpage
+              Your summary should:
+              - Be thorough and comprehensive, capturing all key points
+              - Format the content using markdown, including headings, lists, and tables
+              - Include specific details, numbers, and quotes when relevant
+              - Be concise and to the point, avoiding unnecessary fluff
+              - Answer the user's query, which is: ${query}
+              - Output your answer in an XML format, with the summary inside the \`summary\` XML tag
+              - If the content is not relevant to the query, respond with "not_needed" to start the summary tag, followed by a one line description of why the source is not needed
+                - E.g. "not_needed: There is relevant information in the source, but it doesn't contain specifics about X"
+                - Make sure the reason the source is not needed is very specific and detailed
+              - Include useful links to external resources, if applicable
+
+              Here is the content to summarize:
+              ${webContent.metadata.html ? webContent.metadata.html : webContent.pageContent}
+            `);
+
+            const summarizedContent = await summaryParser.parse(
+              summary.content as string,
+            );
+
+            if (
+              summarizedContent.toLocaleLowerCase().startsWith('not_needed')
+            ) {
+              console.log(
+                `LLM response for URL "${url}" indicates it's not needed:`,
+                summarizedContent,
+              );
+              continue; // Skip this document if not needed
+            }
+
+            //console.log(`LLM response for URL "${url}":`, summarizedContent);
+            enhancedDocs.push(
+              new Document({
+                pageContent: summarizedContent,
+                metadata: {
+                  ...webContent.metadata,
+                  url: url,
+                },
+              }),
+            );
+          }
+        } catch (error) {
+          console.error(`Error processing URL ${result.metadata.url}:`, error);
+        }
+      }
+
+      // Add relevant file documents
+      const fileDocs = await getRankedDocs(queryEmbedding, true, false, 5);
+
+      return [...enhancedDocs, ...fileDocs];
     }
 
     return [];
   }
 
   private processDocs(docs: Document[]) {
-    return docs
+    const fullDocs = docs
       .map(
         (_, index) =>
-          `${index + 1}. ${docs[index].metadata.title} ${docs[index].pageContent}`,
+          `<${index + 1}>\n
+<title>${docs[index].metadata.title}</title>\n
+${docs[index].metadata?.url.toLowerCase().includes('file') ? '' : '\n<url>' + docs[index].metadata.url + '</url>\n'}
+<content>\n${docs[index].pageContent}\n</content>\n
+</${index + 1}>\n`,
       )
       .join('\n');
+    // console.log('Processed docs:', fullDocs);
+    return fullDocs;
   }
 
   private async handleStream(
@@ -513,38 +666,7 @@ class MetaSearchAgent implements MetaSearchAgentType {
         event.event === 'on_chain_end' &&
         event.name === 'FinalResponseGenerator'
       ) {
-        // Get model name safely with better detection
-        let modelName = 'Unknown';
-        try {
-          // @ts-ignore - Different LLM implementations have different properties
-          if (llm.modelName) {
-            // @ts-ignore
-            modelName = llm.modelName;
-            // @ts-ignore
-          } else if (llm._llm && llm._llm.modelName) {
-            // @ts-ignore
-            modelName = llm._llm.modelName;
-            // @ts-ignore
-          } else if (llm.model && llm.model.modelName) {
-            // @ts-ignore
-            modelName = llm.model.modelName;
-          } else if ('model' in llm) {
-            // @ts-ignore
-            const model = llm.model;
-            if (typeof model === 'string') {
-              modelName = model;
-              // @ts-ignore
-            } else if (model && model.modelName) {
-              // @ts-ignore
-              modelName = model.modelName;
-            }
-          } else if (llm.constructor && llm.constructor.name) {
-            // Last resort: use the class name
-            modelName = llm.constructor.name;
-          }
-        } catch (e) {
-          console.error('Failed to get model name:', e);
-        }
+        const modelName = getModelName(llm);
 
         // Send model info before ending
         emitter.emit(
@@ -581,6 +703,7 @@ class MetaSearchAgent implements MetaSearchAgentType {
       optimizationMode,
       systemInstructions,
       signal,
+      emitter,
     );
 
     const stream = answeringChain.streamEvents(

src/lib/utils/documents.ts

@@ -3,6 +3,9 @@ import { htmlToText } from 'html-to-text';
 import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter';
 import { Document } from '@langchain/core/documents';
 import pdfParse from 'pdf-parse';
+import { JSDOM } from 'jsdom';
+import { Readability } from '@mozilla/readability';
+import fetch from 'node-fetch';
 
 export const getDocumentsFromLinks = async ({ links }: { links: string[] }) => {
   const splitter = new RecursiveCharacterTextSplitter();
@@ -97,3 +100,55 @@ export const getDocumentsFromLinks = async ({ links }: { links: string[] }) => {
 
   return docs;
 };
+
+export const getWebContent = async (
+  url: string,
+  getHtml: boolean = false,
+): Promise<Document | null> => {
+  try {
+    const response = await fetch(url, { timeout: 5000 });
+    const html = await response.text();
+
+    // Create a DOM from the fetched HTML
+    const dom = new JSDOM(html, { url });
+
+    // Get title before we modify the DOM
+    const originalTitle = dom.window.document.title;
+
+    // Use Readability to parse the article content
+    const reader = new Readability(dom.window.document, { charThreshold: 25 });
+    const article = reader.parse();
+
+    if (!article) {
+      console.warn(`Failed to parse article content for URL: ${url}`);
+      return null;
+    }
+
+    // Normalize the text content by removing extra spaces and newlines. Iterate through the lines one by one and throw out the ones that are empty or contain only whitespace.
+    const normalizedText =
+      article?.textContent
+        ?.split('\n')
+        .map((line) => line.trim())
+        .filter((line) => line.length > 0)
+        .join('\n') || '';
+
+    // Create a Document with the parsed content
+    return new Document({
+      pageContent: normalizedText || '',
+      metadata: {
+        html: getHtml ? article.content : undefined,
+        title: article.title || originalTitle,
+        url: url,
+        excerpt: article.excerpt || undefined,
+        byline: article.byline || undefined,
+        siteName: article.siteName || undefined,
+        readingTime: article.length
+          ? Math.ceil(article.length / 1000)
+          : undefined,
+      },
+    });
+  } catch (error) {
+    console.error(`Error fetching/parsing URL ${url}:`); //, error);
+    return null;
+  }
+};

src/lib/utils/modelUtils.ts

@@ -0,0 +1,52 @@
+import { BaseChatModel } from '@langchain/core/language_models/chat_models';
+
+/**
+ * Extract the model name from an LLM instance
+ * Handles different LLM implementations that may store the model name in different properties
+ * @param llm The LLM instance
+ * @returns The model name or 'Unknown' if not found
+ */
+export function getModelName(llm: BaseChatModel): string {
+  try {
+    // @ts-ignore - Different LLM implementations have different properties
+    if (llm.modelName) {
+      // @ts-ignore
+      return llm.modelName;
+    }
+
+    // @ts-ignore
+    if (llm._llm && llm._llm.modelName) {
+      // @ts-ignore
+      return llm._llm.modelName;
+    }
+
+    // @ts-ignore
+    if (llm.model && llm.model.modelName) {
+      // @ts-ignore
+      return llm.model.modelName;
+    }
+
+    if ('model' in llm) {
+      // @ts-ignore
+      const model = llm.model;
+      if (typeof model === 'string') {
+        return model;
+      }
+      // @ts-ignore
+      if (model && model.modelName) {
+        // @ts-ignore
+        return model.modelName;
+      }
+    }
+
+    if (llm.constructor && llm.constructor.name) {
+      // Last resort: use the class name
+      return llm.constructor.name;
+    }
+
+    return 'Unknown';
+  } catch (e) {
+    console.error('Failed to get model name:', e);
+    return 'Unknown';
+  }
+}