sk/Perplexica
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(media search): Improve media search UI and prompts to more reliably return data.

Browse source
This commit is contained in:
Willie Zutz 2025-05-10 02:09:56 -06:00
parent 21e50db489
commit 1f74b815c8
4 changed files with 193 additions and 68 deletions
Download patch file Download diff file Expand all files Collapse all files

40
src/components/SearchImages.tsx
View file

@ -25,15 +25,27 @@ const SearchImages = ({
const [loading, setLoading] = useState(true);
const [open, setOpen] = useState(false);
const [slides, setSlides] = useState<any[]>([]);
const hasLoadedRef = useRef(false);
const [displayLimit, setDisplayLimit] = useState(10); // Initially show only 10 images
const loadedMessageIdsRef = useRef<Set<string>>(new Set());
// Function to show more images when the Show More button is clicked
const handleShowMore = () => {
// If we're already showing all images, don't do anything
if (images && displayLimit >= images.length) return;
// Otherwise, increase the display limit by 10, or show all images
setDisplayLimit(prev => images ? Math.min(prev + 10, images.length) : prev);
};
useEffect(() => {
// Skip fetching if images are already loaded for this message
if (hasLoadedRef.current) {
if (loadedMessageIdsRef.current.has(messageId)) {
return;
}
const fetchImages = async () => {
// Mark as loaded to prevent refetching
loadedMessageIdsRef.current.add(messageId);
setLoading(true);
const chatModelProvider = localStorage.getItem('chatModelProvider');
@ -80,8 +92,7 @@ const SearchImages = ({
if (onImagesLoaded && images.length > 0) {
onImagesLoaded(images.length);
}
// Mark as loaded to prevent refetching
hasLoadedRef.current = true;
} catch (error) {
console.error('Error fetching images:', error);
} finally {
@ -91,11 +102,7 @@ const SearchImages = ({
fetchImages();
// Reset the loading state when component unmounts
return () => {
hasLoadedRef.current = false;
};
}, [query, messageId]);
}, [query, messageId, chatHistory, onImagesLoaded]);
return (
<>
@ -111,8 +118,8 @@ const SearchImages = ({
)}
{images !== null && images.length > 0 && (
<>
<div className="grid grid-cols-2 gap-2">
{images.map((image, i) => (
<div className="grid grid-cols-2 gap-2" key={`image-results-${messageId}`}>
{images.slice(0, displayLimit).map((image, i) => (
<img
onClick={() => {
setOpen(true);
@ -129,6 +136,17 @@ const SearchImages = ({
/>
))}
</div>
{images.length > displayLimit && (
<div className="flex justify-center mt-4">
<button
onClick={handleShowMore}
className="px-4 py-2 bg-light-secondary dark:bg-dark-secondary hover:bg-light-200 dark:hover:bg-dark-200 text-black/70 dark:text-white/70 hover:text-black dark:hover:text-white rounded-md transition duration-200 flex items-center space-x-2"
>
<span>Show More Images</span>
<span className="text-sm opacity-75">({displayLimit} of {images.length})</span>
</button>
</div>
)}
<Lightbox open={open} close={() => setOpen(false)} slides={slides} />
</>
)}

37
src/components/SearchVideos.tsx
View file

@ -40,12 +40,22 @@ const Searchvideos = ({
const [open, setOpen] = useState(false);
const [slides, setSlides] = useState<VideoSlide[]>([]);
const [currentIndex, setCurrentIndex] = useState(0);
const [displayLimit, setDisplayLimit] = useState(10); // Initially show only 10 videos
const videoRefs = useRef<(HTMLIFrameElement | null)[]>([]);
const hasLoadedRef = useRef(false);
const loadedMessageIdsRef = useRef<Set<string>>(new Set());
// Function to show more videos when the Show More button is clicked
const handleShowMore = () => {
// If we're already showing all videos, don't do anything
if (videos && displayLimit >= videos.length) return;
// Otherwise, increase the display limit by 10, or show all videos
setDisplayLimit(prev => videos ? Math.min(prev + 10, videos.length) : prev);
};
useEffect(() => {
// Skip fetching if videos are already loaded for this message
if (hasLoadedRef.current) {
if (loadedMessageIdsRef.current.has(messageId)) {
return;
}
@ -99,7 +109,7 @@ const Searchvideos = ({
onVideosLoaded(videos.length);
}
// Mark as loaded to prevent refetching
hasLoadedRef.current = true;
loadedMessageIdsRef.current.add(messageId);
} catch (error) {
console.error('Error fetching videos:', error);
} finally {
@ -109,11 +119,7 @@ const Searchvideos = ({
fetchVideos();
// Reset the loading state when component unmounts
return () => {
hasLoadedRef.current = false;
};
}, [query, messageId]);
}, [query, messageId, chatHistory, onVideosLoaded]);
return (
<>
@ -129,8 +135,8 @@ const Searchvideos = ({
)}
{videos !== null && videos.length > 0 && (
<>
<div className="grid grid-cols-2 gap-2">
{videos.map((video, i) => (
<div className="grid grid-cols-2 gap-2" key={`video-results-${messageId}`}>
{videos.slice(0, displayLimit).map((video, i) => (
<div
onClick={() => {
setOpen(true);
@ -155,6 +161,17 @@ const Searchvideos = ({
</div>
))}
</div>
{videos.length > displayLimit && (
<div className="flex justify-center mt-4">
<button
onClick={handleShowMore}
className="px-4 py-2 bg-light-secondary dark:bg-dark-secondary hover:bg-light-200 dark:hover:bg-dark-200 text-black/70 dark:text-white/70 hover:text-black dark:hover:text-white rounded-md transition duration-200 flex items-center space-x-2"
>
<span>Show More Videos</span>
<span className="text-sm opacity-75">({displayLimit} of {videos.length})</span>
</button>
</div>
)}
<Lightbox
open={open}
close={() => setOpen(false)}

87
src/lib/chains/imageSearchAgent.ts
View file

@ -6,29 +6,73 @@ import {
import { PromptTemplate } from '@langchain/core/prompts';
import formatChatHistoryAsString from '../utils/formatHistory';
import { BaseMessage } from '@langchain/core/messages';
import { StringOutputParser } from '@langchain/core/output_parsers';
import LineOutputParser from '../outputParsers/lineOutputParser';
import { searchSearxng } from '../searxng';
import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
const imageSearchChainPrompt = `
You will be given a conversation below and a follow up question. You need to rephrase the follow-up question so it is a standalone question that can be used by the LLM to search the web for images.
You need to make sure the rephrased question agrees with the conversation and is relevant to the conversation.
# Instructions
- You will be given a question from a user and a conversation history
- Rephrase the question based on the conversation so it is a standalone question that can be used to search for images that are relevant to the question
- Ensure the rephrased question agrees with the conversation and is relevant to the conversation
- If you are thinking or reasoning, use <think> tags to indicate your thought process
- If you are thinking or reasoning, do not use <answer> and </answer> tags in your thinking. Those tags should only be used in the final output
- Use the provided date to ensure the rephrased question is relevant to the current date and time if applicable
Example:
1. Follow up question: What is a cat?
Rephrased: A cat
# Data locations
- The history is contained in the <conversation> tag after the <examples> below
- The user question is contained in the <question> tag after the <examples> below
- Output your answer in an <answer> tag
- Current date & time in ISO format (UTC timezone) is: {date}
- Do not include any other text in your answer
2. Follow up question: What is a car? How does it works?
Rephrased: Car working
<examples>
## Example 1 input
<conversation>
Who won the last F1 race?\nAyrton Senna won the Monaco Grand Prix. It was a tight race with lots of overtakes. Alain Prost was in the lead for most of the race until the last lap when Senna overtook them.
</conversation>
<question>
What were the highlights of the race?
</question>
3. Follow up question: How does an AC work?
Rephrased: AC working
## Example 1 output
<answer>
F1 Monaco Grand Prix highlights
</answer>
Conversation:
## Example 2 input
<conversation>
What is the theory of relativity?
</conversation>
<question>
What is the theory of relativity?
</question>
## Example 2 output
<answer>
Theory of relativity
</answer>
## Example 3 input
<conversation>
I'm looking for a nice vacation spot. Where do you suggest?\nI suggest you go to Hawaii. It's a beautiful place with lots of beaches and activities to do.\nI love the beach! What are some activities I can do there?\nYou can go surfing, snorkeling, or just relax on the beach.
</conversation>
<question>
What are some activities I can do in Hawaii?
</question>
## Example 3 output
<answer>
Hawaii activities
</answer>
</examples>
<conversation>
{chat_history}
Follow up question: {query}
Rephrased question:
</conversation>
<question>
{query}
</question>
`;
type ImageSearchChainInput = {
@ -42,7 +86,9 @@ interface ImageSearchResult {
title: string;
}
const strParser = new StringOutputParser();
const outputParser = new LineOutputParser({
key: 'answer',
});
const createImageSearchChain = (llm: BaseChatModel) => {
return RunnableSequence.from([
@ -53,14 +99,13 @@ const createImageSearchChain = (llm: BaseChatModel) => {
query: (input: ImageSearchChainInput) => {
return input.query;
},
date: () => new Date().toISOString(),
}),
PromptTemplate.fromTemplate(imageSearchChainPrompt),
llm,
strParser,
RunnableLambda.from(async (input: string) => {
input = input.replace(/<think>.*?<\/think>/g, '');
const res = await searchSearxng(input, {
outputParser,
RunnableLambda.from(async (searchQuery: string) => {
const res = await searchSearxng(searchQuery, {
engines: ['bing images', 'google images'],
});
@ -76,7 +121,7 @@ const createImageSearchChain = (llm: BaseChatModel) => {
}
});
return images.slice(0, 10);
return images;
}),
]);
};

87
src/lib/chains/videoSearchAgent.ts
View file

@ -6,29 +6,73 @@ import {
import { PromptTemplate } from '@langchain/core/prompts';
import formatChatHistoryAsString from '../utils/formatHistory';
import { BaseMessage } from '@langchain/core/messages';
import { StringOutputParser } from '@langchain/core/output_parsers';
import LineOutputParser from '../outputParsers/lineOutputParser';
import { searchSearxng } from '../searxng';
import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
const VideoSearchChainPrompt = `
You will be given a conversation below and a follow up question. You need to rephrase the follow-up question so it is a standalone question that can be used by the LLM to search Youtube for videos.
You need to make sure the rephrased question agrees with the conversation and is relevant to the conversation.
# Instructions
- You will be given a question from a user and a conversation history
- Rephrase the question based on the conversation so it is a standalone question that can be used to search Youtube for videos
- Ensure the rephrased question agrees with the conversation and is relevant to the conversation
- If you are thinking or reasoning, use <think> tags to indicate your thought process
- If you are thinking or reasoning, do not use <answer> and </answer> tags in your thinking. Those tags should only be used in the final output
- Use the provided date to ensure the rephrased question is relevant to the current date and time if applicable
Example:
1. Follow up question: How does a car work?
Rephrased: How does a car work?
# Data locations
- The history is contained in the <conversation> tag after the <examples> below
- The user question is contained in the <question> tag after the <examples> below
- Output your answer in an <answer> tag
- Current date & time in ISO format (UTC timezone) is: {date}
- Do not include any other text in your answer
2. Follow up question: What is the theory of relativity?
Rephrased: What is theory of relativity
<examples>
## Example 1 input
<conversation>
Who won the last F1 race?\nAyrton Senna won the Monaco Grand Prix. It was a tight race with lots of overtakes. Alain Prost was in the lead for most of the race until the last lap when Senna overtook them.
</conversation>
<question>
What were the highlights of the race?
</question>
3. Follow up question: How does an AC work?
Rephrased: How does an AC work
## Example 1 output
<answer>
F1 Monaco Grand Prix highlights
</answer>
Conversation:
## Example 2 input
<conversation>
What is the theory of relativity?
</conversation>
<question>
What is the theory of relativity?
</question>
## Example 2 output
<answer>
What is the theory of relativity?
</answer>
## Example 3 input
<conversation>
I'm looking for a nice vacation spot. Where do you suggest?\nI suggest you go to Hawaii. It's a beautiful place with lots of beaches and activities to do.\nI love the beach! What are some activities I can do there?\nYou can go surfing, snorkeling, or just relax on the beach.
</conversation>
<question>
What are some activities I can do in Hawaii?
</question>
## Example 3 output
<answer>
Activities to do in Hawaii
</answer>
</examples>
<conversation>
{chat_history}
Follow up question: {query}
Rephrased question:
</conversation>
<question>
{query}
</question>
`;
type VideoSearchChainInput = {
@ -43,7 +87,9 @@ interface VideoSearchResult {
iframe_src: string;
}
const strParser = new StringOutputParser();
const answerParser = new LineOutputParser({
key: 'answer',
});
const createVideoSearchChain = (llm: BaseChatModel) => {
return RunnableSequence.from([
@ -54,14 +100,13 @@ const createVideoSearchChain = (llm: BaseChatModel) => {
query: (input: VideoSearchChainInput) => {
return input.query;
},
date: () => new Date().toISOString(),
}),
PromptTemplate.fromTemplate(VideoSearchChainPrompt),
llm,
strParser,
RunnableLambda.from(async (input: string) => {
input = input.replace(/<think>.*?<\/think>/g, '');
const res = await searchSearxng(input, {
answerParser,
RunnableLambda.from(async (searchQuery: string) => {
const res = await searchSearxng(searchQuery, {
engines: ['youtube'],
});
@ -83,7 +128,7 @@ const createVideoSearchChain = (llm: BaseChatModel) => {
}
});
return videos.slice(0, 10);
return videos;
}),
]);
};