From 03fe59979d3f6fac670e468db75804a17fefbea1 Mon Sep 17 00:00:00 2001 From: ahaapple Date: Thu, 19 Dec 2024 16:09:25 +0800 Subject: [PATCH] Improve web images processing --- .../components/modal/web-images-model.tsx | 164 ++++++++++++++++++ frontend/components/search/search-bar.tsx | 73 ++++---- frontend/components/search/search-message.tsx | 22 +-- frontend/components/search/search-window.tsx | 22 +-- frontend/hooks/use-upload-file.ts | 2 +- frontend/lib/llm/llm.ts | 12 +- frontend/lib/shared-utils.ts | 18 ++ frontend/lib/tools/auto.ts | 3 - frontend/messages/en.json | 2 +- frontend/messages/zh.json | 2 +- 10 files changed, 255 insertions(+), 65 deletions(-) create mode 100644 frontend/components/modal/web-images-model.tsx diff --git a/frontend/components/modal/web-images-model.tsx b/frontend/components/modal/web-images-model.tsx new file mode 100644 index 00000000..7810e3a9 --- /dev/null +++ b/frontend/components/modal/web-images-model.tsx @@ -0,0 +1,164 @@ +import React, { useState } from 'react'; +import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogFooter } from '@/components/ui/dialog'; +import { Input } from '@/components/ui/input'; +import { Button } from '@/components/ui/button'; +import { toast } from 'sonner'; +import { X } from 'lucide-react'; + +export interface WebImageFile { + url: string; + type: 'image'; + name: string; + size: number; +} + +interface WebImageUrlModalProps { + open: boolean; + onOpenChange: (open: boolean) => void; + onImagesAdded: (images: WebImageFile[]) => void; +} + +const WebImageModal: React.FC = ({ open, onOpenChange, onImagesAdded }) => { + const [imageUrls, setImageUrls] = useState(['']); + const [previews, setPreviews] = useState([]); + const [urlValidities, setUrlValidities] = useState([false]); + + const isValidImageUrl = (url: string): boolean => { + try { + const urlObj = new URL(url); + const validExtensions = ['.jpg', '.jpeg', '.png', '.webp', '.gif']; + return validExtensions.some((ext) => urlObj.pathname.toLowerCase().endsWith(ext)); + } catch { + return false; + } + }; + + const checkImageValidity = async (url: string): Promise => { + if (!isValidImageUrl(url)) return false; + + return new Promise((resolve) => { + const img = new Image(); + img.onload = () => resolve(true); + img.onerror = () => resolve(false); + img.src = url; + }); + }; + + const updateUrlInput = async (index: number, value: string) => { + const newUrls = [...imageUrls]; + newUrls[index] = value; + setImageUrls(newUrls); + + if (value.trim()) { + const isValid = await checkImageValidity(value); + const newValidities = [...urlValidities]; + newValidities[index] = isValid; + setUrlValidities(newValidities); + + const newPreviews = [...previews]; + if (isValid) { + newPreviews[index] = value; + } else { + newPreviews[index] = ''; + } + setPreviews(newPreviews); + } + }; + + const addUrlInput = () => { + setImageUrls((prev) => [...prev, '']); + setUrlValidities((prev) => [...prev, false]); + setPreviews((prev) => [...prev, '']); + }; + + const removeUrlInput = (index: number) => { + const newUrls = imageUrls.filter((_, i) => i !== index); + const newValidities = urlValidities.filter((_, i) => i !== index); + const newPreviews = previews.filter((_, i) => i !== index); + + setImageUrls(newUrls); + setUrlValidities(newValidities); + setPreviews(newPreviews); + }; + + const handleImageUrlSubmit = async () => { + const validUrls = imageUrls.filter((_, index) => urlValidities[index]); + + if (validUrls.length === 0) { + toast.error('Please enter valid image URLs'); + return; + } + + try { + const newUploadedFiles: WebImageFile[] = validUrls.map((url) => ({ + url, + type: 'image', + name: `web-image-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`, + size: 1, + })); + + onImagesAdded(newUploadedFiles); + + setImageUrls(['']); + setUrlValidities([false]); + setPreviews([]); + onOpenChange(false); + + toast.success(`${validUrls.length} image(s) added successfully`); + } catch (error) { + toast.error('Failed to add image URLs'); + } + }; + + return ( + + + + Add Public Web Image URLs + +
+ {imageUrls.map((url, index) => ( +
+
+ updateUrlInput(index, e.target.value)} + placeholder="Enter image URL" + className={` + ${urlValidities[index] ? 'border-green-500' : url.trim() ? 'border-red-500' : ''} + `} + /> +
+ +
+ ))} + +

Supported Image Formats: JPG, PNG, WebP, GIF

+ + {previews.some((preview) => preview) && ( +
+ {previews.map( + (preview, index) => + preview && {`Preview, + )} +
+ )} +
+ + + + + + + ); +}; + +export default WebImageModal; diff --git a/frontend/components/search/search-bar.tsx b/frontend/components/search/search-bar.tsx index 95a8ac12..6a4bb770 100644 --- a/frontend/components/search/search-bar.tsx +++ b/frontend/components/search/search-bar.tsx @@ -2,8 +2,7 @@ import React, { KeyboardEvent, useMemo, useRef, useState } from 'react'; import { useSigninModal } from '@/hooks/use-signin-modal'; -import { SendHorizontal, FileTextIcon, Database, Image as ImageIcon } from 'lucide-react'; -import { useIndexModal } from '@/hooks/use-index-modal'; +import { SendHorizontal, FileTextIcon, Database, Image as ImageIcon, Link } from 'lucide-react'; import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip'; import { ModelSelection } from '@/components/search/model-selection'; import { SourceSelection } from '@/components/search/source-selection'; @@ -11,7 +10,6 @@ import TextareaAutosize from 'react-textarea-autosize'; import { useUIStore, useUserStore } from '@/lib/store'; import { toast } from 'sonner'; import { Icons } from '@/components/shared/icons'; -import Image from 'next/image'; import { type FileRejection, useDropzone } from 'react-dropzone'; import { useUploadFile } from '@/hooks/use-upload-file'; import { useUpgradeModal } from '@/hooks/use-upgrade-modal'; @@ -22,6 +20,7 @@ import dynamic from 'next/dynamic'; import { Switch } from '@/components/ui/switch'; import { Label } from '@/components/ui/label'; import { SearchType } from '@/lib/types'; +import WebImageModal, { WebImageFile } from '@/components/modal/web-images-model'; interface Props { handleSearch: (key: string, attachments?: string[]) => void; @@ -46,10 +45,27 @@ const SearchBar: React.FC = ({ }) => { const [content, setContent] = useState(''); const signInModal = useSigninModal(); - const indexModal = useIndexModal(); const upgradeModal = useUpgradeModal(); const user = useUserStore((state) => state.user); + const [showImageUrlModal, setShowImageUrlModal] = useState(false); + const handleImagesAdded = (images: WebImageFile[]) => { + try { + setUploadedFiles((prev) => [...prev, ...images]); + const newPreviewFiles: FileWithPreview[] = images.map( + (image) => + ({ + name: image.name, + type: 'image/jpeg', + preview: image.url, + }) as FileWithPreview, + ); + setFiles((prev) => [...prev, ...newPreviewFiles]); + } catch (error) { + toast.error('Failed to add images'); + } + }; + const checkEmptyInput = () => { if (isUploading) { return true; @@ -76,7 +92,6 @@ const SearchBar: React.FC = ({ return; } if (uploadedFiles && uploadedFiles.length > 0) { - console.log('uploadedFiles', uploadedFiles); const fileUrls = uploadedFiles.map((file) => file.url); handleSearch(content, fileUrls); setFiles([]); @@ -148,7 +163,6 @@ const SearchBar: React.FC = ({ } const processedImageFiles = await processImageFiles(acceptedFiles); - const newFiles = processedImageFiles.map( (file) => Object.assign(file, { @@ -214,7 +228,7 @@ const SearchBar: React.FC = ({ {files.map((file, index) => (
{file.type.startsWith('image/') ? ( - {file.name} = ({ >
- {/* {showIndexButton && ( - - - - - {t('index-tip')} - - )} */} -
@@ -305,6 +295,25 @@ const SearchBar: React.FC = ({ {searchType === SearchType.SEARCH ? t('attach-tip') : t('image-tip')} + + +
+ +
+ + Attach Web Image +
@@ -326,6 +335,8 @@ const SearchBar: React.FC = ({
+ +
{showShadcnUI && (
diff --git a/frontend/components/search/search-message.tsx b/frontend/components/search/search-message.tsx index c813dff0..75a0aa43 100644 --- a/frontend/components/search/search-message.tsx +++ b/frontend/components/search/search-message.tsx @@ -7,7 +7,6 @@ import React, { memo, useMemo } from 'react'; import AnswerSection from '@/components/search/answer-section'; import QuestionSection from '@/components/search/question-section'; import ActionButtons from '@/components/search/action-buttons'; -import { extractAllImageUrls } from '@/lib/shared-utils'; import VideoGallery from '@/components/search/video-gallery'; import ExpandableSection from '@/components/search/expandable-section'; import MindMap from '@/components/search/mindmap'; @@ -36,17 +35,18 @@ const SearchMessage = memo( const isUser = role === 'user'; const message = props.message; + const attachments = message.attachments; - const attachments = useMemo(() => { - let initialAttachments = message.attachments ?? []; - if (isUser) { - const imageUrls = extractAllImageUrls(content); - if (imageUrls.length > 0) { - initialAttachments = initialAttachments.concat(imageUrls); - } - } - return initialAttachments; - }, [message.attachments, isUser, content]); + // const attachments = useMemo(() => { + // let initialAttachments = message.attachments ?? []; + // if (isUser) { + // const imageUrls = extractAllImageUrls(content); + // if (imageUrls.length > 0) { + // initialAttachments = initialAttachments.concat(imageUrls); + // } + // } + // return initialAttachments; + // }, [message.attachments, isUser, content]); const t = useTranslations('SearchMessage'); const { showMindMap } = useUIStore(); diff --git a/frontend/components/search/search-window.tsx b/frontend/components/search/search-window.tsx index c819d04f..11c377f4 100644 --- a/frontend/components/search/search-window.tsx +++ b/frontend/components/search/search-window.tsx @@ -12,7 +12,7 @@ import { ImageSource, Message, SearchType, TextSource, User, VideoSource } from import { LoaderCircle } from 'lucide-react'; import { useScrollAnchor } from '@/hooks/use-scroll-anchor'; import { toast } from 'sonner'; -import { isProUser, extractAllImageUrls, generateId } from '@/lib/shared-utils'; +import { isProUser, generateId } from '@/lib/shared-utils'; import { useUpgradeModal } from '@/hooks/use-upgrade-modal'; import { useSearchStore } from '@/lib/store/local-history'; import { ButtonScrollToBottom } from '@/components/button-scroll-to-bottom'; @@ -110,16 +110,16 @@ export default function SearchWindow({ id, initialMessages, user, isReadOnly = f messageValue = 'Please generate the same UI as the image'; } - const imageUrls = extractAllImageUrls(messageValue); - if (imageUrls.length > 1 && user && !isProUser(user)) { - toast.error(t('multi-image-free-limit')); - upgradeModal.onOpen(); - return; - } - if (imageUrls.length > 5) { - toast.error(t('multi-image-pro-limit')); - return; - } + // const imageUrls = extractAllImageUrls(messageValue); + // if (imageUrls.length > 1 && user && !isProUser(user)) { + // toast.error(t('multi-image-free-limit')); + // upgradeModal.onOpen(); + // return; + // } + // if (imageUrls.length > 5) { + // toast.error(t('multi-image-pro-limit')); + // return; + // } setInput(''); setIsLoading(true); diff --git a/frontend/hooks/use-upload-file.ts b/frontend/hooks/use-upload-file.ts index 48fddae1..c52dd7d5 100644 --- a/frontend/hooks/use-upload-file.ts +++ b/frontend/hooks/use-upload-file.ts @@ -63,7 +63,7 @@ async function uploadSingleFile(file: File): Promise { } export function useUploadFile() { - const [uploadedFiles, setUploadedFiles] = React.useState(); + const [uploadedFiles, setUploadedFiles] = React.useState([]); const [isUploading, setIsUploading] = React.useState(false); const { compressImage, compressionError } = useImageCompression({}); diff --git a/frontend/lib/llm/llm.ts b/frontend/lib/llm/llm.ts index 0ce2207e..7c5a5c34 100644 --- a/frontend/lib/llm/llm.ts +++ b/frontend/lib/llm/llm.ts @@ -72,12 +72,12 @@ export function convertToCoreMessages(messages: Message[]): CoreMessage[] { export function createUserMessages(query: string, attachments: string[] = []) { let text = query; - if (attachments.length === 0) { - attachments = extractAllImageUrls(query); - if (attachments.length > 0) { - text = replaceImageUrl(query, attachments); - } - } + // if (attachments.length === 0) { + // attachments = extractAllImageUrls(query); + // if (attachments.length > 0) { + // text = replaceImageUrl(query, attachments); + // } + // } return { role: 'user', content: [{ type: 'text', text: text }, ...attachmentsToParts(attachments)], diff --git a/frontend/lib/shared-utils.ts b/frontend/lib/shared-utils.ts index 66f46a0a..3a7699e9 100644 --- a/frontend/lib/shared-utils.ts +++ b/frontend/lib/shared-utils.ts @@ -24,6 +24,24 @@ export function isValidUrl(input: string): boolean { } } +export function isValidImageUrl(url: string): boolean { + if (!url || typeof url !== 'string') { + return false; + } + + try { + new URL(url); + if (!url.startsWith('http://') && !url.startsWith('https://')) { + return false; + } + + const imageExtensionRegex = /\.(jpg|jpeg|png|gif|bmp|webp)$/i; + return imageExtensionRegex.test(url); + } catch (error) { + return false; + } +} + const ONE_DAY_MS = 24 * 60 * 60 * 1000; function isSubscriptionActive(user: any): boolean { diff --git a/frontend/lib/tools/auto.ts b/frontend/lib/tools/auto.ts index 25cb4f96..9caac7c6 100644 --- a/frontend/lib/tools/auto.ts +++ b/frontend/lib/tools/auto.ts @@ -81,9 +81,6 @@ export async function autoAnswer( }, }), }, - onFinish({ finishReason, usage }) { - console.log('auto answer finish', { finishReason, usage }); - }, }); let titlePromise; diff --git a/frontend/messages/en.json b/frontend/messages/en.json index a2de285d..68f5b795 100644 --- a/frontend/messages/en.json +++ b/frontend/messages/en.json @@ -70,7 +70,7 @@ "index-button": "Index", "attach-button": "Attach", "index-tip": "Index Web Pages and Local Files", - "attach-tip": "Attach Image and File to Ask", + "attach-tip": "Attach Local Image and File to Ask", "image-tip": "Generate UI Page from Image", "search-tip": "Enter to send, Shift + Enter to wrap" }, diff --git a/frontend/messages/zh.json b/frontend/messages/zh.json index 1d1b29c6..979a6c03 100644 --- a/frontend/messages/zh.json +++ b/frontend/messages/zh.json @@ -70,7 +70,7 @@ "index-button": "索引", "attach-button": "附加", "index-tip": "索引网页和本地文件", - "attach-tip": "附加图像和文件进行提问", + "attach-tip": "附加本地图像和文件进行提问", "image-tip": "根据图像生成 UI 页面", "search-tip": "按 Enter 发送，Shift + Enter 换行" },