feat:support showing image

source/lambda/online/common_logic/common_utils/prompt_utils.py

@@ -131,43 +131,38 @@ def prompt_template_render(self, prompt_template: dict):
 You are a customer service agent responding to user queries. ALWAYS adhere to these response rules:
 
 <response_rules>
-1. Image Handling:
-   - If <docs></docs> contains markdown-formatted images, append them unaltered to the end of your response.
-   - Only process markdown-formatted images within <docs></docs>.
-   - IMPORTANT: If no markdown-formatted images are present in <docs></docs>, do not add any image references to your response.
-
-2. Language and Tone:
+1. Language and Tone:
    - Never use phrases like "According to search results," "Hello everyone," "Thank you," or "According to this document..."
    - Provide concise and clear answers.
    - Maintain a professional and helpful tone throughout the response.
 
-3. Relevance:
+2. Relevance:
    - If the query is unrelated to the content in <docs></docs>, respond with: "根据内部知识库，找不到相关内容。"
 
-4. Reference Citation:
+3. Reference Citation:
    - Include the context ID you refer to in your response using the <reference> tag.
    - The context ID should be the index of the document in the <docs> tag.
    - Always use the correct format: \n<reference>X</reference>, where X is the document index.
    - Example: \n<reference>1</reference> for <doc index="1"></doc>
    - All <reference> tags should be in the beginning of the response
    - IMPORTANT: Ensure that the opening tag <reference> always comes before the number, and the closing tag </reference> always comes after the number.
 
-5. Language Adaptation:
+4. Language Adaptation:
    - Respond in the same language as the user's query.
    - If the query is in a language other than English, adapt your response accordingly.
 
-6. Confidentiality:
+5. Confidentiality:
    - Do not disclose any information not present in the provided documents.
    - If asked about topics outside your knowledge base, politely state that you don't have that information.
 
-7. Formatting:
+6. Formatting:
    - Use appropriate formatting (bold, italics, bullet points) to enhance readability when necessary.
 
-8. Completeness:
+7. Completeness:
    - Ensure your response addresses all aspects of the user's query.
    - If multiple relevant documents are provided, synthesize the information coherently.
 
-9. Tag Verification:
+8. Tag Verification:
    - Before finalizing your response, double-check all <reference> tags to ensure they are correctly formatted.
    - If you find any incorrectly formatted tags (e.g., 1</reference>), correct them to the proper format (\n<reference>1</reference>).
 </response_rules>

source/lambda/online/common_logic/common_utils/response_utils.py

@@ -137,12 +137,60 @@ def stream_response(event_body: dict, response: dict):
             }
 
             figure = response.get("extra_response").get("ref_figures", [])
+            # Show at most two figures
             if figure:
                 context_msg["ddb_additional_kwargs"]["figure"] = figure[:2]
+
+            ref_doc = response.get("extra_response").get("ref_docs", [])
+            if ref_doc:
+                md_images = []
+                md_image_list = []
+                for doc in ref_doc:
+                    # Look for markdown image pattern in reference doc: ![alt text](image_path)
+                    doc_content = doc.get("page_content", "")
+                    for line in doc_content.split('\n'):
+                        img_start = line.find("![")
+                        while img_start != -1:
+                            try:
+                                alt_end = line.find("](", img_start)
+                                img_end = line.find(")", alt_end)
+
+                                if alt_end != -1 and img_end != -1:
+                                    image_path = line[alt_end + 2:img_end]
+                                    # Remove optional title if present
+                                    if '"' in image_path or "'" in image_path:
+                                        image_path = image_path.split('"')[0].split("'")[0].strip()
+                                    if image_path:
+                                        have_same_image = False
+                                        for md_image_item in md_image_list:
+                                            if image_path in md_image_item:
+                                                have_same_image = True
+
+                                        md_image_json = {
+                                            "content_type": "md_image",
+                                            "figure_path": image_path
+                                        }
+                                        if not have_same_image and md_image_json not in md_images:
+                                            md_images.append(md_image_json)
+                                            md_image_list.append(image_path)
+                                # Look for next image in the same line
+                                img_start = line.find("![", img_start + 2)
+                            except Exception as e:
+                                logger.error(
+                                    f"Error processing markdown image: {str(e)}, in line: {line}")
+                                # Skip to next image pattern in this line
+                                img_start = line.find("![", img_start + 2)
+                                continue
+
+                if md_images:
+                    context_msg["ddb_additional_kwargs"].setdefault("figure", []).extend(md_images)
+
             send_to_ws_client(
                 message=context_msg,
                 ws_connection_id=ws_connection_id
             )
+            print("lvning context message")
+            print(context_msg)
 
         # send end
         send_to_ws_client(
@@ -188,4 +236,5 @@ def process_response(event_body, response):
     stream = event_body.get("stream", True)
     if stream:
         return stream_response(event_body, response)
+
     return api_response(event_body, response)

source/portal/src/pages/chatbot/ChatBot.tsx

@@ -275,12 +275,22 @@ const ChatBot: React.FC<ChatBotProps> = (props: ChatBotProps) => {
       // handle context message
       if (message.ddb_additional_kwargs?.figure?.length > 0) {
         message.ddb_additional_kwargs.figure.forEach((item) => {
-          setCurrentAIMessage((prev) => {
-            return (
-              prev +
-              ` \n ![${item.content_type}](/${encodeURIComponent(item.figure_path)})`
-            );
-          });
+          if (item.content_type === "md_image") {
+            setCurrentAIMessage((prev) => {
+              return (
+                prev +
+                ` \n ![${item.content_type}](${item.figure_path})`
+              );
+            });
+          } else {
+            setCurrentAIMessage((prev) => {
+              return (
+                prev +
+                ` \n ![${item.content_type}](/${encodeURIComponent(item.figure_path)})`
+              );
+            });
+          }
+
         });
       }
     } else if (message.message_type === 'END') {