Przeglądaj źródła

feat(FlowChart): 为消息节点添加图片预览功能

新增图片提取工具函数和预览模态框,支持从消息内容中提取并显示图片缩略图
在流程图和详情面板中点击缩略图可查看大图预览
移除调试用的 console.log 语句
max_liu 3 dni temu
rodzic
commit
286f3b628e

+ 44 - 10
frontend/react-template/src/components/DetailPanel/DetailPanel.tsx

@@ -1,7 +1,10 @@
 import ReactMarkdown from "react-markdown";
 import ReactMarkdown from "react-markdown";
+import { useState } from "react";
 import type { Goal } from "../../types/goal";
 import type { Goal } from "../../types/goal";
 import type { Edge, Message } from "../../types/message";
 import type { Edge, Message } from "../../types/message";
 import styles from "./DetailPanel.module.css";
 import styles from "./DetailPanel.module.css";
+import { ImagePreviewModal } from "../ImagePreview/ImagePreviewModal";
+import { extractImagesFromMessage } from "../../utils/imageExtraction";
 
 
 interface DetailPanelProps {
 interface DetailPanelProps {
   node: Goal | Message | null;
   node: Goal | Message | null;
@@ -11,6 +14,26 @@ interface DetailPanelProps {
 }
 }
 
 
 export const DetailPanel = ({ node, edge, messages = [], onClose }: DetailPanelProps) => {
 export const DetailPanel = ({ node, edge, messages = [], onClose }: DetailPanelProps) => {
+  const [previewImage, setPreviewImage] = useState<string | null>(null);
+
+  const renderImages = (msg: Message) => {
+    const images = extractImagesFromMessage(msg);
+    if (images.length === 0) return null;
+    return (
+      <div className="grid grid-cols-3 gap-2 mt-2">
+        {images.map((img, idx) => (
+          <img
+            key={idx}
+            src={img.url}
+            alt={img.alt || "Extracted"}
+            className="w-full h-20 object-cover rounded border border-gray-200 cursor-pointer hover:opacity-80 transition-opacity bg-gray-50"
+            onClick={() => setPreviewImage(img.url)}
+          />
+        ))}
+      </div>
+    );
+  };
+
   const title = node ? "节点详情" : edge ? "连线详情" : "详情";
   const title = node ? "节点详情" : edge ? "连线详情" : "详情";
 
 
   const renderMessageContent = (content: Message["content"]) => {
   const renderMessageContent = (content: Message["content"]) => {
@@ -60,9 +83,7 @@ export const DetailPanel = ({ node, edge, messages = [], onClose }: DetailPanelP
             <div className={styles.knowledgeHeader}>
             <div className={styles.knowledgeHeader}>
               <span className={styles.knowledgeId}>{item.id}</span>
               <span className={styles.knowledgeId}>{item.id}</span>
               <div className={styles.knowledgeMetrics}>
               <div className={styles.knowledgeMetrics}>
-                {item.score !== undefined && (
-                  <span className={styles.metricScore}>⭐ {item.score}</span>
-                )}
+                {item.score !== undefined && <span className={styles.metricScore}>⭐ {item.score}</span>}
                 {item.quality_score !== undefined && (
                 {item.quality_score !== undefined && (
                   <span className={styles.metricQuality}>✨ {item.quality_score.toFixed(1)}</span>
                   <span className={styles.metricQuality}>✨ {item.quality_score.toFixed(1)}</span>
                 )}
                 )}
@@ -118,6 +139,10 @@ export const DetailPanel = ({ node, edge, messages = [], onClose }: DetailPanelP
               <div className={styles.label}>ID</div>
               <div className={styles.label}>ID</div>
               <div className={styles.value}>{isMessageNode(node) ? node.message_id || node.id : node.id}</div>
               <div className={styles.value}>{isMessageNode(node) ? node.message_id || node.id : node.id}</div>
             </div>
             </div>
+            <div className={styles.section}>
+              <div className={styles.label}>图片</div>
+              {isMessageNode(node) && renderImages(node)}
+            </div>
 
 
             {isGoal(node) ? (
             {isGoal(node) ? (
               <>
               <>
@@ -162,12 +187,13 @@ export const DetailPanel = ({ node, edge, messages = [], onClose }: DetailPanelP
                     <div className={styles.value}>{node.role}</div>
                     <div className={styles.value}>{node.role}</div>
                   </div>
                   </div>
                 )}
                 )}
-                {node.content && (
-                  <div className={styles.section}>
-                    <div className={styles.label}>内容</div>
-                    <div className={styles.value}>{renderMessageContent(node.content)}</div>
-                  </div>
-                )}
+                <div className={styles.section}>
+                  <div className={styles.label}>内容</div>
+                  {/* <div className={styles.value}>
+                    {node.content && renderMessageContent(node.content)}
+                    {renderImages(node)}
+                  </div> */}
+                </div>
                 {node.goal_id && (
                 {node.goal_id && (
                   <div className={styles.section}>
                   <div className={styles.section}>
                     <div className={styles.label}>所属目标</div>
                     <div className={styles.label}>所属目标</div>
@@ -198,13 +224,21 @@ export const DetailPanel = ({ node, edge, messages = [], onClose }: DetailPanelP
                 </div>
                 </div>
                 <div className={styles.section}>
                 <div className={styles.section}>
                   <div className={styles.label}>内容</div>
                   <div className={styles.label}>内容</div>
-                  <div className={styles.value}>{renderMessageContent(msg.content)}</div>
+                  <div className={styles.value}>
+                    {msg.content && renderMessageContent(msg.content)}
+                    {renderImages(msg)}
+                  </div>
                 </div>
                 </div>
               </div>
               </div>
             ))}
             ))}
           </div>
           </div>
         )}
         )}
       </div>
       </div>
+      <ImagePreviewModal
+        visible={!!previewImage}
+        onClose={() => setPreviewImage(null)}
+        src={previewImage || ""}
+      />
     </aside>
     </aside>
   );
   );
 };
 };

+ 40 - 3
frontend/react-template/src/components/FlowChart/FlowChart.tsx

@@ -16,6 +16,8 @@ import type { Edge as EdgeType, Message } from "../../types/message";
 import { ArrowMarkers } from "./components/ArrowMarkers";
 import { ArrowMarkers } from "./components/ArrowMarkers";
 import styles from "./styles/FlowChart.module.css";
 import styles from "./styles/FlowChart.module.css";
 import { Tooltip } from "@douyinfe/semi-ui";
 import { Tooltip } from "@douyinfe/semi-ui";
+import { ImagePreviewModal } from "../ImagePreview/ImagePreviewModal";
+import { extractImagesFromMessage } from "../../utils/imageExtraction";
 
 
 /**
 /**
  * FlowChart 组件的 Props
  * FlowChart 组件的 Props
@@ -120,6 +122,9 @@ const FlowChartComponent: ForwardRefRenderFunction<FlowChartRef, FlowChartProps>
   // 视图模式状态:scroll (滚动) 或 panzoom (拖拽缩放)
   // 视图模式状态:scroll (滚动) 或 panzoom (拖拽缩放)
   const [viewMode, setViewMode] = useState<"scroll" | "panzoom">("scroll");
   const [viewMode, setViewMode] = useState<"scroll" | "panzoom">("scroll");
 
 
+  // 图片预览状态
+  const [previewImage, setPreviewImage] = useState<string | null>(null);
+
   // 限制缩放比例在允许范围内
   // 限制缩放比例在允许范围内
   const clampZoom = (value: number) => Math.min(zoomRange.max, Math.max(zoomRange.min, value));
   const clampZoom = (value: number) => Math.min(zoomRange.max, Math.max(zoomRange.min, value));
 
 
@@ -523,7 +528,12 @@ const FlowChartComponent: ForwardRefRenderFunction<FlowChartRef, FlowChartProps>
             const currentBranchNodes: LayoutNode[] = [];
             const currentBranchNodes: LayoutNode[] = [];
 
 
             branch.forEach((msg, idx) => {
             branch.forEach((msg, idx) => {
-              const nodeId = `invalid-${msg.id || Math.random()}`;
+              // Ensure we have a stable ID. If msg.id is missing, use a deterministic fallback.
+              // Note: Using Math.random() is bad for React reconciliation, but here we need something unique if ID is missing.
+              // Better fallback: `${parentNode.id}-branch-${idx}`
+              const stableId = msg.id || `${parentNode.id}-branch-${idx}`;
+              const nodeId = `invalid-${stableId}`;
+
               const node: LayoutNode = {
               const node: LayoutNode = {
                 id: nodeId,
                 id: nodeId,
                 x: parentNode.x + X_OFFSET,
                 x: parentNode.x + X_OFFSET,
@@ -989,6 +999,12 @@ const FlowChartComponent: ForwardRefRenderFunction<FlowChartRef, FlowChartProps>
                 const data = node.data as Goal;
                 const data = node.data as Goal;
                 const text = isGoal ? data.description : (node.data as Message).description || "";
                 const text = isGoal ? data.description : (node.data as Message).description || "";
 
 
+                let thumbnail: string | null = null;
+                if (node.type === "message") {
+                  const images = extractImagesFromMessage(node.data as Message);
+                  if (images.length > 0) thumbnail = images[0].url;
+                }
+
                 let textColor = "#3b82f6"; // Blue 500
                 let textColor = "#3b82f6"; // Blue 500
                 if (node.type === "message") {
                 if (node.type === "message") {
                   textColor = "#64748b"; // Slate 500
                   textColor = "#64748b"; // Slate 500
@@ -1034,12 +1050,27 @@ const FlowChartComponent: ForwardRefRenderFunction<FlowChartRef, FlowChartProps>
                     >
                     >
                       <Tooltip content={text}>
                       <Tooltip content={text}>
                         <div
                         <div
-                          className="w-full h-full overflow-hidden flex items-center justify-center"
+                          className="w-full h-full overflow-hidden flex items-center px-2 gap-2"
                           style={{
                           style={{
                             color: textColor,
                             color: textColor,
+                            justifyContent: thumbnail ? "space-between" : "center",
                           }}
                           }}
                         >
                         >
-                          <span className="text-xs line-clamp-3 px-1">{text}</span>
+                          <span className={`text-xs line-clamp-3 ${thumbnail ? "flex-1 text-left" : "text-center"}`}>
+                            {text}
+                          </span>
+                          {thumbnail && (
+                            <img
+                              src={thumbnail}
+                              alt="thumb"
+                              className="w-8 h-8 object-cover rounded border border-gray-200 bg-white flex-shrink-0 hover:scale-110 transition-transform"
+                              loading="lazy"
+                              onClick={(e) => {
+                                e.stopPropagation();
+                                setPreviewImage(thumbnail);
+                              }}
+                            />
+                          )}
                         </div>
                         </div>
                       </Tooltip>
                       </Tooltip>
                     </foreignObject>
                     </foreignObject>
@@ -1086,6 +1117,12 @@ const FlowChartComponent: ForwardRefRenderFunction<FlowChartRef, FlowChartProps>
           {viewMode === "scroll" ? "切换拖拽" : "切换滚动"}
           {viewMode === "scroll" ? "切换拖拽" : "切换滚动"}
         </button>
         </button>
       </div>
       </div>
+
+      <ImagePreviewModal
+        visible={!!previewImage}
+        onClose={() => setPreviewImage(null)}
+        src={previewImage || ""}
+      />
     </div>
     </div>
   );
   );
 };
 };

+ 0 - 1
frontend/react-template/src/components/FlowChart/hooks/useFlowChartData.ts

@@ -149,7 +149,6 @@ export const useFlowChartData = (traceId: string | null, refreshTrigger?: number
 
 
       const messagesRoot = isRecord(messagesJson) ? messagesJson : {};
       const messagesRoot = isRecord(messagesJson) ? messagesJson : {};
       const list = Array.isArray(messagesRoot.messages) ? (messagesRoot.messages as Message[]) : [];
       const list = Array.isArray(messagesRoot.messages) ? (messagesRoot.messages as Message[]) : [];
-      console.log("%c [ list ]-149", "font-size:13px; background:pink; color:#bf2c9f;", list);
 
 
       const filtered = list.filter((message) => (message as { status?: string }).status !== "abandoned");
       const filtered = list.filter((message) => (message as { status?: string }).status !== "abandoned");
       const nextMessages = [...filtered].sort(messageComparator);
       const nextMessages = [...filtered].sort(messageComparator);

+ 49 - 0
frontend/react-template/src/components/ImagePreview/ImagePreviewModal.tsx

@@ -0,0 +1,49 @@
+import type { FC } from "react";
+import { Modal } from "@douyinfe/semi-ui";
+
+interface ImagePreviewModalProps {
+  visible: boolean;
+  onClose: () => void;
+  src: string;
+  alt?: string;
+}
+
+export const ImagePreviewModal: FC<ImagePreviewModalProps> = ({ visible, onClose, src, alt }) => {
+  return (
+    <Modal
+      visible={visible}
+      onCancel={onClose}
+      footer={null}
+      centered
+      maskClosable={true}
+      width={1000}
+      bodyStyle={{
+        padding: 0,
+        display: "flex",
+        justifyContent: "center",
+        alignItems: "center",
+        background: "transparent",
+        height: "auto",
+        maxHeight: "85vh",
+      }}
+      style={{
+        maxWidth: "95vw",
+        background: "transparent",
+        boxShadow: "none",
+      }}
+    >
+      <img
+        src={src}
+        alt={alt || "Preview"}
+        style={{
+          maxWidth: "100%",
+          maxHeight: "80vh",
+          objectFit: "contain",
+          borderRadius: 8,
+          boxShadow: "0 4px 12px rgba(0,0,0,0.3)",
+        }}
+        onClick={(e) => e.stopPropagation()}
+      />
+    </Modal>
+  );
+};

+ 10 - 0
frontend/react-template/src/types/message.ts

@@ -3,10 +3,20 @@ export interface ToolCall {
   name: string;
   name: string;
   arguments: Record<string, unknown>;
   arguments: Record<string, unknown>;
 }
 }
+export interface MsgResult {
+  type?: string;
+  text?: string;
+  image_url?: MsgResultDict;
+}
+export interface MsgResultDict {
+  url: string;
+}
 
 
 export interface MessageContent {
 export interface MessageContent {
   text?: string;
   text?: string;
   tool_calls?: ToolCall[];
   tool_calls?: ToolCall[];
+  tool_name?: string;
+  result?: string | MsgResult[];
 }
 }
 
 
 export interface Message {
 export interface Message {

+ 110 - 0
frontend/react-template/src/utils/__tests__/imageExtraction.test.ts

@@ -0,0 +1,110 @@
+import { describe, it, expect } from "vitest";
+import { extractImagesFromResult, extractImagesFromMessage } from "../imageExtraction";
+import type { Message } from "../../types/message";
+
+describe("extractImagesFromResult", () => {
+  it("should return empty array for null/undefined", () => {
+    expect(extractImagesFromResult(null)).toEqual([]);
+    expect(extractImagesFromResult(undefined)).toEqual([]);
+  });
+
+  it("should extract images from OpenAI-like MsgResult array", () => {
+    const input = [
+      { type: "text", text: "hello" },
+      { type: "image_url", image_url: { url: "http://example.com/1.png" } },
+    ];
+    const result = extractImagesFromResult(input);
+    expect(result).toHaveLength(1);
+    expect(result[0].url).toBe("http://example.com/1.png");
+  });
+
+  it('should extract images from custom type="image" with base64', () => {
+    const input = [
+      {
+        type: "image",
+        source: {
+          media_type: "image/jpeg",
+          data: "base64data",
+        },
+      },
+    ];
+    const result = extractImagesFromResult(input);
+    expect(result).toHaveLength(1);
+    expect(result[0].url).toBe("data:image/jpeg;base64,base64data");
+    expect(result[0].alt).toBe("Base64 Image");
+  });
+
+  it("should extract markdown images from string result", () => {
+    const input =
+      "Here is an image: ![alt text](http://example.com/2.png) and another ![img2](http://example.com/3.png)";
+    const result = extractImagesFromResult(input);
+    expect(result).toHaveLength(2);
+    expect(result[0].url).toBe("http://example.com/2.png");
+    expect(result[0].alt).toBe("alt text");
+    expect(result[1].url).toBe("http://example.com/3.png");
+    expect(result[1].alt).toBe("img2");
+  });
+
+  it("should extract HTML img tags from string result", () => {
+    const input = 'Some text <img src="http://example.com/4.png" alt="html img" /> end';
+    const result = extractImagesFromResult(input);
+    expect(result).toHaveLength(1);
+    expect(result[0].url).toBe("http://example.com/4.png");
+  });
+
+  it("should extract images from JSON-embedded string (read_file output)", () => {
+    const input = `
+    # 01_file.json
+    <file>
+        1| {
+        2|   "image_url": "http://res.cybertogether.net/crawler/image/test.jpeg",
+        3|   "other": "value"
+        4| }
+    `;
+    const result = extractImagesFromResult(input);
+    expect(result).toHaveLength(1);
+    expect(result[0].url).toBe("http://res.cybertogether.net/crawler/image/test.jpeg");
+  });
+
+  it("should extract images from complex escaped JSON string (user case)", () => {
+    // This simulates the string exactly as provided by the user, where the "result" is a string containing file content
+    // Note: The user provided example shows "image_url": " http://..."
+    // We need to be careful about matching the quote and spaces.
+    const input = `# 01_图片分段_07_g3_人物与玫瑰花.json
+
+<file>
+    1| {
+    2|   "image_url": "http://res.cybertogether.net/crawler/image/e70bbea964cfcf0225744da00e8e7939.jpeg",
+    3|   "sections": [
+    4|     {
+    5|       "名称": "人物与玫瑰花",
+`;
+    const result = extractImagesFromResult(input);
+    expect(result).toHaveLength(1);
+    expect(result[0].url).toBe("http://res.cybertogether.net/crawler/image/e70bbea964cfcf0225744da00e8e7939.jpeg");
+  });
+});
+
+describe("extractImagesFromMessage", () => {
+  it("should extract from message with string content", () => {
+    const msg: Message = {
+      content: "![test](http://example.com/test.png)",
+    };
+    const result = extractImagesFromMessage(msg);
+    expect(result).toHaveLength(1);
+    expect(result[0].url).toBe("http://example.com/test.png");
+  });
+
+  it("should extract from message with MessageContent object", () => {
+    const msg: Message = {
+      content: {
+        result: [{ image_url: { url: "http://example.com/obj.png" } }],
+      },
+    };
+    // Need to cast to any because Message type definition might be strict about content structure
+    // but at runtime this is what we expect
+    const result = extractImagesFromMessage(msg);
+    expect(result).toHaveLength(1);
+    expect(result[0].url).toBe("http://example.com/obj.png");
+  });
+});

+ 116 - 0
frontend/react-template/src/utils/imageExtraction.ts

@@ -0,0 +1,116 @@
+import type { Message } from "../types/message";
+
+export interface ExtractedImage {
+  url: string;
+  alt?: string;
+}
+
+/**
+ * Extracts images from a message's content or result.
+ * Handles both JSON array format (MsgResult[]) and Rich Text (Markdown/HTML).
+ *
+ * @param result - The content or result field from a message
+ * @returns Array of extracted images
+ */
+export const extractImagesFromResult = (result: unknown): ExtractedImage[] => {
+  const images: ExtractedImage[] = [];
+
+  if (!result) return images;
+
+  // Case 0: result IS the message content which might be an array directly
+  if (Array.isArray(result)) {
+    result.forEach((item) => {
+      if (typeof item === "object" && item !== null) {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const msgItem = item as any;
+
+        // 1. Check for standard OpenAI-like image_url
+        if (msgItem.image_url && typeof msgItem.image_url === "object" && msgItem.image_url.url) {
+          images.push({
+            url: msgItem.image_url.url,
+            alt: "Attached Image",
+          });
+        }
+
+        // 2. Check for type="image" with source
+        if (msgItem.type === "image" && msgItem.source && typeof msgItem.source === "object") {
+          const source = msgItem.source;
+          if (source.data) {
+            const mimeType = source.media_type || "image/png";
+            images.push({
+              url: `data:${mimeType};base64,${source.data}`,
+              alt: "Base64 Image",
+            });
+          } else if (source.url) {
+            images.push({
+              url: source.url,
+              alt: "Image URL",
+            });
+          }
+        }
+      }
+    });
+  }
+
+  // Case 2: result is a string (Rich Text / Markdown)
+  if (typeof result === "string") {
+    // 1. Match Markdown images: ![alt](url)
+    const markdownRegex = /!\[([^\]]*)\]\(([^)]+)\)/g;
+    let match;
+    while ((match = markdownRegex.exec(result)) !== null) {
+      images.push({
+        alt: match[1] || "Markdown Image",
+        url: match[2],
+      });
+    }
+
+    // 2. Match HTML img tags: <img ... src="..." ...>
+    const htmlRegex = /<img\s+[^>]*src=["']([^"']+)["'][^>]*>/g;
+    while ((match = htmlRegex.exec(result)) !== null) {
+      images.push({
+        alt: "HTML Image",
+        url: match[1],
+      });
+    }
+
+    // 3. Match JSON "image_url": "..." patterns embedded in text
+    // Matches "image_url"\s*:\s*"([^"]+)"
+    const jsonRegex = /"image_url"\s*:\s*"([^"]+)"/g;
+    while ((match = jsonRegex.exec(result)) !== null) {
+      // Basic filtering to avoid matching non-URL strings if the key is reused
+      if (match[1].startsWith("http") || match[1].startsWith("data:")) {
+        images.push({
+          alt: "JSON Image",
+          url: match[1],
+        });
+      }
+    }
+  }
+  return images;
+};
+
+/**
+ * Helper to extract images from a Message object
+ */
+export const extractImagesFromMessage = (message: Message): ExtractedImage[] => {
+  if (!message.content) return [];
+
+  // If content is a string, treat it as result
+  if (typeof message.content === "string") {
+    return extractImagesFromResult(message.content);
+  }
+
+  // If content is an object (MessageContent)
+  if (typeof message.content === "object") {
+    // Check 'result' field
+    if ("result" in message.content && message.content.result) {
+      return extractImagesFromResult(message.content.result);
+    }
+    // Also check if content itself is an array (e.g. standard MessageContent array)
+    if (Array.isArray(message.content)) {
+      return extractImagesFromResult(message.content);
+    }
+  }
+
+  return [];
+};