ホーム エクスプローラ ヘルプ
サインイン
lisihan
/
Create-knowledge-find-decode
1
0
フォーク 0
ファイル 課題 0 プルリクエスト 0 Wiki
ソースを参照

perf: slim query detail responses

SamLee 2 週間 前
親
44200efaaf
コミット
a061ee1669
11 ファイル変更、132 行追加、27 行削除
分割表示 差分情報を表示
  1. 28 2
      acquisition/repositories/postgres.py
  2. 2 0
      app/api.py
  3. 8 0
      app/frontend/dist/assets/index-BsOtccz2.js
  4. 0 8
      app/frontend/dist/assets/index-CxkG5OsZ.js
  5. 1 1
      app/frontend/dist/index.html
  6. 24 3
      app/frontend/src/features/query-board/QueryBoardPage.jsx
  7. 4 5
      app/frontend/src/features/query-board/model.js
  8. 12 2
      app/frontend/src/features/query-board/useQueryBoard.js
  9. 27 4
      app/routes/query_generation.py
  10. 24 2
      tests/test_app_api.py
  11. 2 0
      tests/test_postgres_repository_contract.py

+ 28 - 2
acquisition/repositories/postgres.py
ファイルの表示 

@@ -733,7 +733,21 @@ class PostgresAcquisitionRepository:
 
             }
 
         jobs = self._all(
 
             """
 
-            SELECT aj.* FROM acquisition_jobs aj
 
+            SELECT
 
+                aj.id,
 
+                aj.run_id,
 
+                aj.query_id,
 
+                aj.platform,
 
+                aj.status,
 
+                aj.attempt_count,
 
+                aj.display_limit,
 
+                aj.search_limit,
 
+                aj.error_message,
 
+                aj.started_at,
 
+                aj.finished_at,
 
+                aj.created_at,
 
+                aj.updated_at
 
+            FROM acquisition_jobs aj
 
             WHERE aj.query_id = %s
 
             ORDER BY aj.created_at, aj.platform
 
             """,
 
@@ -750,7 +764,19 @@ class PostgresAcquisitionRepository:
 
                 LEFT(ci.raw_summary, 700) AS raw_summary,
 
                 ci.status,
 
                 ci.content_mode,
 
-                ci.metadata,
 
+                jsonb_strip_nulls(jsonb_build_object(
 
+                    'page_index', ci.metadata -> 'page_index',
 
+                    'page_rank', ci.metadata -> 'page_rank',
 
+                    'source_cursor', ci.metadata -> 'source_cursor',
 
+                    'content_mode', ci.metadata -> 'content_mode',
 
+                    'search_provider', ci.metadata -> 'search_provider',
 
+                    'detail_provider', ci.metadata -> 'detail_provider',
 
+                    'acquisition_match_status', ci.metadata -> 'acquisition_match_status',
 
+                    'matched_unique_key', ci.metadata -> 'matched_unique_key',
 
+                    'skip_reason', ci.metadata -> 'skip_reason',
 
+                    'unsupported_raw_type', ci.metadata -> 'unsupported_raw_type',
 
+                    'video_url_missing', ci.metadata -> 'video_url_missing'
 
+                )) AS metadata,
 
                 ci.created_at,
 
                 ci.updated_at
 
             FROM candidate_items ci

+ 2 - 0
app/api.py
ファイルの表示 

@@ -6,6 +6,7 @@ from pathlib import Path
 
 
 from fastapi import FastAPI
 
 from fastapi.middleware.cors import CORSMiddleware
 
+from fastapi.middleware.gzip import GZipMiddleware
 
 from fastapi.staticfiles import StaticFiles
 
 
 from app.routes import acquisition, decode, payloads, query_generation, runs
 
@@ -35,6 +36,7 @@ def create_app() -> FastAPI:
 
         allow_methods=["GET", "POST", "PATCH", "OPTIONS"],
 
         allow_headers=["Authorization", "Content-Type"],
 
     )
 
+    app.add_middleware(GZipMiddleware, minimum_size=1024)
 
     app.include_router(acquisition.router)
 
     app.include_router(decode.router)
 
     app.include_router(payloads.router)

ファイルの差分が大きいため隠しています
+ 8 - 0
app/frontend/dist/assets/index-BsOtccz2.js


ファイルの差分が大きいため隠しています
+ 0 - 8
app/frontend/dist/assets/index-CxkG5OsZ.js


+ 1 - 1
app/frontend/dist/index.html
ファイルの表示 

@@ -4,7 +4,7 @@
 
     <meta charset="UTF-8" />
 
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
 
     <title>创作知识</title>
 
-    <script type="module" crossorigin src="/app/assets/index-CxkG5OsZ.js"></script>
 
+    <script type="module" crossorigin src="/app/assets/index-BsOtccz2.js"></script>
 
     <link rel="stylesheet" crossorigin href="/app/assets/index-C-saPy1c.css">
 
   </head>
 
   <body>

+ 24 - 3
app/frontend/src/features/query-board/QueryBoardPage.jsx
ファイルの表示 

@@ -8,6 +8,20 @@ import SearchResultColumn from './SearchResultColumn.jsx'
 
 import { useLatestQueryDetail, useQueryBoard } from './useQueryBoard.js'
 
 import { isQuerySearched } from './model.js'
 
 
+function selectedQueryPayload(queryText, latest) {
 
+  if (!latest?.query_id) return null
 
+  return {
 
+    query: queryText,
 
+    query_id: latest.query_id,
 
+    detail_version: [
 
+      latest.candidate_count || 0,
 
+      latest.creation_hit_count || 0,
 
+      latest.decoded_count || 0,
 
+      latest.payload_count || 0,
 
+    ].join(':'),
 
+  }
 
+}
 
+
 
 export default function QueryBoardPage() {
 
   const {
 
     preview,
 
@@ -43,11 +57,18 @@ export default function QueryBoardPage() {
 
 
   useEffect(() => {
 
     if (!familyItems.length) return
 
-    if (selectedQuery && familyItems.some((item) => item.query === selectedQuery.query)) return
 
+    if (selectedQuery && familyItems.some((item) => item.query === selectedQuery.query)) {
 
+      const latest = latestByQuery.get(selectedQuery.query)
 
+      const next = selectedQueryPayload(selectedQuery.query, latest)
 
+      if (next && (next.query_id !== selectedQuery.query_id || next.detail_version !== selectedQuery.detail_version)) {
 
+        setSelectedQuery(next)
 
+      }
 
+      return
 
+    }
 
     const first = familyItems.find((item) => isQuerySearched(latestByQuery.get(item.query)))
 
     if (first) {
 
       const latest = latestByQuery.get(first.query)
 
-      setSelectedQuery({ query: first.query, query_id: latest.query_id })
 
+      setSelectedQuery(selectedQueryPayload(first.query, latest))
 
     } else {
 
       setSelectedQuery(null)
 
     }
 
@@ -96,7 +117,7 @@ export default function QueryBoardPage() {
 
           latestByQuery={latestByQuery}
 
           selectedQuery={selectedQuery}
 
           onSelect={(queryItem, queryLatest) => {
 
-            setSelectedQuery({ query: queryItem.query, query_id: queryLatest.query_id })
 
+            setSelectedQuery(selectedQueryPayload(queryItem.query, queryLatest))
 
           }}
 
           familyRunCount={familyRunCount}
 
           totalRunCount={totalRunCount}

+ 4 - 5
app/frontend/src/features/query-board/model.js
ファイルの表示 

@@ -113,14 +113,13 @@ export function singletonMaps(singleton) {
 
 }
 
 
 export function flatDetailItems(detail) {
 
-  const platforms = detail?.platforms || {}
 
-  return ['xiaohongshu', 'weixin', 'douyin'].flatMap((platform) => {
 
-    const [label, tone] = PLATFORM_LABEL[platform] || [platform, platform]
 
-    return (platforms[platform]?.items || []).map((item) => ({
 
+  return (detail?.items || []).map((item) => {
 
+    const [label, tone] = PLATFORM_LABEL[item.platform] || [item.platform, item.platform]
 
+    return {
 
       ...item,
 
       platform_label: label,
 
       platform_tone: tone,
 
-    }))
 
+    }
 
   })
 
 }

+ 12 - 2
app/frontend/src/features/query-board/useQueryBoard.js
ファイルの表示 

@@ -1,4 +1,4 @@
 
-import { useCallback, useEffect, useMemo, useState } from 'react'
 
+import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
 
 import { getLatestBoard, getLatestQueryDetail, getQueryPreview } from '../../api/workbench.js'
 
 import { isLiveRun, singletonMaps } from './model.js'
 
 
@@ -78,13 +78,23 @@ export function useLatestQueryDetail(selectedQuery) {
 
   const [detail, setDetail] = useState(null)
 
   const [loading, setLoading] = useState(false)
 
   const [error, setError] = useState('')
 
+  const cacheRef = useRef(new Map())
 
 
   const load = useCallback(({ signal, background = false } = {}) => {
 
     if (!selectedQuery?.query_id) return Promise.resolve(null)
 
+    const cacheKey = `${selectedQuery.query_id}:${selectedQuery.detail_version || ''}`
 
+    const cached = cacheRef.current.get(cacheKey)
 
+    if (cached && !background) {
 
+      setDetail(cached)
 
+      setLoading(false)
 
+      setError('')
 
+      return Promise.resolve(cached)
 
+    }
 
     if (!background) setLoading(true)
 
     setError('')
 
     return getLatestQueryDetail(selectedQuery.query_id, { signal })
 
       .then((payload) => {
 
+        cacheRef.current.set(cacheKey, payload)
 
         setDetail(payload)
 
         return payload
 
       })
 
@@ -95,7 +105,7 @@ export function useLatestQueryDetail(selectedQuery) {
 
       .finally(() => {
 
         if (!background) setLoading(false)
 
       })
 
-  }, [selectedQuery?.query_id])
 
+  }, [selectedQuery?.query_id, selectedQuery?.detail_version])
 
 
   useEffect(() => {
 
     setDetail(null)

+ 27 - 4
app/routes/query_generation.py
ファイルの表示 

@@ -30,6 +30,27 @@ def _summary(generated: dict[str, Any]) -> dict[str, Any]:
 
     }
 
 
 
+LIGHT_ITEM_METADATA_KEYS = {
 
+    "page_index",
 
+    "page_rank",
 
+    "source_cursor",
 
+    "content_mode",
 
+    "search_provider",
 
+    "detail_provider",
 
+    "acquisition_match_status",
 
+    "matched_unique_key",
 
+    "skip_reason",
 
+    "unsupported_raw_type",
 
+    "video_url_missing",
 
+}
 
+
 
+
 
+def _light_metadata(metadata: dict[str, Any] | None) -> dict[str, Any]:
 
+    if not isinstance(metadata, dict):
 
+        return {}
 
+    return {key: metadata[key] for key in LIGHT_ITEM_METADATA_KEYS if key in metadata}
 
+
 
+
 
 def _normalize_light_query_detail(detail: dict[str, Any]) -> dict[str, Any]:
 
     media_by_item = {media["item_id"]: media for media in detail.get("media_assets") or []}
 
     classification_by_item = {row["item_id"]: row for row in detail.get("classifications") or []}
 
@@ -49,7 +70,8 @@ def _normalize_light_query_detail(detail: dict[str, Any]) -> dict[str, Any]:
 
                 "display_limit": job.get("display_limit"),
 
                 "search_limit": job.get("search_limit"),
 
                 "error_message": job.get("error_message"),
 
-                "items": [],
 
+                "item_ids": [],
 
+                "item_count": 0,
 
             },
 
         )
 
 
@@ -65,7 +87,7 @@ def _normalize_light_query_detail(detail: dict[str, Any]) -> dict[str, Any]:
 
             "raw_summary": item.get("raw_summary"),
 
             "status": item.get("status"),
 
             "content_mode": item.get("content_mode"),
 
-            "metadata": item.get("metadata") or {},
 
+            "metadata": _light_metadata(item.get("metadata")),
 
             "classification": _model_dump(classification) if classification else None,
 
             "decode_summary": _model_dump(decode_by_item[item_id]) if item_id in decode_by_item else None,
 
             "media_assets": [],
 
@@ -76,11 +98,12 @@ def _normalize_light_query_detail(detail: dict[str, Any]) -> dict[str, Any]:
 
         platform = payload["platform"]
 
         group = platforms.setdefault(
 
             platform,
 
-            {"platform": platform, "status": "done", "items": []},
 
+            {"platform": platform, "status": "done", "item_ids": [], "item_count": 0},
 
         )
 
         if group.get("status") in {None, "pending"}:
 
             group["status"] = "done"
 
-        group["items"].append(payload)
 
+        group.setdefault("item_ids", []).append(item_id)
 
+        group["item_count"] = len(group["item_ids"])
 
 
     return {
 
         "query": QuerySchema.model_validate(detail["query"]).model_dump(mode="json"),

+ 24 - 2
tests/test_app_api.py
ファイルの表示 

@@ -161,7 +161,11 @@ class FakeAcquisitionRepo:
 
                     "raw_summary": "先定受众",
 
                     "status": "candidate",
 
                     "content_mode": "image_post",
 
-                    "metadata": {},
 
+                    "metadata": {
 
+                        "page_index": 1,
 
+                        "matched_candidate": {"raw": "large"},
 
+                        "source_payload": {"raw": "large"},
 
+                    },
 
                 }
 
             ],
 
             "media_assets": [
 
@@ -313,12 +317,30 @@ def test_app_health_and_formal_acquisition_routes():
 
         assert item["classification"]["is_creation_knowledge"] is True
 
 
         latest = client.get(f"/api/query-generation/latest/queries/{repo.query_id}").json()
 
-        lightweight_item = latest["platforms"]["xiaohongshu"]["items"][0]
 
+        lightweight_item = latest["items"][0]
 
         assert "body_text" not in lightweight_item
 
         assert "source_payload" not in lightweight_item
 
+        assert "matched_candidate" not in lightweight_item["metadata"]
 
+        assert "source_payload" not in lightweight_item["metadata"]
 
         assert lightweight_item["raw_summary"] == "先定受众"
 
         assert len(lightweight_item["media_assets"]) == 1
 
         assert lightweight_item["decode_summary"]["particle_count"] == 1
 
+        assert latest["platforms"]["xiaohongshu"]["item_ids"] == [str(repo.item_id)]
 
+        assert "items" not in latest["platforms"]["xiaohongshu"]
 
+    finally:
 
+        app.dependency_overrides.clear()
 
+
 
+
 
+def test_app_compresses_large_json_responses():
 
+    repo = FakeAcquisitionRepo()
 
+    client = _client(repo)
 
+    try:
 
+        response = client.get(
 
+            f"/api/query-generation/latest/queries/{repo.query_id}",
 
+            headers={"Accept-Encoding": "gzip"},
 
+        )
 
+        assert response.status_code == 200
 
+        assert response.headers.get("content-encoding") == "gzip"
 
     finally:
 
         app.dependency_overrides.clear()

+ 2 - 0
tests/test_postgres_repository_contract.py
ファイルの表示 

@@ -282,6 +282,8 @@ def test_postgres_latest_query_result_list_uses_lightweight_projection():
 
     assert "source_payload" not in item_sql
 
     assert "body_text" not in item_sql
 
     assert "LEFT(ci.raw_summary, 700) AS raw_summary" in item_sql
 
+    assert "jsonb_build_object" in item_sql
 
+    assert "matched_candidate" not in item_sql
 
     assert "SELECT DISTINCT ON (item_id)" in media_sql
 
     assert "media_type IN ('cover', 'image', 'frame')" in media_sql
 
     assert "SELECT DISTINCT ON (item_id)" in classification_sql

この差分においてかなりの量のファイルが変更されているため、一部のファイルを表示していません