Trang chủ Khám phá Trợ giúp
Đăng nhập
lisihan
/
Create-knowledge-find-decode
1
0
Fork 0
Các tập tin Các vấn đề 0 Yêu cầu khéo về 0 Wiki
Browse Source

perf: slim query detail responses

SamLee 2 tuần trước cách đây
mục cha
44200efaaf
commit
a061ee1669
11 tập tin đã thay đổi với 132 bổ sung và 27 xóa
View chưa được định nghĩa Hiển thị tình trạng sai khác
  1. 28 2
      acquisition/repositories/postgres.py
  2. 2 0
      app/api.py
  3. 8 0
      app/frontend/dist/assets/index-BsOtccz2.js
  4. 0 8
      app/frontend/dist/assets/index-CxkG5OsZ.js
  5. 1 1
      app/frontend/dist/index.html
  6. 24 3
      app/frontend/src/features/query-board/QueryBoardPage.jsx
  7. 4 5
      app/frontend/src/features/query-board/model.js
  8. 12 2
      app/frontend/src/features/query-board/useQueryBoard.js
  9. 27 4
      app/routes/query_generation.py
  10. 24 2
      tests/test_app_api.py
  11. 2 0
      tests/test_postgres_repository_contract.py

+ 28 - 2
acquisition/repositories/postgres.py
Xem Tập Tin 

@@ -733,7 +733,21 @@ class PostgresAcquisitionRepository:
 
             }
 
             }
 
         jobs = self._all(
 
         jobs = self._all(
 
             """
 
             """
 
-            SELECT aj.* FROM acquisition_jobs aj
 
+            SELECT
 
+                aj.id,
 
+                aj.run_id,
 
+                aj.query_id,
 
+                aj.platform,
 
+                aj.status,
 
+                aj.attempt_count,
 
+                aj.display_limit,
 
+                aj.search_limit,
 
+                aj.error_message,
 
+                aj.started_at,
 
+                aj.finished_at,
 
+                aj.created_at,
 
+                aj.updated_at
 
+            FROM acquisition_jobs aj
 
             WHERE aj.query_id = %s
 
             WHERE aj.query_id = %s
 
             ORDER BY aj.created_at, aj.platform
 
             ORDER BY aj.created_at, aj.platform
 
             """,
 
             """,
 
@@ -750,7 +764,19 @@ class PostgresAcquisitionRepository:
 
                 LEFT(ci.raw_summary, 700) AS raw_summary,
 
                 LEFT(ci.raw_summary, 700) AS raw_summary,
 
                 ci.status,
 
                 ci.status,
 
                 ci.content_mode,
 
                 ci.content_mode,
 
-                ci.metadata,
 
+                jsonb_strip_nulls(jsonb_build_object(
 
+                    'page_index', ci.metadata -> 'page_index',
 
+                    'page_rank', ci.metadata -> 'page_rank',
 
+                    'source_cursor', ci.metadata -> 'source_cursor',
 
+                    'content_mode', ci.metadata -> 'content_mode',
 
+                    'search_provider', ci.metadata -> 'search_provider',
 
+                    'detail_provider', ci.metadata -> 'detail_provider',
 
+                    'acquisition_match_status', ci.metadata -> 'acquisition_match_status',
 
+                    'matched_unique_key', ci.metadata -> 'matched_unique_key',
 
+                    'skip_reason', ci.metadata -> 'skip_reason',
 
+                    'unsupported_raw_type', ci.metadata -> 'unsupported_raw_type',
 
+                    'video_url_missing', ci.metadata -> 'video_url_missing'
 
+                )) AS metadata,
 
                 ci.created_at,
 
                 ci.created_at,
 
                 ci.updated_at
 
                 ci.updated_at
 
             FROM candidate_items ci
 
             FROM candidate_items ci

+ 2 - 0
app/api.py
Xem Tập Tin 

@@ -6,6 +6,7 @@ from pathlib import Path
 
 
 
 from fastapi import FastAPI
 
 from fastapi import FastAPI
 
 from fastapi.middleware.cors import CORSMiddleware
 
 from fastapi.middleware.cors import CORSMiddleware
 
+from fastapi.middleware.gzip import GZipMiddleware
 
 from fastapi.staticfiles import StaticFiles
 
 from fastapi.staticfiles import StaticFiles
 
 
 
 from app.routes import acquisition, decode, payloads, query_generation, runs
 
 from app.routes import acquisition, decode, payloads, query_generation, runs
 
@@ -35,6 +36,7 @@ def create_app() -> FastAPI:
 
         allow_methods=["GET", "POST", "PATCH", "OPTIONS"],
 
         allow_methods=["GET", "POST", "PATCH", "OPTIONS"],
 
         allow_headers=["Authorization", "Content-Type"],
 
         allow_headers=["Authorization", "Content-Type"],
 
     )
 
     )
 
+    app.add_middleware(GZipMiddleware, minimum_size=1024)
 
     app.include_router(acquisition.router)
 
     app.include_router(acquisition.router)
 
     app.include_router(decode.router)
 
     app.include_router(decode.router)
 
     app.include_router(payloads.router)
 
     app.include_router(payloads.router)

Những thai đổi đã bị hủy bỏ vì nó quá lớn
+ 8 - 0
app/frontend/dist/assets/index-BsOtccz2.js


Những thai đổi đã bị hủy bỏ vì nó quá lớn
+ 0 - 8
app/frontend/dist/assets/index-CxkG5OsZ.js


+ 1 - 1
app/frontend/dist/index.html
Xem Tập Tin 

@@ -4,7 +4,7 @@
 
     <meta charset="UTF-8" />
 
     <meta charset="UTF-8" />
 
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
 
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
 
     <title>创作知识</title>
 
     <title>创作知识</title>
 
-    <script type="module" crossorigin src="/app/assets/index-CxkG5OsZ.js"></script>
 
+    <script type="module" crossorigin src="/app/assets/index-BsOtccz2.js"></script>
 
     <link rel="stylesheet" crossorigin href="/app/assets/index-C-saPy1c.css">
 
     <link rel="stylesheet" crossorigin href="/app/assets/index-C-saPy1c.css">
 
   </head>
 
   </head>
 
   <body>
 
   <body>

+ 24 - 3
app/frontend/src/features/query-board/QueryBoardPage.jsx
Xem Tập Tin 

@@ -8,6 +8,20 @@ import SearchResultColumn from './SearchResultColumn.jsx'
 
 import { useLatestQueryDetail, useQueryBoard } from './useQueryBoard.js'
 
 import { useLatestQueryDetail, useQueryBoard } from './useQueryBoard.js'
 
 import { isQuerySearched } from './model.js'
 
 import { isQuerySearched } from './model.js'
 
 
 
+function selectedQueryPayload(queryText, latest) {
 
+  if (!latest?.query_id) return null
 
+  return {
 
+    query: queryText,
 
+    query_id: latest.query_id,
 
+    detail_version: [
 
+      latest.candidate_count || 0,
 
+      latest.creation_hit_count || 0,
 
+      latest.decoded_count || 0,
 
+      latest.payload_count || 0,
 
+    ].join(':'),
 
+  }
 
+}
 
+
 
 export default function QueryBoardPage() {
 
 export default function QueryBoardPage() {
 
   const {
 
   const {
 
     preview,
 
     preview,
 
@@ -43,11 +57,18 @@ export default function QueryBoardPage() {
 
 
 
   useEffect(() => {
 
   useEffect(() => {
 
     if (!familyItems.length) return
 
     if (!familyItems.length) return
 
-    if (selectedQuery && familyItems.some((item) => item.query === selectedQuery.query)) return
 
+    if (selectedQuery && familyItems.some((item) => item.query === selectedQuery.query)) {
 
+      const latest = latestByQuery.get(selectedQuery.query)
 
+      const next = selectedQueryPayload(selectedQuery.query, latest)
 
+      if (next && (next.query_id !== selectedQuery.query_id || next.detail_version !== selectedQuery.detail_version)) {
 
+        setSelectedQuery(next)
 
+      }
 
+      return
 
+    }
 
     const first = familyItems.find((item) => isQuerySearched(latestByQuery.get(item.query)))
 
     const first = familyItems.find((item) => isQuerySearched(latestByQuery.get(item.query)))
 
     if (first) {
 
     if (first) {
 
       const latest = latestByQuery.get(first.query)
 
       const latest = latestByQuery.get(first.query)
 
-      setSelectedQuery({ query: first.query, query_id: latest.query_id })
 
+      setSelectedQuery(selectedQueryPayload(first.query, latest))
 
     } else {
 
     } else {
 
       setSelectedQuery(null)
 
       setSelectedQuery(null)
 
     }
 
     }
 
@@ -96,7 +117,7 @@ export default function QueryBoardPage() {
 
           latestByQuery={latestByQuery}
 
           latestByQuery={latestByQuery}
 
           selectedQuery={selectedQuery}
 
           selectedQuery={selectedQuery}
 
           onSelect={(queryItem, queryLatest) => {
 
           onSelect={(queryItem, queryLatest) => {
 
-            setSelectedQuery({ query: queryItem.query, query_id: queryLatest.query_id })
 
+            setSelectedQuery(selectedQueryPayload(queryItem.query, queryLatest))
 
           }}
 
           }}
 
           familyRunCount={familyRunCount}
 
           familyRunCount={familyRunCount}
 
           totalRunCount={totalRunCount}
 
           totalRunCount={totalRunCount}

+ 4 - 5
app/frontend/src/features/query-board/model.js
Xem Tập Tin 

@@ -113,14 +113,13 @@ export function singletonMaps(singleton) {
 
 }
 
 }
 
 
 
 export function flatDetailItems(detail) {
 
 export function flatDetailItems(detail) {
 
-  const platforms = detail?.platforms || {}
 
-  return ['xiaohongshu', 'weixin', 'douyin'].flatMap((platform) => {
 
-    const [label, tone] = PLATFORM_LABEL[platform] || [platform, platform]
 
-    return (platforms[platform]?.items || []).map((item) => ({
 
+  return (detail?.items || []).map((item) => {
 
+    const [label, tone] = PLATFORM_LABEL[item.platform] || [item.platform, item.platform]
 
+    return {
 
       ...item,
 
       ...item,
 
       platform_label: label,
 
       platform_label: label,
 
       platform_tone: tone,
 
       platform_tone: tone,
 
-    }))
 
+    }
 
   })
 
   })
 
 }
 
 }

+ 12 - 2
app/frontend/src/features/query-board/useQueryBoard.js
Xem Tập Tin 

@@ -1,4 +1,4 @@
 
-import { useCallback, useEffect, useMemo, useState } from 'react'
 
+import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
 
 import { getLatestBoard, getLatestQueryDetail, getQueryPreview } from '../../api/workbench.js'
 
 import { getLatestBoard, getLatestQueryDetail, getQueryPreview } from '../../api/workbench.js'
 
 import { isLiveRun, singletonMaps } from './model.js'
 
 import { isLiveRun, singletonMaps } from './model.js'
 
 
 
@@ -78,13 +78,23 @@ export function useLatestQueryDetail(selectedQuery) {
 
   const [detail, setDetail] = useState(null)
 
   const [detail, setDetail] = useState(null)
 
   const [loading, setLoading] = useState(false)
 
   const [loading, setLoading] = useState(false)
 
   const [error, setError] = useState('')
 
   const [error, setError] = useState('')
 
+  const cacheRef = useRef(new Map())
 
 
 
   const load = useCallback(({ signal, background = false } = {}) => {
 
   const load = useCallback(({ signal, background = false } = {}) => {
 
     if (!selectedQuery?.query_id) return Promise.resolve(null)
 
     if (!selectedQuery?.query_id) return Promise.resolve(null)
 
+    const cacheKey = `${selectedQuery.query_id}:${selectedQuery.detail_version || ''}`
 
+    const cached = cacheRef.current.get(cacheKey)
 
+    if (cached && !background) {
 
+      setDetail(cached)
 
+      setLoading(false)
 
+      setError('')
 
+      return Promise.resolve(cached)
 
+    }
 
     if (!background) setLoading(true)
 
     if (!background) setLoading(true)
 
     setError('')
 
     setError('')
 
     return getLatestQueryDetail(selectedQuery.query_id, { signal })
 
     return getLatestQueryDetail(selectedQuery.query_id, { signal })
 
       .then((payload) => {
 
       .then((payload) => {
 
+        cacheRef.current.set(cacheKey, payload)
 
         setDetail(payload)
 
         setDetail(payload)
 
         return payload
 
         return payload
 
       })
 
       })
 
@@ -95,7 +105,7 @@ export function useLatestQueryDetail(selectedQuery) {
 
       .finally(() => {
 
       .finally(() => {
 
         if (!background) setLoading(false)
 
         if (!background) setLoading(false)
 
       })
 
       })
 
-  }, [selectedQuery?.query_id])
 
+  }, [selectedQuery?.query_id, selectedQuery?.detail_version])
 
 
 
   useEffect(() => {
 
   useEffect(() => {
 
     setDetail(null)
 
     setDetail(null)

+ 27 - 4
app/routes/query_generation.py
Xem Tập Tin 

@@ -30,6 +30,27 @@ def _summary(generated: dict[str, Any]) -> dict[str, Any]:
 
     }
 
     }
 
 
 
 
 
+LIGHT_ITEM_METADATA_KEYS = {
 
+    "page_index",
 
+    "page_rank",
 
+    "source_cursor",
 
+    "content_mode",
 
+    "search_provider",
 
+    "detail_provider",
 
+    "acquisition_match_status",
 
+    "matched_unique_key",
 
+    "skip_reason",
 
+    "unsupported_raw_type",
 
+    "video_url_missing",
 
+}
 
+
 
+
 
+def _light_metadata(metadata: dict[str, Any] | None) -> dict[str, Any]:
 
+    if not isinstance(metadata, dict):
 
+        return {}
 
+    return {key: metadata[key] for key in LIGHT_ITEM_METADATA_KEYS if key in metadata}
 
+
 
+
 
 def _normalize_light_query_detail(detail: dict[str, Any]) -> dict[str, Any]:
 
 def _normalize_light_query_detail(detail: dict[str, Any]) -> dict[str, Any]:
 
     media_by_item = {media["item_id"]: media for media in detail.get("media_assets") or []}
 
     media_by_item = {media["item_id"]: media for media in detail.get("media_assets") or []}
 
     classification_by_item = {row["item_id"]: row for row in detail.get("classifications") or []}
 
     classification_by_item = {row["item_id"]: row for row in detail.get("classifications") or []}
 
@@ -49,7 +70,8 @@ def _normalize_light_query_detail(detail: dict[str, Any]) -> dict[str, Any]:
 
                 "display_limit": job.get("display_limit"),
 
                 "display_limit": job.get("display_limit"),
 
                 "search_limit": job.get("search_limit"),
 
                 "search_limit": job.get("search_limit"),
 
                 "error_message": job.get("error_message"),
 
                 "error_message": job.get("error_message"),
 
-                "items": [],
 
+                "item_ids": [],
 
+                "item_count": 0,
 
             },
 
             },
 
         )
 
         )
 
 
 
@@ -65,7 +87,7 @@ def _normalize_light_query_detail(detail: dict[str, Any]) -> dict[str, Any]:
 
             "raw_summary": item.get("raw_summary"),
 
             "raw_summary": item.get("raw_summary"),
 
             "status": item.get("status"),
 
             "status": item.get("status"),
 
             "content_mode": item.get("content_mode"),
 
             "content_mode": item.get("content_mode"),
 
-            "metadata": item.get("metadata") or {},
 
+            "metadata": _light_metadata(item.get("metadata")),
 
             "classification": _model_dump(classification) if classification else None,
 
             "classification": _model_dump(classification) if classification else None,
 
             "decode_summary": _model_dump(decode_by_item[item_id]) if item_id in decode_by_item else None,
 
             "decode_summary": _model_dump(decode_by_item[item_id]) if item_id in decode_by_item else None,
 
             "media_assets": [],
 
             "media_assets": [],
 
@@ -76,11 +98,12 @@ def _normalize_light_query_detail(detail: dict[str, Any]) -> dict[str, Any]:
 
         platform = payload["platform"]
 
         platform = payload["platform"]
 
         group = platforms.setdefault(
 
         group = platforms.setdefault(
 
             platform,
 
             platform,
 
-            {"platform": platform, "status": "done", "items": []},
 
+            {"platform": platform, "status": "done", "item_ids": [], "item_count": 0},
 
         )
 
         )
 
         if group.get("status") in {None, "pending"}:
 
         if group.get("status") in {None, "pending"}:
 
             group["status"] = "done"
 
             group["status"] = "done"
 
-        group["items"].append(payload)
 
+        group.setdefault("item_ids", []).append(item_id)
 
+        group["item_count"] = len(group["item_ids"])
 
 
 
     return {
 
     return {
 
         "query": QuerySchema.model_validate(detail["query"]).model_dump(mode="json"),
 
         "query": QuerySchema.model_validate(detail["query"]).model_dump(mode="json"),

+ 24 - 2
tests/test_app_api.py
Xem Tập Tin 

@@ -161,7 +161,11 @@ class FakeAcquisitionRepo:
 
                     "raw_summary": "先定受众",
 
                     "raw_summary": "先定受众",
 
                     "status": "candidate",
 
                     "status": "candidate",
 
                     "content_mode": "image_post",
 
                     "content_mode": "image_post",
 
-                    "metadata": {},
 
+                    "metadata": {
 
+                        "page_index": 1,
 
+                        "matched_candidate": {"raw": "large"},
 
+                        "source_payload": {"raw": "large"},
 
+                    },
 
                 }
 
                 }
 
             ],
 
             ],
 
             "media_assets": [
 
             "media_assets": [
 
@@ -313,12 +317,30 @@ def test_app_health_and_formal_acquisition_routes():
 
         assert item["classification"]["is_creation_knowledge"] is True
 
         assert item["classification"]["is_creation_knowledge"] is True
 
 
 
         latest = client.get(f"/api/query-generation/latest/queries/{repo.query_id}").json()
 
         latest = client.get(f"/api/query-generation/latest/queries/{repo.query_id}").json()
 
-        lightweight_item = latest["platforms"]["xiaohongshu"]["items"][0]
 
+        lightweight_item = latest["items"][0]
 
         assert "body_text" not in lightweight_item
 
         assert "body_text" not in lightweight_item
 
         assert "source_payload" not in lightweight_item
 
         assert "source_payload" not in lightweight_item
 
+        assert "matched_candidate" not in lightweight_item["metadata"]
 
+        assert "source_payload" not in lightweight_item["metadata"]
 
         assert lightweight_item["raw_summary"] == "先定受众"
 
         assert lightweight_item["raw_summary"] == "先定受众"
 
         assert len(lightweight_item["media_assets"]) == 1
 
         assert len(lightweight_item["media_assets"]) == 1
 
         assert lightweight_item["decode_summary"]["particle_count"] == 1
 
         assert lightweight_item["decode_summary"]["particle_count"] == 1
 
+        assert latest["platforms"]["xiaohongshu"]["item_ids"] == [str(repo.item_id)]
 
+        assert "items" not in latest["platforms"]["xiaohongshu"]
 
+    finally:
 
+        app.dependency_overrides.clear()
 
+
 
+
 
+def test_app_compresses_large_json_responses():
 
+    repo = FakeAcquisitionRepo()
 
+    client = _client(repo)
 
+    try:
 
+        response = client.get(
 
+            f"/api/query-generation/latest/queries/{repo.query_id}",
 
+            headers={"Accept-Encoding": "gzip"},
 
+        )
 
+        assert response.status_code == 200
 
+        assert response.headers.get("content-encoding") == "gzip"
 
     finally:
 
     finally:
 
         app.dependency_overrides.clear()
 
         app.dependency_overrides.clear()

+ 2 - 0
tests/test_postgres_repository_contract.py
Xem Tập Tin 

@@ -282,6 +282,8 @@ def test_postgres_latest_query_result_list_uses_lightweight_projection():
 
     assert "source_payload" not in item_sql
 
     assert "source_payload" not in item_sql
 
     assert "body_text" not in item_sql
 
     assert "body_text" not in item_sql
 
     assert "LEFT(ci.raw_summary, 700) AS raw_summary" in item_sql
 
     assert "LEFT(ci.raw_summary, 700) AS raw_summary" in item_sql
 
+    assert "jsonb_build_object" in item_sql
 
+    assert "matched_candidate" not in item_sql
 
     assert "SELECT DISTINCT ON (item_id)" in media_sql
 
     assert "SELECT DISTINCT ON (item_id)" in media_sql
 
     assert "media_type IN ('cover', 'image', 'frame')" in media_sql
 
     assert "media_type IN ('cover', 'image', 'frame')" in media_sql
 
     assert "SELECT DISTINCT ON (item_id)" in classification_sql
 
     assert "SELECT DISTINCT ON (item_id)" in classification_sql

Một số tệp đã không được hiển thị bởi vì quá nhiều tập tin thay đổi trong này khác