|
|
@@ -63,6 +63,68 @@ CREATE TABLE IF NOT EXISTS post_class (
|
|
|
points TEXT, -- 结构化知识点 json(按卡片/分段)
|
|
|
created_at INTEGER NOT NULL
|
|
|
);
|
|
|
+
|
|
|
+-- 430 query 真实采集:run / query×platform job / item / AI 判断
|
|
|
+CREATE TABLE IF NOT EXISTS creation_search_runs (
|
|
|
+ run_id TEXT PRIMARY KEY,
|
|
|
+ status TEXT NOT NULL,
|
|
|
+ total_queries INTEGER NOT NULL DEFAULT 0,
|
|
|
+ note TEXT,
|
|
|
+ started_at INTEGER NOT NULL,
|
|
|
+ finished_at INTEGER
|
|
|
+);
|
|
|
+
|
|
|
+CREATE TABLE IF NOT EXISTS creation_search_jobs (
|
|
|
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
|
+ run_id TEXT NOT NULL,
|
|
|
+ query TEXT NOT NULL,
|
|
|
+ platform TEXT NOT NULL,
|
|
|
+ status TEXT NOT NULL,
|
|
|
+ attempts INTEGER NOT NULL DEFAULT 0,
|
|
|
+ searched_count INTEGER NOT NULL DEFAULT 0,
|
|
|
+ display_count INTEGER NOT NULL DEFAULT 0,
|
|
|
+ error TEXT,
|
|
|
+ started_at INTEGER,
|
|
|
+ finished_at INTEGER,
|
|
|
+ UNIQUE(run_id, query, platform)
|
|
|
+);
|
|
|
+CREATE INDEX IF NOT EXISTS ix_csj_query ON creation_search_jobs(query);
|
|
|
+CREATE INDEX IF NOT EXISTS ix_csj_run_platform ON creation_search_jobs(run_id, platform);
|
|
|
+
|
|
|
+CREATE TABLE IF NOT EXISTS creation_search_items (
|
|
|
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
|
+ run_id TEXT NOT NULL,
|
|
|
+ query TEXT NOT NULL,
|
|
|
+ platform TEXT NOT NULL,
|
|
|
+ rank INTEGER NOT NULL,
|
|
|
+ source_id TEXT,
|
|
|
+ url TEXT,
|
|
|
+ title TEXT,
|
|
|
+ author TEXT,
|
|
|
+ body_text TEXT,
|
|
|
+ cover_url TEXT,
|
|
|
+ image_urls TEXT NOT NULL DEFAULT '[]',
|
|
|
+ video_url TEXT,
|
|
|
+ media_json TEXT NOT NULL DEFAULT '{}',
|
|
|
+ raw_json TEXT NOT NULL DEFAULT '{}',
|
|
|
+ is_displayable INTEGER NOT NULL DEFAULT 0,
|
|
|
+ error TEXT,
|
|
|
+ created_at INTEGER NOT NULL,
|
|
|
+ UNIQUE(run_id, query, platform, rank)
|
|
|
+);
|
|
|
+CREATE INDEX IF NOT EXISTS ix_csi_query ON creation_search_items(query);
|
|
|
+CREATE INDEX IF NOT EXISTS ix_csi_run_platform ON creation_search_items(run_id, platform);
|
|
|
+
|
|
|
+CREATE TABLE IF NOT EXISTS creation_item_classifications (
|
|
|
+ item_id INTEGER PRIMARY KEY,
|
|
|
+ is_creation INTEGER,
|
|
|
+ reason TEXT,
|
|
|
+ knowledge TEXT,
|
|
|
+ prompt_version TEXT,
|
|
|
+ error TEXT,
|
|
|
+ classified_at INTEGER NOT NULL,
|
|
|
+ FOREIGN KEY(item_id) REFERENCES creation_search_items(id) ON DELETE CASCADE
|
|
|
+);
|
|
|
"""
|
|
|
|
|
|
# 旧库迁移:post_class 早期没有 knowledge/points 列,补上(已存在则忽略)
|
|
|
@@ -325,3 +387,290 @@ def list_methods(conn: sqlite3.Connection, table: str = "queries") -> list[str]:
|
|
|
raise ValueError(table)
|
|
|
rows = conn.execute(f"SELECT DISTINCT method FROM {table} ORDER BY method").fetchall()
|
|
|
return [r[0] for r in rows]
|
|
|
+
|
|
|
+
|
|
|
+# ---- 430 query 真实采集 ----------------------------------------------------
|
|
|
+
|
|
|
+CREATION_PLATFORMS = ("xiaohongshu", "weixin", "douyin")
|
|
|
+
|
|
|
+
|
|
|
+def create_creation_run(conn: sqlite3.Connection, run_id: str, *,
|
|
|
+ total_queries: int, note: str = "", ts: Optional[int] = None,
|
|
|
+ status: str = "running") -> None:
|
|
|
+ t = _now(ts)
|
|
|
+ conn.execute(
|
|
|
+ "INSERT OR REPLACE INTO creation_search_runs"
|
|
|
+ "(run_id, status, total_queries, note, started_at, finished_at) VALUES(?,?,?,?,?,NULL)",
|
|
|
+ (run_id, status, int(total_queries), note, t),
|
|
|
+ )
|
|
|
+ conn.commit()
|
|
|
+
|
|
|
+
|
|
|
+def finish_creation_run(conn: sqlite3.Connection, run_id: str, *,
|
|
|
+ status: str = "finished", ts: Optional[int] = None) -> None:
|
|
|
+ conn.execute(
|
|
|
+ "UPDATE creation_search_runs SET status=?, finished_at=? WHERE run_id=?",
|
|
|
+ (status, _now(ts), run_id),
|
|
|
+ )
|
|
|
+ conn.commit()
|
|
|
+
|
|
|
+
|
|
|
+def creation_run_exists(conn: sqlite3.Connection, run_id: str) -> bool:
|
|
|
+ row = conn.execute(
|
|
|
+ "SELECT 1 FROM creation_search_runs WHERE run_id=? LIMIT 1", (run_id,)
|
|
|
+ ).fetchone()
|
|
|
+ return row is not None
|
|
|
+
|
|
|
+
|
|
|
+def ensure_creation_job(conn: sqlite3.Connection, run_id: str, query: str, platform: str,
|
|
|
+ *, ts: Optional[int] = None) -> int:
|
|
|
+ t = _now(ts)
|
|
|
+ conn.execute(
|
|
|
+ "INSERT OR IGNORE INTO creation_search_jobs"
|
|
|
+ "(run_id, query, platform, status, attempts, started_at) VALUES(?,?,?,?,?,?)",
|
|
|
+ (run_id, query, platform, "pending", 0, t),
|
|
|
+ )
|
|
|
+ conn.commit()
|
|
|
+ row = conn.execute(
|
|
|
+ "SELECT id FROM creation_search_jobs WHERE run_id=? AND query=? AND platform=?",
|
|
|
+ (run_id, query, platform),
|
|
|
+ ).fetchone()
|
|
|
+ return int(row["id"])
|
|
|
+
|
|
|
+
|
|
|
+def update_creation_job(conn: sqlite3.Connection, run_id: str, query: str, platform: str,
|
|
|
+ *, status: str, attempts: Optional[int] = None,
|
|
|
+ searched_count: Optional[int] = None,
|
|
|
+ display_count: Optional[int] = None,
|
|
|
+ error: Optional[str] = None, ts: Optional[int] = None) -> None:
|
|
|
+ ensure_creation_job(conn, run_id, query, platform, ts=ts)
|
|
|
+ sets = ["status=?"]
|
|
|
+ params: list[Any] = [status]
|
|
|
+ if attempts is not None:
|
|
|
+ sets.append("attempts=?"); params.append(int(attempts))
|
|
|
+ if searched_count is not None:
|
|
|
+ sets.append("searched_count=?"); params.append(int(searched_count))
|
|
|
+ if display_count is not None:
|
|
|
+ sets.append("display_count=?"); params.append(int(display_count))
|
|
|
+ if error is not None:
|
|
|
+ sets.append("error=?"); params.append(error)
|
|
|
+ if status in ("done", "failed", "partial"):
|
|
|
+ sets.append("finished_at=?"); params.append(_now(ts))
|
|
|
+ elif status == "running":
|
|
|
+ sets.append("started_at=?"); params.append(_now(ts))
|
|
|
+ params.extend([run_id, query, platform])
|
|
|
+ conn.execute(
|
|
|
+ f"UPDATE creation_search_jobs SET {', '.join(sets)} "
|
|
|
+ "WHERE run_id=? AND query=? AND platform=?",
|
|
|
+ params,
|
|
|
+ )
|
|
|
+ conn.commit()
|
|
|
+
|
|
|
+
|
|
|
+def creation_job_is_done(conn: sqlite3.Connection, run_id: str, query: str, platform: str,
|
|
|
+ *, display_limit: int = 5) -> bool:
|
|
|
+ row = conn.execute(
|
|
|
+ "SELECT status, display_count FROM creation_search_jobs "
|
|
|
+ "WHERE run_id=? AND query=? AND platform=?",
|
|
|
+ (run_id, query, platform),
|
|
|
+ ).fetchone()
|
|
|
+ return bool(row and row["status"] == "done" and int(row["display_count"] or 0) >= display_limit)
|
|
|
+
|
|
|
+
|
|
|
+def upsert_creation_item(conn: sqlite3.Connection, *, run_id: str, query: str, platform: str,
|
|
|
+ rank: int, source_id: str = "", url: str = "", title: str = "",
|
|
|
+ author: str = "", body_text: str = "", cover_url: str = "",
|
|
|
+ image_urls: Optional[list[str]] = None, video_url: str = "",
|
|
|
+ media: Optional[dict] = None, raw: Optional[dict] = None,
|
|
|
+ is_displayable: bool = False, error: str = "",
|
|
|
+ ts: Optional[int] = None) -> int:
|
|
|
+ t = _now(ts)
|
|
|
+ conn.execute(
|
|
|
+ "INSERT INTO creation_search_items"
|
|
|
+ "(run_id, query, platform, rank, source_id, url, title, author, body_text, cover_url, "
|
|
|
+ "image_urls, video_url, media_json, raw_json, is_displayable, error, created_at) "
|
|
|
+ "VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) "
|
|
|
+ "ON CONFLICT(run_id, query, platform, rank) DO UPDATE SET "
|
|
|
+ "source_id=excluded.source_id, url=excluded.url, title=excluded.title, "
|
|
|
+ "author=excluded.author, body_text=excluded.body_text, cover_url=excluded.cover_url, "
|
|
|
+ "image_urls=excluded.image_urls, video_url=excluded.video_url, media_json=excluded.media_json, "
|
|
|
+ "raw_json=excluded.raw_json, is_displayable=excluded.is_displayable, "
|
|
|
+ "error=excluded.error, created_at=excluded.created_at",
|
|
|
+ (
|
|
|
+ run_id, query, platform, int(rank), source_id, url, title, author, body_text,
|
|
|
+ cover_url, json.dumps(image_urls or [], ensure_ascii=False), video_url,
|
|
|
+ json.dumps(media or {}, ensure_ascii=False), json.dumps(raw or {}, ensure_ascii=False),
|
|
|
+ 1 if is_displayable else 0, error, t,
|
|
|
+ ),
|
|
|
+ )
|
|
|
+ conn.commit()
|
|
|
+ row = conn.execute(
|
|
|
+ "SELECT id FROM creation_search_items WHERE run_id=? AND query=? AND platform=? AND rank=?",
|
|
|
+ (run_id, query, platform, int(rank)),
|
|
|
+ ).fetchone()
|
|
|
+ return int(row["id"])
|
|
|
+
|
|
|
+
|
|
|
+def upsert_creation_classification(conn: sqlite3.Connection, item_id: int,
|
|
|
+ is_creation: Optional[int], reason: str = "",
|
|
|
+ knowledge: str = "", prompt_version: str = "",
|
|
|
+ error: str = "", ts: Optional[int] = None) -> None:
|
|
|
+ conn.execute(
|
|
|
+ "INSERT INTO creation_item_classifications"
|
|
|
+ "(item_id, is_creation, reason, knowledge, prompt_version, error, classified_at) "
|
|
|
+ "VALUES(?,?,?,?,?,?,?) "
|
|
|
+ "ON CONFLICT(item_id) DO UPDATE SET "
|
|
|
+ "is_creation=excluded.is_creation, reason=excluded.reason, knowledge=excluded.knowledge, "
|
|
|
+ "prompt_version=excluded.prompt_version, error=excluded.error, classified_at=excluded.classified_at",
|
|
|
+ (int(item_id), is_creation, reason, knowledge, prompt_version, error, _now(ts)),
|
|
|
+ )
|
|
|
+ conn.commit()
|
|
|
+
|
|
|
+
|
|
|
+def latest_creation_run_id(conn: sqlite3.Connection) -> Optional[str]:
|
|
|
+ row = conn.execute(
|
|
|
+ "SELECT run_id FROM creation_search_runs ORDER BY started_at DESC, run_id DESC LIMIT 1"
|
|
|
+ ).fetchone()
|
|
|
+ return row["run_id"] if row else None
|
|
|
+
|
|
|
+
|
|
|
+def creation_search_summary(conn: sqlite3.Connection, run_id: Optional[str] = None) -> dict:
|
|
|
+ rid = run_id or latest_creation_run_id(conn)
|
|
|
+ if not rid:
|
|
|
+ return {"run_id": None, "queries": {}}
|
|
|
+ def default_query_entry() -> dict:
|
|
|
+ return {
|
|
|
+ "platforms": {},
|
|
|
+ "done": 0,
|
|
|
+ "failed": 0,
|
|
|
+ "display_count": 0,
|
|
|
+ "imgtext_creation_count": 0,
|
|
|
+ "imgtext_classified_count": 0,
|
|
|
+ "imgtext_total_count": 0,
|
|
|
+ "imgtext_target_count": 10,
|
|
|
+ }
|
|
|
+
|
|
|
+ rows = conn.execute(
|
|
|
+ "SELECT query, platform, status, attempts, searched_count, display_count, error "
|
|
|
+ "FROM creation_search_jobs WHERE run_id=? ORDER BY query, platform",
|
|
|
+ (rid,),
|
|
|
+ ).fetchall()
|
|
|
+ out: dict[str, dict] = {}
|
|
|
+ for r in rows:
|
|
|
+ q = r["query"]
|
|
|
+ ent = out.setdefault(q, default_query_entry())
|
|
|
+ p = {
|
|
|
+ "status": r["status"],
|
|
|
+ "attempts": r["attempts"],
|
|
|
+ "searched_count": r["searched_count"],
|
|
|
+ "display_count": r["display_count"],
|
|
|
+ "error": r["error"],
|
|
|
+ }
|
|
|
+ ent["platforms"][r["platform"]] = p
|
|
|
+ ent["display_count"] += r["display_count"] or 0
|
|
|
+ if r["status"] == "done":
|
|
|
+ ent["done"] += 1
|
|
|
+ if r["status"] == "failed":
|
|
|
+ ent["failed"] += 1
|
|
|
+ count_rows = conn.execute(
|
|
|
+ "SELECT i.query, COUNT(*) AS total_count, "
|
|
|
+ "SUM(CASE WHEN c.item_id IS NOT NULL THEN 1 ELSE 0 END) AS classified_count, "
|
|
|
+ "SUM(CASE WHEN c.is_creation=1 THEN 1 ELSE 0 END) AS creation_count "
|
|
|
+ "FROM creation_search_items i "
|
|
|
+ "LEFT JOIN creation_item_classifications c ON c.item_id=i.id "
|
|
|
+ "WHERE i.run_id=? AND i.is_displayable=1 "
|
|
|
+ "AND i.platform IN ('xiaohongshu', 'weixin') "
|
|
|
+ "GROUP BY i.query",
|
|
|
+ (rid,),
|
|
|
+ ).fetchall()
|
|
|
+ for r in count_rows:
|
|
|
+ ent = out.setdefault(r["query"], default_query_entry())
|
|
|
+ ent["imgtext_creation_count"] = int(r["creation_count"] or 0)
|
|
|
+ ent["imgtext_classified_count"] = int(r["classified_count"] or 0)
|
|
|
+ ent["imgtext_total_count"] = int(r["total_count"] or 0)
|
|
|
+ return {"run_id": rid, "queries": out}
|
|
|
+
|
|
|
+
|
|
|
+def get_creation_query_detail(conn: sqlite3.Connection, query: str, *,
|
|
|
+ run_id: Optional[str] = None, display_limit: int = 5) -> dict:
|
|
|
+ rid = run_id or latest_creation_run_id(conn)
|
|
|
+ platforms = {p: {"status": "not_started", "items": [], "error": None} for p in CREATION_PLATFORMS}
|
|
|
+ if not rid:
|
|
|
+ return {"run_id": None, "query": query, "platforms": platforms}
|
|
|
+ job_rows = conn.execute(
|
|
|
+ "SELECT platform, status, attempts, searched_count, display_count, error "
|
|
|
+ "FROM creation_search_jobs WHERE run_id=? AND query=?",
|
|
|
+ (rid, query),
|
|
|
+ ).fetchall()
|
|
|
+ for r in job_rows:
|
|
|
+ platforms[r["platform"]].update({
|
|
|
+ "status": r["status"],
|
|
|
+ "attempts": r["attempts"],
|
|
|
+ "searched_count": r["searched_count"],
|
|
|
+ "display_count": r["display_count"],
|
|
|
+ "error": r["error"],
|
|
|
+ })
|
|
|
+ item_rows = conn.execute(
|
|
|
+ "SELECT i.*, c.is_creation, c.reason AS class_reason, c.knowledge, "
|
|
|
+ "c.prompt_version, c.error AS class_error, c.classified_at "
|
|
|
+ "FROM creation_search_items i "
|
|
|
+ "LEFT JOIN creation_item_classifications c ON c.item_id=i.id "
|
|
|
+ "WHERE i.run_id=? AND i.query=? AND i.is_displayable=1 "
|
|
|
+ "ORDER BY i.platform, i.rank",
|
|
|
+ (rid, query),
|
|
|
+ ).fetchall()
|
|
|
+ per_platform_counts = {p: 0 for p in CREATION_PLATFORMS}
|
|
|
+ for r in item_rows:
|
|
|
+ p = r["platform"]
|
|
|
+ if p not in platforms or per_platform_counts[p] >= display_limit:
|
|
|
+ continue
|
|
|
+ item = dict(r)
|
|
|
+ item["image_urls"] = json.loads(item.get("image_urls") or "[]")
|
|
|
+ item["media"] = json.loads(item.pop("media_json") or "{}")
|
|
|
+ item.pop("raw_json", None) # 原始回包很大;详情 API 只返回前端展示需要的字段
|
|
|
+ item["classification"] = {
|
|
|
+ "is_creation": item.pop("is_creation"),
|
|
|
+ "reason": item.pop("class_reason") or "",
|
|
|
+ "knowledge": item.pop("knowledge") or "",
|
|
|
+ "prompt_version": item.pop("prompt_version") or "",
|
|
|
+ "error": item.pop("class_error") or "",
|
|
|
+ "classified_at": item.pop("classified_at"),
|
|
|
+ }
|
|
|
+ platforms[p]["items"].append(item)
|
|
|
+ per_platform_counts[p] += 1
|
|
|
+ return {"run_id": rid, "query": query, "platforms": platforms}
|
|
|
+
|
|
|
+
|
|
|
+def creation_items_to_classify(conn: sqlite3.Connection, *, run_id: Optional[str] = None,
|
|
|
+ platforms: Optional[list[str]] = None,
|
|
|
+ retry_failed: bool = True,
|
|
|
+ limit: Optional[int] = None) -> list[dict]:
|
|
|
+ rid = run_id or latest_creation_run_id(conn)
|
|
|
+ if not rid:
|
|
|
+ return []
|
|
|
+ where = ["i.run_id=?", "i.is_displayable=1"]
|
|
|
+ params: list[Any] = [rid]
|
|
|
+ if platforms:
|
|
|
+ qs = ",".join("?" * len(platforms))
|
|
|
+ where.append(f"i.platform IN ({qs})")
|
|
|
+ params.extend(platforms)
|
|
|
+ if retry_failed:
|
|
|
+ where.append("(c.item_id IS NULL OR c.is_creation IS NULL)")
|
|
|
+ else:
|
|
|
+ where.append("c.item_id IS NULL")
|
|
|
+ sql = (
|
|
|
+ "SELECT i.id, i.platform, i.title, i.body_text, i.image_urls, i.video_url "
|
|
|
+ "FROM creation_search_items i "
|
|
|
+ "LEFT JOIN creation_item_classifications c ON c.item_id=i.id "
|
|
|
+ f"WHERE {' AND '.join(where)} ORDER BY i.platform, i.id"
|
|
|
+ )
|
|
|
+ if limit is not None and int(limit) > 0:
|
|
|
+ sql += " LIMIT ?"
|
|
|
+ params.append(int(limit))
|
|
|
+ rows = conn.execute(sql, params).fetchall()
|
|
|
+ out = []
|
|
|
+ for r in rows:
|
|
|
+ d = dict(r)
|
|
|
+ d["image_urls"] = json.loads(d.get("image_urls") or "[]")
|
|
|
+ out.append(d)
|
|
|
+ return out
|