|
|
@@ -59,10 +59,18 @@ CREATE TABLE IF NOT EXISTS post_class (
|
|
|
url TEXT PRIMARY KEY,
|
|
|
is_creation INTEGER, -- 1=创作知识 / 0=非
|
|
|
reason TEXT,
|
|
|
+ knowledge TEXT, -- 提取出的完整创作知识(创作帖才有)
|
|
|
+ points TEXT, -- 结构化知识点 json(按卡片/分段)
|
|
|
created_at INTEGER NOT NULL
|
|
|
);
|
|
|
"""
|
|
|
|
|
|
+# 旧库迁移:post_class 早期没有 knowledge/points 列,补上(已存在则忽略)
|
|
|
+_MIGRATIONS = [
|
|
|
+ "ALTER TABLE post_class ADD COLUMN knowledge TEXT",
|
|
|
+ "ALTER TABLE post_class ADD COLUMN points TEXT",
|
|
|
+]
|
|
|
+
|
|
|
|
|
|
def connect(db_path: Path | str = DB_PATH) -> sqlite3.Connection:
|
|
|
"""打开(必要时建)库,建表,行按 dict 取。busy_timeout 让多进程并发写不报锁。"""
|
|
|
@@ -72,6 +80,11 @@ def connect(db_path: Path | str = DB_PATH) -> sqlite3.Connection:
|
|
|
conn.row_factory = sqlite3.Row
|
|
|
conn.execute("PRAGMA busy_timeout=30000") # 撞锁等 30s 而非报错(与微信补下载并发安全)
|
|
|
conn.executescript(_SCHEMA)
|
|
|
+ for sql in _MIGRATIONS:
|
|
|
+ try:
|
|
|
+ conn.execute(sql)
|
|
|
+ except sqlite3.OperationalError:
|
|
|
+ pass # 列已存在
|
|
|
return conn
|
|
|
|
|
|
|
|
|
@@ -221,9 +234,11 @@ def get_unclassified_posts(conn: sqlite3.Connection) -> list[dict]:
|
|
|
return out
|
|
|
|
|
|
|
|
|
-def upsert_class(conn: sqlite3.Connection, url: str, is_creation: int, reason: str, ts: int) -> None:
|
|
|
- conn.execute("INSERT OR REPLACE INTO post_class(url, is_creation, reason, created_at) VALUES(?,?,?,?)",
|
|
|
- (url, is_creation, reason, ts))
|
|
|
+def upsert_class(conn: sqlite3.Connection, url: str, is_creation: int, reason: str, ts: int,
|
|
|
+ knowledge: str = "", points: str = "") -> None:
|
|
|
+ conn.execute(
|
|
|
+ "INSERT OR REPLACE INTO post_class(url, is_creation, reason, knowledge, points, created_at) "
|
|
|
+ "VALUES(?,?,?,?,?,?)", (url, is_creation, reason, knowledge, points, ts))
|
|
|
conn.commit()
|
|
|
|
|
|
|
|
|
@@ -245,13 +260,20 @@ def posts_to_classify(conn: sqlite3.Connection, platforms: list[str]) -> list[di
|
|
|
|
|
|
|
|
|
def class_map(conn: sqlite3.Connection, urls) -> dict:
|
|
|
- """{url: {is_creation, reason}},给取数时挂分类。"""
|
|
|
+ """{url: {is_creation, reason, knowledge, points}},给取数时挂分类 + 提取的知识点。"""
|
|
|
urls = [u for u in urls if u]
|
|
|
if not urls:
|
|
|
return {}
|
|
|
qs = ",".join("?" * len(urls))
|
|
|
- rows = conn.execute(f"SELECT url, is_creation, reason FROM post_class WHERE url IN ({qs})", urls).fetchall()
|
|
|
- return {r["url"]: {"is_creation": r["is_creation"], "reason": r["reason"]} for r in rows}
|
|
|
+ rows = conn.execute(
|
|
|
+ f"SELECT url, is_creation, reason, knowledge, points FROM post_class WHERE url IN ({qs})", urls
|
|
|
+ ).fetchall()
|
|
|
+ out = {}
|
|
|
+ for r in rows:
|
|
|
+ out[r["url"]] = {"is_creation": r["is_creation"], "reason": r["reason"],
|
|
|
+ "knowledge": r["knowledge"] or "",
|
|
|
+ "points": json.loads(r["points"]) if r["points"] else []}
|
|
|
+ return out
|
|
|
|
|
|
|
|
|
def class_counts(conn: sqlite3.Connection) -> dict:
|