coarse.py 4.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154
  1. """Formal coarse classifier for creation-knowledge candidates."""
  2. from __future__ import annotations
  3. import hashlib
  4. from dataclasses import dataclass, field
  5. from pathlib import Path
  6. from typing import Any
  7. from acquisition.classify import (
  8. MAX_CARDS,
  9. _data_url,
  10. _is_http_url,
  11. _judge,
  12. classify_video as _legacy_classify_video,
  13. )
  14. from core.config import Settings
  15. from core.prompts import load_prompt
  16. from core.text_limits import CLASSIFY_BODY_MAX_CHARS, clip_text
  17. ROOT = Path(__file__).resolve().parents[2]
  18. @dataclass(frozen=True)
  19. class ClassificationResult:
  20. is_creation_knowledge: bool | None
  21. label: str | None
  22. confidence: float | None
  23. reason: str
  24. knowledge: str = ""
  25. prompt_version: str | None = None
  26. result_payload: dict[str, Any] = field(default_factory=dict)
  27. status: str = "classified"
  28. error_message: str | None = None
  29. def prompt_version(*names: str) -> str:
  30. h = hashlib.sha256()
  31. for name in names:
  32. path = ROOT / "prompts" / f"{name}.txt"
  33. h.update(name.encode("utf-8"))
  34. if path.exists():
  35. h.update(path.read_bytes())
  36. return h.hexdigest()[:16]
  37. def classify_imgtext(payload: dict[str, Any], settings: Settings) -> tuple:
  38. """Classify image-text content, accepting both HTTP image URLs and /data paths."""
  39. user = [
  40. {
  41. "type": "text",
  42. "text": (
  43. f"平台:{payload.get('platform')}\n"
  44. f"标题:{payload.get('title', '')}\n"
  45. f"正文:{clip_text(payload.get('body_text') or '', CLASSIFY_BODY_MAX_CHARS)}\n"
  46. "(下附帖子图片,请一并看完)"
  47. ),
  48. }
  49. ]
  50. for image in (payload.get("images") or [])[:MAX_CARDS]:
  51. if _is_http_url(image):
  52. user.append({"type": "image_url", "image_url": {"url": image}})
  53. continue
  54. data_url = _data_url(image, settings)
  55. if data_url:
  56. user.append({"type": "image_url", "image_url": {"url": data_url}})
  57. messages = [
  58. {"role": "system", "content": load_prompt("classify_imgtext")},
  59. {"role": "user", "content": user},
  60. ]
  61. return _judge(messages, settings, timeout=120)
  62. def classify_video(payload: dict[str, Any], settings: Settings) -> tuple:
  63. return _legacy_classify_video(payload, settings)
  64. def coarse_classify_item(
  65. *,
  66. platform: str,
  67. content_mode: str | None = None,
  68. title: str = "",
  69. body_text: str = "",
  70. image_urls: list[str] | None = None,
  71. video_url: str = "",
  72. settings: Settings,
  73. ) -> ClassificationResult:
  74. if content_mode == "unsupported":
  75. return ClassificationResult(
  76. is_creation_knowledge=None,
  77. label="unsupported_content_mode",
  78. confidence=None,
  79. reason="内容模态暂不支持,跳过粗筛",
  80. prompt_version="content_mode_guard",
  81. result_payload={"content_mode": content_mode},
  82. status="skipped",
  83. error_message="unsupported_content_mode",
  84. )
  85. if content_mode == "video_post" and not video_url:
  86. return ClassificationResult(
  87. is_creation_knowledge=None,
  88. label="video_missing",
  89. confidence=None,
  90. reason="视频帖缺少可处理的视频地址,跳过粗筛",
  91. prompt_version="content_mode_guard",
  92. result_payload={"content_mode": content_mode, "video_url_missing": True},
  93. status="skipped",
  94. error_message="video_url_missing",
  95. )
  96. if content_mode == "video_post" or (content_mode is None and video_url):
  97. version = prompt_version("classify_video")
  98. is_creation, reason, knowledge, points = classify_video(
  99. {
  100. "platform": platform,
  101. "title": title,
  102. "body_text": body_text,
  103. "video": video_url,
  104. },
  105. settings,
  106. )
  107. else:
  108. version = prompt_version("classify_imgtext")
  109. is_creation, reason, knowledge, points = classify_imgtext(
  110. {
  111. "platform": platform,
  112. "title": title,
  113. "body_text": body_text,
  114. "images": image_urls or [],
  115. },
  116. settings,
  117. )
  118. if is_creation is None:
  119. return ClassificationResult(
  120. is_creation_knowledge=None,
  121. label=None,
  122. confidence=None,
  123. reason=reason,
  124. knowledge=knowledge,
  125. prompt_version=version,
  126. result_payload={"knowledge": knowledge, "points": points},
  127. status="failed",
  128. error_message=reason,
  129. )
  130. is_hit = bool(is_creation)
  131. return ClassificationResult(
  132. is_creation_knowledge=is_hit,
  133. label="creation" if is_hit else "not_creation",
  134. confidence=1.0,
  135. reason=reason,
  136. knowledge=knowledge,
  137. prompt_version=version,
  138. result_payload={"knowledge": knowledge, "points": points},
  139. status="classified",
  140. )