coarse.py 3.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130
  1. """Formal coarse classifier for creation-knowledge candidates."""
  2. from __future__ import annotations
  3. import hashlib
  4. from dataclasses import dataclass, field
  5. from pathlib import Path
  6. from typing import Any
  7. from acquisition.classify import (
  8. MAX_CARDS,
  9. _data_url,
  10. _is_http_url,
  11. _judge,
  12. classify_video as _legacy_classify_video,
  13. )
  14. from core.config import Settings
  15. from core.prompts import load_prompt
  16. ROOT = Path(__file__).resolve().parents[2]
  17. @dataclass(frozen=True)
  18. class ClassificationResult:
  19. is_creation_knowledge: bool | None
  20. label: str | None
  21. confidence: float | None
  22. reason: str
  23. knowledge: str = ""
  24. prompt_version: str | None = None
  25. result_payload: dict[str, Any] = field(default_factory=dict)
  26. status: str = "classified"
  27. error_message: str | None = None
  28. def prompt_version(*names: str) -> str:
  29. h = hashlib.sha256()
  30. for name in names:
  31. path = ROOT / "prompts" / f"{name}.txt"
  32. h.update(name.encode("utf-8"))
  33. if path.exists():
  34. h.update(path.read_bytes())
  35. return h.hexdigest()[:16]
  36. def classify_imgtext(payload: dict[str, Any], settings: Settings) -> tuple:
  37. """Classify image-text content, accepting both HTTP image URLs and /data paths."""
  38. user = [
  39. {
  40. "type": "text",
  41. "text": (
  42. f"平台:{payload.get('platform')}\n"
  43. f"标题:{payload.get('title', '')}\n"
  44. f"正文:{(payload.get('body_text') or '')[:1500]}\n"
  45. "(下附帖子图片,请一并看完)"
  46. ),
  47. }
  48. ]
  49. for image in (payload.get("images") or [])[:MAX_CARDS]:
  50. if _is_http_url(image):
  51. user.append({"type": "image_url", "image_url": {"url": image}})
  52. continue
  53. data_url = _data_url(image, settings)
  54. if data_url:
  55. user.append({"type": "image_url", "image_url": {"url": data_url}})
  56. messages = [
  57. {"role": "system", "content": load_prompt("classify_imgtext")},
  58. {"role": "user", "content": user},
  59. ]
  60. return _judge(messages, settings, timeout=120)
  61. def classify_video(payload: dict[str, Any], settings: Settings) -> tuple:
  62. return _legacy_classify_video(payload, settings)
  63. def coarse_classify_item(
  64. *,
  65. platform: str,
  66. title: str = "",
  67. body_text: str = "",
  68. image_urls: list[str] | None = None,
  69. video_url: str = "",
  70. settings: Settings,
  71. ) -> ClassificationResult:
  72. if platform == "douyin" or video_url:
  73. version = prompt_version("classify_video")
  74. is_creation, reason, knowledge, points = classify_video(
  75. {
  76. "platform": platform,
  77. "title": title,
  78. "body_text": body_text,
  79. "video": video_url,
  80. },
  81. settings,
  82. )
  83. else:
  84. version = prompt_version("classify_imgtext")
  85. is_creation, reason, knowledge, points = classify_imgtext(
  86. {
  87. "platform": platform,
  88. "title": title,
  89. "body_text": body_text,
  90. "images": image_urls or [],
  91. },
  92. settings,
  93. )
  94. if is_creation is None:
  95. return ClassificationResult(
  96. is_creation_knowledge=None,
  97. label=None,
  98. confidence=None,
  99. reason=reason,
  100. knowledge=knowledge,
  101. prompt_version=version,
  102. result_payload={"knowledge": knowledge, "points": points},
  103. status="failed",
  104. error_message=reason,
  105. )
  106. is_hit = bool(is_creation)
  107. return ClassificationResult(
  108. is_creation_knowledge=is_hit,
  109. label="creation" if is_hit else "not_creation",
  110. confidence=1.0,
  111. reason=reason,
  112. knowledge=knowledge,
  113. prompt_version=version,
  114. result_payload={"knowledge": knowledge, "points": points},
  115. status="classified",
  116. )