RAG/utils/data_utils.py

import json

import requests

from core.config import logger
from core.database_data import DatabaseHelper


def add_data(text):
    try:
        response = requests.post(
            url='http://61.48.133.26:8001/api/chunk',
            json={
                "text": text,
                "text_type": 1},
            headers={"Content-Type": "application/json"},
        )
        return response.json()['doc_id']
    except Exception as e:
        logger.error(e)
        return e


def select_data():
    db_helper = DatabaseHelper()
    # 执行查询
    query = """
        SELECT c.crawl_data as json_text
        FROM knowledge_extraction_content a
        LEFT JOIN knowledge_parsing_content b ON a.parsing_id = b.id AND b.request_id = a.request_id
        LEFT JOIN knowledge_crawl_content c ON c.content_id = b.content_id AND c.request_id = a.request_id
        LEFT JOIN knowledge_request d ON d.request_id = a.request_id
        LEFT JOIN knowledge_query e ON e.id = d.query_id
        WHERE a.request_id > '20250905022700393495252' AND e.knowledge_type = '整体' AND a.score >= 0 AND e.category_id = 0
        ORDER BY a.id DESC
        """
    result = db_helper.execute_query(query)
    for row in result:
        add_data(json.loads(row['json_text'])['body_text'])