import json import requests from flask import Flask, request from flask import jsonify from common.db.mysql_help import MysqlHelper from user_spider.user_info import * app = Flask(__name__) app.config['JSON_AS_ASCII'] = False # 只接受get方法访问 @app.route("/v1/crawler/source/getall", methods=["GET"]) def getSource(): # 获取传入的params参数 get_data = request.args.to_dict() fields = get_data.get('fields') # # 对参数进行操作 sql = 'select source, task_type, spider_name from crawler_source' result = MysqlHelper.get_values(sql) if not result: return jsonify({'return_code': '200', 'result': [], 'message': 'no data'}) source_list = list() for source, task_type, spider_name in result: data = dict( source=source, task_type=task_type, spider_name=spider_name ) source_list.append(data) return jsonify({'return_code': '200', 'result': source_list}) @app.route("/v1/crawler/task/insert", methods=["POST"]) def insertTask(): data = request.form outer_info = data.get(('spider_link')) source = data.get('source') exist_outer_info = list() for link in outer_info: s_sql = f'select spider_link from crawler_task where source={source}' result = MysqlHelper.get_values(s_sql) if link in eval(result[0]): exist_outer_info.append(link) if exist_outer_info: return jsonify({'code': 200, 'message': '名单重复', 'spider_link': exist_outer_info}) # 获取到一个以键且为逗号分隔的字符串,返回一个字符串 keys = ','.join(data.keys()) values = ','.join(['%s'] * len(data)) sql = 'insert into {table}({keys}) VALUES({values})'.format(table='crawler_task', keys=keys, values=values) MysqlHelper.insert_values(sql, tuple(data.values())) return jsonify({'code': 200, 'message': 'task create success'}) @app.route("/v1/crawler/task/gettask", methods=["GET"]) def getAllTask(): get_data = request.args.to_dict() page = get_data.get('page', 1) offset = get_data.get('offset', 10) start_count = (page * offset) - offset end_count = page * offset sql = f"""select task_id, task_name from crawler_task limit {start_count}, {end_count}""" result = MysqlHelper.get_values(sql) if not result: return jsonify({'return_code': '200', 'result': [], 'message': 'no data'}) source_list = list() for task_id, task_name in result: data = dict( task_id=task_id, task_name=task_name, ) source_list.append(data) return jsonify({'code': '200', 'result': source_list}) # @app.route("/v1/crawler/task/getone", methods=["GET"]) # def getOneTask(): # get_data = request.args.to_dict() # task_id = get_data['task_id'] # sql = f'select task_id, spider_link from crawler_task where task_id={task_id}' # result = MysqlHelper.get_values(sql) # if not result: # return jsonify({'code': '200', 'result': [], 'message': 'no data'}) # for task_id, spider_link in result: # data = dict( # task_id=task_id, # spider_link=spider_link, # ) # return jsonify({'code': '200', 'result': data}) @app.route("/v1/crawler/task/update", methods=["POST"]) def updateTask(): task_id = request.form.get('task_id') spider_link = request.form.get('spider_link') print(spider_link, task_id) sql = f"""UPDATE crawler_task SET spider_link='{spider_link}' where task_id = {task_id}""" print(sql) result = MysqlHelper.update_values(sql) if result: return jsonify({'code': 200, 'message': 'task update success'}) else: return jsonify({'code': 400, 'message': 'task update faild'}) def get_user_info(source): source_spider = { 'xigua': xigua_user_info } return source_spider.get(source) @app.route("/v1/crawler/author/create", methods=["POST"]) def createUser(): get_media_url = 'http://videotest-internal.yishihui.com/longvideoapi/user/virtual/crawler/registerVirtualUser' data = request.form.get('spider_link') source = request.form.get('source') user_tag = request.form.get('user_tag') for author_url in eval(data): # crawler = get_user_info(source) # user_info = crawler(author_url) post_data = { # 'count': 1, # (必须)账号个数:传1 # 'accountType': 4, # (必须)账号类型 :传 4 app虚拟账号 'pwd': '', # 密码 默认 12346 'nickName': '', # 昵称 默认 vuser...... 'avatarUrl': '', # 头像Url 默认 http://weapppiccdn.yishihui.com/resources/images/pic_normal.png 'tagName': user_tag, # 多条数据用英文逗号分割 } response = requests.post(url=get_media_url, params=post_data) # print(response.text) media_id = response.json()['data'] return jsonify({'data': data}) if __name__ == "__main__": app.run(debug=True,port=5050)