|
|
@@ -7,9 +7,8 @@ import json
|
|
|
import time
|
|
|
import uuid
|
|
|
import random
|
|
|
-import asyncio
|
|
|
-import aiohttp
|
|
|
import datetime
|
|
|
+import requests
|
|
|
|
|
|
sys.path.append(os.getcwd())
|
|
|
|
|
|
@@ -36,15 +35,17 @@ class YuanNiFuQiChangZai(object):
|
|
|
self.expire_flag = False
|
|
|
self.aliyun_log = AliyunLogger(platform=self.platform, mode=self.mode)
|
|
|
|
|
|
- async def process_video_obj(self, video_obj):
|
|
|
+ def process_video_obj(self, video_obj):
|
|
|
"""
|
|
|
处理每一个视频内容
|
|
|
:return: None
|
|
|
"""
|
|
|
trace_id = self.platform + str(uuid.uuid1())
|
|
|
our_user = random.choice(self.user_list)
|
|
|
- publish_time_stamp = int(video_obj['update_time'])
|
|
|
- publish_time_str = datetime.datetime.fromtimestamp(publish_time_stamp).strftime('%Y-%m-%d %H:%M:%S')
|
|
|
+ publish_time_stamp = int(video_obj["update_time"])
|
|
|
+ publish_time_str = datetime.datetime.fromtimestamp(publish_time_stamp).strftime(
|
|
|
+ "%Y-%m-%d %H:%M:%S"
|
|
|
+ )
|
|
|
item = VideoItem()
|
|
|
item.add_video_info("user_id", our_user["uid"])
|
|
|
item.add_video_info("user_name", our_user["nick_name"])
|
|
|
@@ -70,21 +71,21 @@ class YuanNiFuQiChangZai(object):
|
|
|
if pipeline.process_item():
|
|
|
self.download_cnt += 1
|
|
|
self.mq.send_msg(mq_obj)
|
|
|
- # print(json.dumps(mq_obj, ensure_ascii=False, indent=4))
|
|
|
+ # 随机等待 5 分钟
|
|
|
+ time.sleep(60 * random.randint(1, 5))
|
|
|
self.aliyun_log.logging(
|
|
|
code="1002",
|
|
|
message="成功发送至 ETL",
|
|
|
data=mq_obj,
|
|
|
)
|
|
|
if self.download_cnt >= int(
|
|
|
- self.rule_dict.get("videos_cnt", {}).get("min", 200)
|
|
|
+ self.rule_dict.get("videos_cnt", {}).get("min", 200)
|
|
|
):
|
|
|
self.expire_flag = True
|
|
|
|
|
|
- async def get_recommend_list(self, session, page_index):
|
|
|
+ def get_recommend_list(self, page_index):
|
|
|
"""
|
|
|
获取推荐页面的video_list
|
|
|
- :param session: aiohttp 的session
|
|
|
:param page_index: 页码
|
|
|
:return: None
|
|
|
"""
|
|
|
@@ -95,97 +96,81 @@ class YuanNiFuQiChangZai(object):
|
|
|
)
|
|
|
return
|
|
|
headers = {
|
|
|
- 'Host': 'ynfqcz.jiabeijian.cn',
|
|
|
- 'content-time': str(int(time.time() * 1000)),
|
|
|
- 'cache-time': str(int(time.time() * 1000)),
|
|
|
- 'chatkey': 'wxa1431c6e7acdd32d',
|
|
|
- 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 MicroMessenger/6.8.0(0x16080000) NetType/WIFI MiniProgramEnv/Mac MacWechat/WMPF MacWechat/3.8.6(0x13080610) XWEB/1156',
|
|
|
- 'content-type': 'application/x-www-form-urlencoded',
|
|
|
- 'visitorkey': '17096322301026589978',
|
|
|
- 'xweb_xhr': '1',
|
|
|
- 'vision': '1.1.0',
|
|
|
- 'token': '',
|
|
|
- 'accept': '*/*',
|
|
|
- 'sec-fetch-site': 'cross-site',
|
|
|
- 'sec-fetch-mode': 'cors',
|
|
|
- 'sec-fetch-dest': 'empty',
|
|
|
- 'referer': 'https://servicewechat.com/wxa1431c6e7acdd32d/2/page-frame.html',
|
|
|
- 'accept-language': 'en-US,en;q=0.9'
|
|
|
+ "Host": "ynfqcz.jiabeijian.cn",
|
|
|
+ "content-time": str(int(time.time() * 1000)),
|
|
|
+ "cache-time": str(int(time.time() * 1000)),
|
|
|
+ "chatkey": "wxa1431c6e7acdd32d",
|
|
|
+ "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 MicroMessenger/6.8.0(0x16080000) NetType/WIFI MiniProgramEnv/Mac MacWechat/WMPF MacWechat/3.8.6(0x13080610) XWEB/1156",
|
|
|
+ "content-type": "application/x-www-form-urlencoded",
|
|
|
+ "visitorkey": "17096322301026589978",
|
|
|
+ "xweb_xhr": "1",
|
|
|
+ "vision": "1.1.0",
|
|
|
+ "token": "",
|
|
|
+ "accept": "*/*",
|
|
|
+ "sec-fetch-site": "cross-site",
|
|
|
+ "sec-fetch-mode": "cors",
|
|
|
+ "sec-fetch-dest": "empty",
|
|
|
+ "referer": "https://servicewechat.com/wxa1431c6e7acdd32d/2/page-frame.html",
|
|
|
+ "accept-language": "en-US,en;q=0.9",
|
|
|
}
|
|
|
po = {
|
|
|
"cid": "",
|
|
|
"page": page_index,
|
|
|
"is_ads": 1,
|
|
|
- "model": random.choice(["Windows", "Mac", "HuaWei", "Xiaomi", "Xiaomi2", "Yandex", "Google", "iphone", "oppo"]),
|
|
|
+ "model": random.choice(
|
|
|
+ [
|
|
|
+ "Windows",
|
|
|
+ "Mac",
|
|
|
+ "HuaWei",
|
|
|
+ "Xiaomi",
|
|
|
+ "Xiaomi2",
|
|
|
+ "Yandex",
|
|
|
+ "Google",
|
|
|
+ "iphone",
|
|
|
+ "oppo",
|
|
|
+ ]
|
|
|
+ ),
|
|
|
"mini_version": "3.8.6",
|
|
|
- "ini_id": "17096322301026589978"
|
|
|
- }
|
|
|
- params = {
|
|
|
- "parameter": json.dumps(po)
|
|
|
+ "ini_id": "17096322301026589978",
|
|
|
}
|
|
|
+ params = {"parameter": json.dumps(po)}
|
|
|
url = "https://ynfqcz.jiabeijian.cn/index.php/v111/index/index"
|
|
|
- await asyncio.sleep(5)
|
|
|
- async with session.get(
|
|
|
- url, headers=headers, params=params, proxy=tunnel_proxies()['https']
|
|
|
- ) as response:
|
|
|
- data = await response.json()
|
|
|
- for index, video_obj in enumerate(data["data"]["list"], 1):
|
|
|
- try:
|
|
|
- self.aliyun_log.logging(
|
|
|
- code="1001",
|
|
|
- message="扫描到一条视频",
|
|
|
- data=video_obj,
|
|
|
- )
|
|
|
- await self.process_video_obj(video_obj)
|
|
|
- except Exception as e:
|
|
|
- self.aliyun_log.logging(
|
|
|
- code="3000",
|
|
|
- message="抓取第{}条的时候出现问题， 报错信息是{}".format(index, e),
|
|
|
- )
|
|
|
+ time.sleep(5)
|
|
|
+ response = requests.get(
|
|
|
+ url=url, headers=headers, params=params, proxies=tunnel_proxies()["https"]
|
|
|
+ )
|
|
|
+ data = response.json()
|
|
|
+ for index, video_obj in enumerate(data["data"]["list"], 1):
|
|
|
+ try:
|
|
|
+ self.aliyun_log.logging(
|
|
|
+ code="1001",
|
|
|
+ message="扫描到一条视频",
|
|
|
+ data=video_obj,
|
|
|
+ )
|
|
|
+ self.process_video_obj(video_obj)
|
|
|
+ except Exception as e:
|
|
|
+ self.aliyun_log.logging(
|
|
|
+ code="3000",
|
|
|
+ message="抓取第{}条的时候出现问题， 报错信息是{}".format(index, e),
|
|
|
+ )
|
|
|
|
|
|
- async def run(self):
|
|
|
+ def run(self):
|
|
|
"""
|
|
|
执行代码
|
|
|
:return: None
|
|
|
"""
|
|
|
- async with aiohttp.ClientSession() as session:
|
|
|
- # for i in range(0, 100, 20):
|
|
|
- # if self.expire_flag:
|
|
|
- # return
|
|
|
- # tasks = [self.get_recommend_list(session, index) for index in range(i, min(i + 20, 200))]
|
|
|
- # await asyncio.gather(*tasks)
|
|
|
- # done, pending = await asyncio.wait(
|
|
|
- # tasks, return_when=asyncio.FIRST_COMPLETED
|
|
|
- # )
|
|
|
- # # 取消所有剩余的任务
|
|
|
- # for task in pending:
|
|
|
- # task.cancel()
|
|
|
-
|
|
|
- # tasks = [self.get_recommend_list(session, index) for index in range(1, 100)]
|
|
|
- # await asyncio.gather(*tasks)
|
|
|
- # done, pending = await asyncio.wait(
|
|
|
- # tasks, return_when=asyncio.FIRST_COMPLETED
|
|
|
- # )
|
|
|
- # # 取消所有剩余的任务
|
|
|
- # for task in pending:
|
|
|
- # task.cancel()
|
|
|
- for page in range(1, 30):
|
|
|
- if self.expire_flag:
|
|
|
+ for page in range(1, 40):
|
|
|
+ if self.expire_flag:
|
|
|
+ self.aliyun_log.logging(
|
|
|
+ code="2000",
|
|
|
+ message="本轮已经抓取到足够的数据，自动退出\t{}".format(self.download_cnt),
|
|
|
+ )
|
|
|
+ return
|
|
|
+ else:
|
|
|
+ try:
|
|
|
+ self.get_recommend_list(page_index=page)
|
|
|
+ except Exception as e:
|
|
|
self.aliyun_log.logging(
|
|
|
- code="2000",
|
|
|
- message="本轮已经抓取到足够的数据，自动退出\t{}".format(self.download_cnt),
|
|
|
+ code="3000",
|
|
|
+ message="抓取第{}页时候出现错误， 报错信息是{}".format(page, e),
|
|
|
)
|
|
|
- # message = "本轮已经抓取到足够的数据，自动退出\t{}".format(self.download_cnt)
|
|
|
- # print(message)
|
|
|
- return
|
|
|
- else:
|
|
|
- try:
|
|
|
- await self.get_recommend_list(session, page_index=page)
|
|
|
- except Exception as e:
|
|
|
- self.aliyun_log.logging(
|
|
|
- code="3000",
|
|
|
- message="抓取第{}页时候出现错误， 报错信息是{}".format(page, e),
|
|
|
- )
|
|
|
- # message = "抓取第{}页时候出现错误， 报错信息是{}".format(page, e)
|
|
|
- # print(message)
|
|
|
-
|
罗俊辉