|
|
@@ -29,20 +29,23 @@ class KanyikanRecommend:
|
|
|
def get_videoList(cls, log_type, crawler, our_uid, rule_dict, env):
|
|
|
mq = MQ(topic_name="topic_crawler_etl_" + env)
|
|
|
try:
|
|
|
+
|
|
|
+ Common.logger(log_type, crawler).info(f"正在抓取列表页{crawler}")
|
|
|
+ Common.logging(log_type, crawler, env, f"正在抓取列表页")
|
|
|
+ Common.logger(log_type, crawler).info(f"Test{crawler}")
|
|
|
+ session = Common.get_session(log_type, crawler, env)
|
|
|
+ if session is None:
|
|
|
+ time.sleep(1)
|
|
|
+ cls.get_videoList(log_type, crawler, our_uid, rule_dict, env)
|
|
|
for i in range(20):
|
|
|
- Common.logger(log_type, crawler).info(f"正在抓取列表页{crawler}")
|
|
|
- Common.logging(log_type, crawler, env, f"正在抓取列表页")
|
|
|
- Common.logger(log_type, crawler).info(f"Test{crawler}")
|
|
|
- session = Common.get_session(log_type, crawler, env)
|
|
|
- if session is None:
|
|
|
- time.sleep(1)
|
|
|
- cls.get_videoList(log_type, crawler, our_uid, rule_dict, env)
|
|
|
url = 'https://search.weixin.qq.com/cgi-bin/recwxa/recwxavideolist?'
|
|
|
vid = random.choice(
|
|
|
["wxv_2914137535516262402", "wxv_3043328250787430408", "wxv_2551550656600670208",
|
|
|
"wxv_2687527849242787845", "wxv_3022753845694726149", "wxv_3086921997647118337"])
|
|
|
channelid = random.choice(
|
|
|
- ["200201", "200", "208"])
|
|
|
+ ["200201", "200", "208", "208201"])
|
|
|
+ switchnewuser = random.choice(
|
|
|
+ ["0", "1"])
|
|
|
switchprofile = random.choice(
|
|
|
["0", "1"])
|
|
|
subscene = random.choice(
|
|
|
@@ -59,7 +62,7 @@ class KanyikanRecommend:
|
|
|
"sharesearchid": '0',
|
|
|
"nettype": 'wifi',
|
|
|
"switchprofile": switchprofile,
|
|
|
- "switchnewuser": "0",
|
|
|
+ "switchnewuser": switchnewuser,
|
|
|
}, {
|
|
|
"session": session,
|
|
|
"wxaVersion": "3.17.8",
|
|
|
@@ -78,7 +81,7 @@ class KanyikanRecommend:
|
|
|
"isFromUgc": "false",
|
|
|
"ad": 0,
|
|
|
"switchprofile": switchprofile,
|
|
|
- "switchnewuser": "0",
|
|
|
+ "switchnewuser": switchnewuser,
|
|
|
}])
|
|
|
header = {
|
|
|
'Host': 'search.weixin.qq.com',
|
|
|
@@ -134,7 +137,7 @@ class KanyikanRecommend:
|
|
|
else:
|
|
|
video_url = feeds[i]["videoInfo"]["videoCdnInfo"]["urlInfo"][0]["url"]
|
|
|
videoId = feeds[i].get("videoId", "")
|
|
|
- videoId = videoId+"kyk_plus"
|
|
|
+ videoId = "{}kyk_plus".format(videoId)
|
|
|
video_dict = {
|
|
|
"video_title": video_title,
|
|
|
"video_id": videoId,
|
|
|
@@ -187,7 +190,7 @@ class KanyikanRecommend:
|
|
|
video_dict["user_id"] = our_uid
|
|
|
video_dict["publish_time"] = video_dict["publish_time_str"]
|
|
|
mq.send_msg(video_dict)
|
|
|
- time.sleep(random.randint(31, 40))
|
|
|
+ time.sleep(random.randint(10, 15))
|
|
|
except Exception as e:
|
|
|
Common.logger(log_type, crawler).error(f"抓取单条视频异常:{e}\n")
|
|
|
Common.logging(log_type, crawler, env, f"抓取单条视频异常:{e}\n")
|
