1. 祝福生活爬虫上线

zhufushenghuo/zhufushenghuo_recommend/zfsh_recommend.py

@@ -8,6 +8,7 @@ import requests
 from common.common import Common
 from common.scheduling_db import MysqlHelper
 from common.mq import MQ
+from common.aliyun_log import AliyunLogger
 from common.public import download_rule, get_config_from_mysql
 
 proxies = {"http": None, "https": None}
@@ -113,6 +114,15 @@ class ZfshRecommend:
                                 cls.download_cnt = 0
                                 cls.element_list = []
                                 return
+                            trace_id = crawler + str(uuid.uuid1())
+                            AliyunLogger.logging(
+                                code="1001",
+                                platform=crawler,
+                                mode=log_type,
+                                env=env,
+                                data=feeds[i],
+                                message="扫描到一条视频"
+                            )
                             cls.i += 1
                             video_title = feeds[i].get("title", "").strip().replace("\n", "") \
                                 .replace("/", "").replace("\\", "").replace("\r", "") \
@@ -171,6 +181,15 @@ class ZfshRecommend:
                             elif cls.repeat_video(log_type, crawler, video_dict["video_id"], env) != 0:
                                 Common.logger(log_type, crawler).info('视频已下载\n')
                                 Common.logging(log_type, crawler, env, '视频已下载\n')
+                                AliyunLogger.logging(
+                                    code="2002",
+                                    platform=crawler,
+                                    mode=log_type,
+                                    message="重复的视频",
+                                    data=video_dict,
+                                    trace_id=trace_id,
+                                    env=env
+                                )
 
                             else:
                                 video_dict["out_user_id"] = video_dict["user_id"]
@@ -183,15 +202,38 @@ class ZfshRecommend:
                                 video_dict["user_id"] = our_uid
                                 video_dict["publish_time"] = video_dict["publish_time_str"]
                                 mq.send_msg(video_dict)
+                                AliyunLogger.logging(
+                                    code="1002",
+                                    platform=crawler,
+                                    mode=log_type,
+                                    message="成功发送至 ETL",
+                                    data=video_dict,
+                                    trace_id=trace_id,
+                                    env=env
+                                )
                                 cls.download_cnt += 1
                                 interval = random.randrange(5, 11)
                                 time.sleep(interval)
                         except Exception as e:
                             Common.logger(log_type, crawler).error(f"抓取单条视频异常:{e}\n")
                             Common.logging(log_type, crawler, env, f"抓取单条视频异常:{e}\n")
+                            AliyunLogger.logging(
+                                code="3000",
+                                platform=crawler,
+                                mode=log_type,
+                                message=f"抓取单条视频异常:{e}\n",
+                                env=env
+                            )
             except Exception as e:
                 Common.logger(log_type, crawler).error(f"抓取第{page}页时异常:{e}\n")
                 Common.logging(log_type, crawler, env, f"抓取第{page}页时异常:{e}\n")
+                AliyunLogger.logging(
+                    code="3000",
+                    platform=crawler,
+                    mode=log_type,
+                    message=f"抓取第{page}页时异常:{e}\n",
+                    env=env
+                )
 
 
 if __name__ == "__main__":