pqAPI 的 cookie 使用 apollo 配置

影响任务：
getOffVideosDaily.py
checkVideoStatusDaily.py

applications/denetMysql.py

@@ -2,7 +2,6 @@
 @author: luojunhui
 """
 import pymysql
-from config import planConfigDict
 
 
 class DeNetMysql(object):
@@ -18,20 +17,6 @@ class DeNetMysql(object):
         charset="utf8mb4"  # 如果数据库里面的文本是utf8编码的，charset指定是utf8
     )
 
-    @classmethod
-    def getUnEmptyPlan(cls):
-        """
-        :return:
-        """
-        sql = "select id, name from crawler_plan where name like '%腾讯互选-%' and crawler_total_num > 0;"
-        cursor = cls.connection.cursor()
-        cursor.execute(sql)
-        data = cursor.fetchall()
-        result = [list(line)[0] for line in data]
-        # print(result)
-        result = [i for i in result if planConfigDict.get(i)]
-        return result
-
     @classmethod
     def update(cls, sql, params):
         """
@@ -53,12 +38,4 @@ class DeNetMysql(object):
         cursor = cls.connection.cursor()
         cursor.execute(sql)
         result = cursor.fetchall()
-        return result
-
-    @classmethod
-    def close(cls):
-        """
-        关闭连接
-        :return:
-        """
-        cls.connection.close()
+        return result

applications/longArticlesMysql.py

@@ -39,12 +39,3 @@ class longArticlesMySQL(object):
         result = cursor.fetchall()
         return result
 
-    @classmethod
-    def close(cls):
-        """
-        关闭连接
-        :return:
-        """
-        cls.connection.close()
-
-

applications/pipeline.py

@@ -33,9 +33,3 @@ class LongArticlesPipeline(object):
         """
         # 查询账号是否已经存在
 
-
-
-s = WeixinSpider()
-response = s.get_account_by_url("https://mp.weixin.qq.com/s/ZOa1JMX-TFCLwfnaXiB7nA")
-print(json.dumps(response, ensure_ascii=False, indent=4))
-

applications/pqMysql.py

@@ -24,8 +24,12 @@ class PQMySQL(object):
         :return:
         """
         cursor = cls.connection.cursor()
-        cursor.execute(sql, params)
-        cls.connection.commit()
+        try:
+            cursor.execute(sql, params)
+            cls.connection.commit()
+        except Exception as e:
+            print(e)
+            cls.connection.rollback()
 
     @classmethod
     def select(cls, sql):
@@ -39,23 +43,17 @@ class PQMySQL(object):
         result = cursor.fetchall()
         return result
 
-    def insertMany(self, sql, params):
-        """
-
-        :param sql:
-        :param params:
-        :return:
-        """
-        return None
-
     @classmethod
-    def close(cls):
+    def insertMany(cls, sql, params_list):
         """
-
+        :param sql:
+        :param params_list:
         :return:
         """
-        cls.connection.close()
-
-
-
-
+        cursor = cls.connection.cursor()
+        try:
+            cursor.executemany(query=sql, args=params_list)
+            cls.connection.commit()
+        except Exception as e:
+            print("Insert Many Defeat--{}".format(e))
+            cls.connection.rollback()

getOffVideosDaily.py

@@ -9,7 +9,7 @@ import requests
 import schedule
 from tqdm import tqdm
 
-from applications import PQMySQL, Functions, log
+from applications import PQMySQL, Functions, PQAPI, log
 from applications.decoratorApi import retryOnTimeout
 
 
@@ -52,6 +52,7 @@ class AutoGetOffVideos(object):
     自动下架视频
     """
     pqMysql = PQMySQL()
+    pqAPI = PQAPI()
 
     @classmethod
     def getLongArticlesVideos(cls, time_stamp):
@@ -107,51 +108,18 @@ class AutoGetOffVideos(object):
     @classmethod
     def changeVideoIdStatus(cls, video_id):
         """
-        修改视频规则
+        修改视频id状态
         :return:
         """
-        url = "https://admin.piaoquantv.com/manager/video/audit/v2/updateAuditStatus"
-        payload = "videoId={}&auditStatus=2&updateReasonJson=&rejectReasonJson=%5B%7B%22reason%22%3A%22%E9%95%BF%E6%96%87%E8%87%AA%E5%8A%A8%E4%B8%8B%E6%9E%B6%22%2C%22reasonId%22%3A-1%7D%5D&adminUid=206".format(
-            video_id)
-        headers = {
-            'accept': 'application/json',
-            'accept-language': 'en,zh;q=0.9,zh-CN;q=0.8',
-            'content-type': 'application/x-www-form-urlencoded;charset=UTF-8',
-            'cookie': 'SESSION=NTljNTg2YjktMTU0MC00YWQ5LWE4ZTktNDFhODY0NzM3NTcx',
-            'origin': 'https://admin.piaoquantv.com',
-            'priority': 'u=1, i',
-            'sec-ch-ua': '"Not/A)Brand";v="8", "Chromium";v="126", "Google Chrome";v="126"',
-            'sec-ch-ua-mobile': '?0',
-            'sec-ch-ua-platform': '"macOS"',
-            'sec-fetch-dest': 'empty',
-            'sec-fetch-mode': 'cors',
-            'sec-fetch-site': 'same-origin',
-            'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36'
-        }
-        response = requests.request(
-            "POST",
-            url,
-            headers=headers,
-            data=payload,
-            timeout=10
-        )
-        if response.status_code == 200:
-            result = response.json()
-            if result.get("code", None) == 0:
-                cls.updateVideoIdStatus(video_id=video_id)
-            else:
-                log(
-                    task="getOffVideosDaily",
-                    function="changeVideoIdStatus",
-                    message="请求票圈修改状态异常---video_id = {}".format(video_id),
-                    data=result
-                )
+        response = cls.pqAPI.changeVideoStatus(video_id, 2)
+        if response:
+            cls.updateVideoIdStatus(video_id=video_id)
         else:
             log(
                 task="getOffVideosDaily",
                 function="changeVideoIdStatus",
                 status="fail",
-                message="请求票圈修改状态异常,状态码非 200 ---video_id = {}".format(video_id),
+                message="请求票圈修改状态异常: ---video_id = {}".format(video_id),
             )
 
     @classmethod
@@ -241,8 +209,3 @@ if __name__ == '__main__':
     while True:
         schedule.run_pending()
         time.sleep(1)
-        # log(
-        #     task="getOffVideos",
-        #     function="main",
-        #     message="自动下架视频任务正常执行"
-        # )

requirements.txt

@@ -6,12 +6,13 @@ async-timeout
 elastic-transport
 elasticsearch
 numpy
-odps
-pandas
+odps~=3.5.1
+pandas~=2.2.2
 pip
-pymysql
+pymysql~=1.1.0
 pyodps
-requests
-schedule
+requests~=2.32.3
+schedule~=1.2.2
 setuptools
-tqdm
+tqdm~=4.66.4
+pyapollos~=0.1.5

stratrgy/upLevel.py

@@ -120,6 +120,66 @@ class articleLevelUp(object):
             "99%": 3.277849462365585
         }
     }
+    firstLevelMap = {
+        "阅读均值倍数": {
+            "mean": 1.0469541000103093,
+            "max": 25.719380724649426,
+            "min": 0.037429819089207735,
+            "median": 0.9521466355025219,
+            "75%": 1.2800839124458492,
+            "80%": 1.370275508982941,
+            "90%": 1.674800845262867,
+            "95%": 1.995613204168999,
+            "99%": 2.9869225601165135
+        },
+        "阅读率": {
+            "mean": 0.016311355353310464,
+            "max": 0.7427434456928839,
+            "min": 0.0006011082360982278,
+            "median": 0.01255841121495327,
+            "75%": 0.020080845617803843,
+            "80%": 0.022950649260452458,
+            "90%": 0.03136776141996209,
+            "95%": 0.0398727631704118,
+            "99%": 0.05986584275411923
+        },
+        "小程序打开率": {
+            "mean": 0.20655535828501095,
+            "max": 0.8,
+            "min": 0.0,
+            "median": 0.19921326215228996,
+            "75%": 0.25838983436476154,
+            "80%": 0.27586206896551724,
+            "90%": 0.32290043225754594,
+            "95%": 0.3709317026683608,
+            "99%": 0.4685840031614304
+        },
+        "T+0裂变率": {
+            "mean": 0.6660929834568661,
+            "max": 46.0,
+            "min": 0.0,
+            "median": 0.5434782608695652,
+            "75%": 0.7940509083886685,
+            "80%": 0.8776439089692103,
+            "90%": 1.159075752014066,
+            "95%": 1.62348848368522,
+            "99%": 2.785400696864109
+        }
+    }
+
+    @classmethod
+    def readRateDebias(cls, row):
+        """
+        阅读均值倍数通过头条消偏
+        :param row:
+        :return:
+        """
+        if row["位置"] != 1:
+            return row["阅读量"] / (
+                    max(1.0, row["头条阅读量"] / row["头条阅读均值"]) * row["阅读均值"]
+            )
+        else:
+            return row["阅读均值倍数"]
 
     @classmethod
     def getBaseData(cls):
@@ -127,7 +187,6 @@ class articleLevelUp(object):
 
         :return:
         """
-        # today = datetime.today().strftime("%Y%m%d")
         sql = f"""
         SELECT
             position, fans, view_count, avg_view_count, first_view_count, first_avg_view_count, read_rate, read_fans_rate, first_read_rate, fission0_first_rate, title, link
@@ -136,6 +195,8 @@ class articleLevelUp(object):
         """
         response = lam.select(sql)
         df = DataFrame(response, columns=cls.columns)
+        df = df.sort_values(by=["阅读均值倍数"], ascending=[False]).reset_index(drop=True)
+        df = df[df["粉丝量"] > 10000].reset_index(drop=True)
         return df
 
     @classmethod
@@ -208,7 +269,7 @@ class articleLevelUp(object):
         dataThreeToEight = dataThreeToEight[dataThreeToEight['位置'].isin([3, 4, 5, 6, 7, 8])]
         filter_data = dataThreeToEight[
             (dataThreeToEight['T+0裂变率'] > cls.statMapThreeToEight['T+0裂变率']['95%'])
-            & (dataThreeToEight['阅读均值倍数'] > cls.statMapThreeToEight['阅读均值倍数']['95%'])
+            & (dataThreeToEight['阅读均值倍数'] > cls.statMapThreeToEight['阅读均值倍数']['80%'])
             ]
         return filter_data
 
@@ -227,8 +288,9 @@ class articleLevelUp(object):
 
 
 U = articleLevelUp()
+U.analysisDF(indexList=[1])
 f_d = U.upLevel2To1()
 for line in list(zip(f_d['标题'], f_d['链接'])):
     print(line[0])
     print(line[1])
-    print("\n")
+    print("\n")