filter update holo->mysql

config.py

@@ -145,6 +145,16 @@ class DevelopmentConfig(BaseConfig):
         'charset': 'utf8'
     }
 
+    # 测试环境 过滤用mysql地址
+    FILTER_MYSQL_INFO = {
+        'host': 'am-bp1g3ys9u00u483uc131930.ads.aliyuncs.com',
+        'port': 3306,
+        'user': 'lv_manager',
+        'password': 'lv_manager@2020',
+        'db': 'longvideo',
+        'charset': 'utf8'
+    }
+
     # Hologres视频状态存储表名
     VIDEO_STATUS = 'longvideo_test.dwd_mdm_item_video_stat'
 
@@ -198,6 +208,16 @@ class TestConfig(BaseConfig):
         'charset': 'utf8'
     }
 
+    # 测试环境 过滤用mysql地址
+    FILTER_MYSQL_INFO = {
+        'host': 'am-bp1g3ys9u00u483uc131930.ads.aliyuncs.com',
+        'port': 3306,
+        'user': 'lv_manager',
+        'password': 'lv_manager@2020',
+        'db': 'longvideo',
+        'charset': 'utf8'
+    }
+
     # Hologres视频状态存储表名
     VIDEO_STATUS = 'longvideo_test.dwd_mdm_item_video_stat'
 
@@ -251,6 +271,16 @@ class PreProductionConfig(BaseConfig):
         'charset': 'utf8'
     }
 
+    # 生产环境 过滤用mysql地址
+    FILTER_MYSQL_INFO = {
+        'host': 'am-bp15tqt957i3b3sgi131950.ads.aliyuncs.com',
+        'port': 3306,
+        'user': 'lv_manager',
+        'password': 'lv_manager@2020',
+        'db': 'longvideo',
+        'charset': 'utf8'
+    }
+
     # Hologres视频状态存储表名
     VIDEO_STATUS = 'longvideo.dwd_mdm_item_video_stat'
 
@@ -304,6 +334,16 @@ class ProductionConfig(BaseConfig):
         'charset': 'utf8'
     }
 
+    # 生产环境 过滤用mysql地址
+    FILTER_MYSQL_INFO = {
+        'host': 'am-bp15tqt957i3b3sgi131950.ads.aliyuncs.com',
+        'port': 3306,
+        'user': 'lv_manager',
+        'password': 'lv_manager@2020',
+        'db': 'longvideo',
+        'charset': 'utf8'
+    }
+
     # Hologres视频状态存储表名
     VIDEO_STATUS = 'longvideo.dwd_mdm_item_video_stat'
 

db_helper.py

@@ -249,11 +249,11 @@ class HologresHelper(object):
 
 
 class MysqlHelper(object):
-    def __init__(self):
+    def __init__(self, mysql_info):
         """
         初始化mysql连接信息
         """
-        self.mysql_info = config_.MYSQL_INFO
+        self.mysql_info = mysql_info
 
     def get_data(self, sql):
         """

rov_train.py

@@ -282,7 +282,7 @@ def predict_test():
     """测试环境数据生成"""
     # 获取测试环境中最近发布的40000条视频
     sql = "SELECT id FROM wx_video ORDER BY id DESC LIMIT 40000;"
-    mysql_helper = MysqlHelper()
+    mysql_helper = MysqlHelper(mysql_info=config_.MYSQL_INFO)
     data = mysql_helper.get_data(sql=sql)
     video_ids = [video[0] for video in data]
     # 视频状态过滤

utils.py

@@ -142,31 +142,46 @@ def filter_video_status(video_ids):
     :param video_ids: 视频id列表 type-list
     :return: filtered_videos
     """
+    mysql_helper = MysqlHelper(mysql_info=config_.FILTER_MYSQL_INFO)
+    video_status_sql = "SELECT t1.id AS 'video_id', " \
+                       "t1.transcode_status AS 'transcoding_status', " \
+                       "t2.audit_status AS 'audit_status', " \
+                       "t2.video_status AS 'open_status', " \
+                       "t2.recommend_status AS 'applet_rec_status', " \
+                       "t2.app_recommend_status AS 'app_rec_status', " \
+                       "t3.charge AS 'payment_status', " \
+                       "case when t4.max_validate_count is null then 0 else t4.max_validate_count end AS 'encryption_status' " \
+                       "FROM longvideo.wx_video t1 " \
+                       "LEFT JOIN longvideo.wx_video_status t2 ON t1.id= t2.video_id " \
+                       "LEFT JOIN longvideo.wx_video_detail t3 ON t1.id= t3.video_id " \
+                       "LEFT JOIN longvideo.wx_video_pwd t4 ON t1.id= t4.video_id"
     if len(video_ids) == 1:
-        sql = "set hg_experimental_enable_shard_pruning=off; " \
-              "SELECT video_id " \
-              "FROM {} " \
+        sql = "SELECT video_id " \
+              "FROM ({}) " \
               "WHERE audit_status = 5 " \
               "AND applet_rec_status IN (1, -6) " \
               "AND open_status = 1 " \
               "AND payment_status = 0 " \
               "AND encryption_status != 5 " \
               "AND transcoding_status = 3 " \
-              "AND video_id IN ({});".format(config_.VIDEO_STATUS, video_ids[0])
-    else:
-        sql = "set hg_experimental_enable_shard_pruning=off; " \
-              "SELECT video_id " \
-              "FROM {} " \
-              "WHERE audit_status = 5 " \
-              "AND applet_rec_status IN (1, -6) " \
-              "AND open_status = 1 " \
-              "AND payment_status = 0 " \
-              "AND encryption_status != 5 " \
-              "AND transcoding_status = 3 " \
-              "AND video_id IN {};".format(config_.VIDEO_STATUS, tuple(video_ids))
+              "AND video_id IN ({});".format(video_status_sql, video_ids[0])
+        data = mysql_helper.get_data(sql=sql)
 
-    hologres_helper = HologresHelper()
-    data = hologres_helper.get_data(sql=sql)
+    else:
+        data = []
+        for i in range(len(video_ids) // 2000 + 1):
+            sql = "SELECT video_id " \
+                  "FROM ({}) " \
+                  "WHERE audit_status = 5 " \
+                  "AND applet_rec_status IN (1, -6) " \
+                  "AND open_status = 1 " \
+                  "AND payment_status = 0 " \
+                  "AND encryption_status != 5 " \
+                  "AND transcoding_status = 3 " \
+                  "AND video_id IN {};".format(video_status_sql, tuple(video_ids[i*2000:(i+1)*2000]))
+            select_res = mysql_helper.get_data(sql=sql)
+            if select_res is not None:
+                data += select_res
     filtered_videos = [int(temp[0]) for temp in data]
     return filtered_videos
 
@@ -177,31 +192,48 @@ def filter_video_status_app(video_ids):
     :param video_ids: 视频id列表 type-list
     :return: filtered_videos
     """
+    mysql_helper = MysqlHelper(mysql_info=config_.FILTER_MYSQL_INFO)
+    video_status_sql = "SELECT t1.id AS 'video_id', " \
+                       "t1.transcode_status AS 'transcoding_status', " \
+                       "t2.audit_status AS 'audit_status', " \
+                       "t2.video_status AS 'open_status', " \
+                       "t2.recommend_status AS 'applet_rec_status', " \
+                       "t2.app_recommend_status AS 'app_rec_status', " \
+                       "t3.charge AS 'payment_status', " \
+                       "case when t4.max_validate_count is null then 0 else t4.max_validate_count end AS 'encryption_status' " \
+                       "FROM longvideo.wx_video t1 " \
+                       "LEFT JOIN longvideo.wx_video_status t2 ON t1.id= t2.video_id " \
+                       "LEFT JOIN longvideo.wx_video_detail t3 ON t1.id= t3.video_id " \
+                       "LEFT JOIN longvideo.wx_video_pwd t4 ON t1.id= t4.video_id"
+
     if len(video_ids) == 1:
-        sql = "set hg_experimental_enable_shard_pruning=off; " \
-              "SELECT video_id " \
-              "FROM {} " \
+        sql = "SELECT video_id " \
+              "FROM ({}) " \
               "WHERE audit_status = 5 " \
               "AND app_rec_status IN (1, -6, 10) " \
               "AND open_status = 1 " \
               "AND payment_status = 0 " \
               "AND encryption_status != 5 " \
               "AND transcoding_status = 3 " \
-              "AND video_id IN ({});".format(config_.VIDEO_STATUS, video_ids[0])
+              "AND video_id IN ({});".format(video_status_sql, video_ids[0])
+        data = mysql_helper.get_data(sql=sql)
+
     else:
-        sql = "set hg_experimental_enable_shard_pruning=off; " \
-              "SELECT video_id " \
-              "FROM {} " \
-              "WHERE audit_status = 5 " \
-              "AND app_rec_status IN (1, -6, 10) " \
-              "AND open_status = 1 " \
-              "AND payment_status = 0 " \
-              "AND encryption_status != 5 " \
-              "AND transcoding_status = 3 " \
-              "AND video_id IN {};".format(config_.VIDEO_STATUS, tuple(video_ids))
+        data = []
+        for i in range(len(video_ids) // 2000 + 1):
+            sql = "SELECT video_id " \
+                  "FROM ({}) " \
+                  "WHERE audit_status = 5 " \
+                  "AND app_rec_status IN (1, -6, 10) " \
+                  "AND open_status = 1 " \
+                  "AND payment_status = 0 " \
+                  "AND encryption_status != 5 " \
+                  "AND transcoding_status = 3 " \
+                  "AND video_id IN {};".format(video_status_sql, tuple(video_ids[i*2000:(i+1)*2000]))
+            select_res = mysql_helper.get_data(sql=sql)
+            if select_res is not None:
+                data += select_res
 
-    hologres_helper = HologresHelper()
-    data = hologres_helper.get_data(sql=sql)
     filtered_videos = [int(temp[0]) for temp in data]
     return filtered_videos
 
@@ -219,7 +251,7 @@ def update_video_w_h_rate(video_ids, key_name):
     else:
         sql = "SELECT id, width, height, rotate FROM longvideo.wx_video WHERE id IN {};".format(tuple(video_ids))
 
-    mysql_helper = MysqlHelper()
+    mysql_helper = MysqlHelper(mysql_info=config_.MYSQL_INFO)
     data = mysql_helper.get_data(sql=sql)
 
     # 更新到redis