Merge branch 'feature/20241112-3rd-gh-update' of algorithm/rov-offline into master

alg_growth_3rd_gh_reply_video_v1.py

@@ -29,24 +29,6 @@ GH_IDS = ('default',)
 
 pd.set_option('display.max_rows', None)
 
-TARGET_GH_IDS = (
-    'gh_250c51d5ce69',
-    'gh_8a29eebc2012',
-    'gh_ff16c412ab97',
-    'gh_1014734791e0',
-    'gh_570967881eae',
-    'gh_a7c21403c493',
-    'gh_7f062810b4e7',
-    'gh_c8060587e6d1',
-    'gh_1da8f62f4a0d',
-    'gh_56b65b7d4520',
-    'gh_eeec7c2e28a5',
-    'gh_7c89d5a3e745',
-    'gh_ee5b4b07ed8b',
-    'gh_0d3c97cc30cc',
-    'gh_c783350a9660',
-)
-
 CDN_IMG_OPERATOR = "?x-oss-process=image/resize,m_fill,w_600,h_480,limit_0/format,jpg/watermark,image_eXNoL3BpYy93YXRlcm1hcmtlci9pY29uX3BsYXlfd2hpdGUucG5nP3gtb3NzLXByb2Nlc3M9aW1hZ2UvcmVzaXplLHdfMTQ0,g_center"
 
 ODS_PROJECT = "loghubods"
@@ -276,33 +258,6 @@ def postprocess_override_by_config(df, dt_version):
     return df
 
 
-def rank_for_base_designate(run_dt, run_hour, stg_key):
-    dt_version = f'{run_dt}{run_hour}'
-    ranked_df = pd.DataFrame()  # 初始化一个空的 DataFrame
-
-    for gh_id in GH_IDS:
-        if gh_id in TARGET_GH_IDS:
-            temp_df = pd.DataFrame({
-                'strategy_key': [stg_key],
-                'dt_version': [dt_version],
-                'gh_id': [gh_id],
-                'sort': [1],
-                'video_id': [13586800],
-                'score': [0.5]
-            })
-        else:
-            temp_df = pd.DataFrame({
-                'strategy_key': [stg_key],
-                'dt_version': [dt_version],
-                'gh_id': [gh_id],
-                'sort': [1],
-                'video_id': [20463342],
-                'score': [0.5]
-            })
-        ranked_df = pd.concat([ranked_df, temp_df], ignore_index=True)
-    return ranked_df
-
-
 def build_and_transfer_data(run_dt, run_hour, project, **kwargs):
     dt_version = f'{run_dt}{run_hour}'
     dry_run = kwargs.get('dry_run', False)
@@ -312,8 +267,6 @@ def build_and_transfer_data(run_dt, run_hour, project, **kwargs):
     layer1_rank = rank_for_layer1(run_dt, run_hour, ODS_PROJECT, EXPLORE_POOL_TABLE, gh_df)
     layer2_rank = rank_for_layer2(run_dt, run_hour, ODS_PROJECT, GH_REPLY_STATS_TABLE, ODPS_RANK_RESULT_TABLE)
     base_rank = rank_for_base(run_dt, run_hour, ODS_PROJECT, GH_REPLY_STATS_TABLE, ODPS_RANK_RESULT_TABLE,BASE_GROUP_NAME)
-    # layer2_rank = rank_for_base_designate(run_dt, run_hour, EXPLORE2_GROUP_NAME)
-    # base_rank = rank_for_base_designate(run_dt, run_hour, BASE_GROUP_NAME)
 
     final_rank_df = pd.concat([layer1_rank, layer2_rank, base_rank]).reset_index(drop=True)
 
@@ -349,9 +302,23 @@ def build_and_transfer_data(run_dt, run_hour, project, **kwargs):
     # sync to MySQL
     data_to_insert = [tuple(row) for row in final_df.itertuples(index=False)]
     data_columns = list(final_df.columns)
+    max_time_to_delete = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
     mysql = MysqlHelper(CONFIG.MYSQL_GROWTH_INFO)
     mysql.batch_insert(RDS_RANK_RESULT_TABLE, data_to_insert, data_columns)
 
+    # remove old data of same version
+    for key in final_df['strategy_key'].unique():
+        sql = f"""
+            update {RDS_RANK_RESULT_TABLE}
+            set is_delete = 1
+            where
+                dt_version = '{dt_version}'
+                and strategy_key = '{key}'
+                and create_time < '{max_time_to_delete}'
+                and is_delete = 0
+        """
+        rows = mysql.execute(sql)
+
 
 def main_loop():
     argparser = ArgumentParser()

configs/3rd_gh_reply_video.json

@@ -222,5 +222,305 @@
         "video_id": 13586800
       }
     ]
+  },
+  "gh_7ad83d502e65": {
+    "3rd-party-explore2": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ],
+    "3rd-party-base": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ],
+    "3rd-party-explore1": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ]
+  },
+  "gh_c0b8f8784c39": {
+    "3rd-party-explore2": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ],
+    "3rd-party-base": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ],
+    "3rd-party-explore1": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ]
+  },
+  "gh_58393bec96ac": {
+    "3rd-party-explore2": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ],
+    "3rd-party-base": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ],
+    "3rd-party-explore1": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ]
+  },
+  "gh_0f3a6904e63e": {
+    "3rd-party-explore2": [
+      {
+        "position": 1,
+        "video_id": 13586800
+      }
+    ],
+    "3rd-party-base": [
+      {
+        "position": 1,
+        "video_id": 13586800
+      }
+    ],
+    "3rd-party-explore1": [
+      {
+        "position": 1,
+        "video_id": 13586800
+      }
+    ]
+  },
+  "gh_f4b6411dc2bb": {
+    "3rd-party-explore2": [
+      {
+        "position": 1,
+        "video_id": 13586800
+      }
+    ],
+    "3rd-party-base": [
+      {
+        "position": 1,
+        "video_id": 13586800
+      }
+    ],
+    "3rd-party-explore1": [
+      {
+        "position": 1,
+        "video_id": 13586800
+      }
+    ]
+  },
+  "gh_de67bb330fb4": {
+    "3rd-party-explore2": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ],
+    "3rd-party-base": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ],
+    "3rd-party-explore1": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ]
+  },
+  "gh_1d224e04ef3a": {
+    "3rd-party-explore2": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ],
+    "3rd-party-base": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ],
+    "3rd-party-explore1": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ]
+  },
+  "gh_97e34f88b026": {
+    "3rd-party-explore2": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ],
+    "3rd-party-base": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ],
+    "3rd-party-explore1": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ]
+  },
+  "gh_e93128efc902": {
+    "3rd-party-explore2": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ],
+    "3rd-party-base": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ],
+    "3rd-party-explore1": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ]
+  },
+  "gh_e58851b7aead": {
+    "3rd-party-explore2": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ],
+    "3rd-party-base": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ],
+    "3rd-party-explore1": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ]
+  },
+  "gh_7e7db0dc9000": {
+    "3rd-party-explore2": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ],
+    "3rd-party-base": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ],
+    "3rd-party-explore1": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ]
+  },
+  "gh_531e2204df56": {
+    "3rd-party-explore2": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ],
+    "3rd-party-base": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ],
+    "3rd-party-explore1": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ]
+  },
+  "gh_5346f126194c": {
+    "3rd-party-explore2": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ],
+    "3rd-party-base": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ],
+    "3rd-party-explore1": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ]
+  },
+  "gh_7bcb1838784c": {
+    "3rd-party-explore2": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ],
+    "3rd-party-base": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ],
+    "3rd-party-explore1": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ]
+  },
+  "gh_5f36052d6818": {
+    "3rd-party-explore2": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ],
+    "3rd-party-base": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ],
+    "3rd-party-explore1": [
+      {
+        "position": 1,
+        "video_id": 20463342
+      }
+    ]
   }
 }

db_helper.py

@@ -362,6 +362,20 @@ class MysqlHelper(object):
         conn.close()
         return data
 
+    def execute(self, sql):
+        conn = pymysql.connect(**self.mysql_info)
+        cursor = conn.cursor()
+        try:
+            cursor.execute(sql)
+            affected_rows = cursor.rowcount
+            conn.commit()
+            return affected_rows
+        except Exception as e:
+            conn.rollback()
+            raise e
+        finally:
+            conn.close()
+
     def batch_insert(self, table, data, columns=None):
         """
         data: data, list[tuple] or list[dict]