update

calCtr.py

@@ -55,7 +55,7 @@ if __name__=="__main__":
     for k, v in data_dict.items():
         #print(v)
         return_users = v[6]
-        print(return_users)
+        #print(return_users)
         view_users = v[0]
         view_pv = v[1]
         share_pv = v[5]

calI2I.py

@@ -102,8 +102,17 @@ if __name__=="__main__":
     final_rec_list = []
     #f = open("rec_result", "w")
     #5. sorted item_list
+    
     for k,v in rec_item_dict.items():
-        sorted_v = sorted(v, key=itemgetter(1), reverse=True)
+        v_set = set('')
+        value_list = v
+        dup_list = []
+        for item in value_list:
+            if item[0] in v_set:
+                continue
+            v_set.add(item[0])
+            dup_list.append(item)
+        sorted_v = sorted(dup_list, key=itemgetter(1), reverse=True)
         final_rec_list.append((k, sorted_v))
     #print(final_rec_list[:1])
     #json_str = json.dumps(final_rec_list)

clean.sh

@@ -0,0 +1,4 @@
+ #!/bin/bash
+source ~/.bash_profile
+source ~/.bashrc
+

config.py

@@ -272,9 +272,90 @@ class TestConfig(BaseConfig):
     # # data 上传oss 目标Bucket指定目录
     # OSS_FOLDER_DATA = 'rov-offline/test/data/'
 
+class ProductionConfig(BaseConfig):
+    """生产环境配置"""
+    # 报警内容 环境区分
+    ENV_TEXT = "生产环境"
+    # 项目存放目录
+    PROJECT_PATH = '/data/rov-offline'
+
+    # 线上环境redis地址
+    REDIS_INFO = {
+        'host': 'r-bp1fogs2mflr1ybfot.redis.rds.aliyuncs.com',
+        'port': 6379,
+        'password': 'Wqsd@2019',
+    }
+
+    # Hologres连接参数，服务器使用
+    HOLOGRES_INFO = {
+        'host': 'hgprecn-cn-7pp28y18c00c-cn-hangzhou-vpc.hologres.aliyuncs.com',
+        'port': 80,
+        'dbname': 'dssm',
+        'user': 'LTAI5tMPqPy9yboQAf1mBCCN',
+        'password': '4BEcOgxREOPq7t3A7EWkjciVULeQGj'
+    }
+
+    # 生产环境mysql地址
+    MYSQL_INFO = {
+        'host': 'rr-bp1x9785e8h5452bi157.mysql.rds.aliyuncs.com',
+        'port': 3306,
+        'user': 'wx2016_longvideo',
+        'password': 'wx2016_longvideoP@assword1234',
+        'db': 'longvideo',
+        'charset': 'utf8'
+    }
+
+    # 生产环境 过滤用mysql地址
+    FILTER_MYSQL_INFO = {
+        'host': 'am-bp15tqt957i3b3sgi131950.ads.aliyuncs.com',
+        'port': 3306,
+        'user': 'lv_manager',
+        'password': 'lv_manager@2020',
+        'db': 'longvideo',
+        'charset': 'utf8'
+    }
+
+    # 日志服务配置
+    ALIYUN_LOG = {
+        'ENDPOINT': 'cn-hangzhou.log.aliyuncs.com',
+        'ACCESSID': 'LTAIWYUujJAm7CbH',
+        'ACCESSKEY': 'RfSjdiWwED1sGFlsjXv0DlfTnZTG1P',
+        'PROJECT': 'rov-server',
+    }
+
+    # Hologres视频状态存储表名
+    VIDEO_STATUS = 'longvideo.dwd_mdm_item_video_stat'
+
+    # 快速曝光流量池ID
+    QUICK_FLOW_POOL_ID = 3
+
+    # 获取流量池分发配置接口地址
+    GET_FLOW_POOL_RECOMMEND_CONFIG_URL = 'http://recommend-common-internal.piaoquantv.com/longvideoapi/openapi/recommend/getConfig'
+    # 从流量池获取视频接口地址
+    GET_VIDEOS_FROM_POOL_URL = 'http://api-internal.piaoquantv.com/flowpool/video/getAllVideo'
+    # 获取视频在流量池中的剩余可分发数接口地址
+    GET_REMAIN_VIEW_COUNT_URL = 'http://api-internal.piaoquantv.com/flowpool/video/remainViewCount'
+    # 计算完ROV通知后端接口地址
+    NOTIFY_BACKEND_UPDATE_ROV_SCORE_URL = 'http://recommend-common-internal.piaoquantv.com/longvideoapi/openapi/recommend/updateRovScore'
+    # 获取置顶视频列表接口地址
+    TOP_VIDEO_LIST_URL = 'http://recommend-common-internal.piaoquantv.com/longvideoapi/openapi/recommend/topVideoList'
+    # 获取首页兜底视频json接口地址
+    BOTTOM_JSON_URL = 'http://recommend-common-internal.piaoquantv.com/longvideoapi/openapi/video/distribute/structure/video/list'
+    # 通知后端更新兜底视频接口地址
+    NOTIFY_BACKEND_updateFallBackVideoList_URL = 'http://recommend-common-internal.piaoquantv.com/longvideoapi/openapi/recommend/updateFallBackVideoList'
+    # 获取限流视频接口地址
+    GET_VIDEO_LIMIT_LIST_URL = 'http://recommend-common-internal.piaoquantv.com/longvideoapi/openapi/recommend/getVideoLimitList'
+    # 获取管理后台设置的广告目标uv值接口地址
+    GET_AD_TARGET_UV_URL = 'https://admin.piaoquantv.com/manager/ad/algo/threshold/productUvTargetList'
+
+    # # logs 上传oss 目标Bucket指定目录
+    # OSS_FOLDER_LOGS = 'rov-offline/pro/logs/'
+    # # data 上传oss 目标Bucket指定目录
+    # OSS_FOLDER_DATA = 'rov-offline/pro/data/'
+
 def set_config():
     # 获取环境变量 ROV_OFFLINE_ENV
     env = os.environ.get('Base_ENV')
-    return TestConfig()
+    return ProductionConfig()
 
 

import_redist.py

@@ -4,6 +4,7 @@ import json
 from db_help import RedisHelper
 
 if __name__=="__main__":
+     f2 = open(sys.argv[2], 'w')
      with  open(sys.argv[1]) as f:
          rec_json_list=json.load(f)
          #print(rec_json_list)
@@ -14,12 +15,16 @@ if __name__=="__main__":
              rec_item_list = []
              for rec_item in rec_list:
                  rec_item_list.append((rec_item[0], round(rec_item[1],3)))
-             print(line[0]+"\t"+json.dumps(rec_item_list))
+             res_info = json.dumps(rec_item_list)
+             
+             f2.write(str(line[0])+"\t"+res_info+"\n")
              key="sim_hot_"+line[0]
-             import_data_dict[key] = json.dumps(rec_item_list)
+             import_data_dict[key] = res_info
          redis_helper = RedisHelper()
          redis_helper.update_batch_setnx_key(import_data_dict, 60*60*24*7)
          #con = redis_helper.connect()
          res = redis_helper.get_data_from_redis("sim_hot_14330133")
          print(res)
+         f2.close()
+     f.close()
    

run.sh

@@ -1,8 +1,9 @@
 #!/bin/bash
 source ~/.bash_profile
 source ~/.bashrc
+conda activate base 
 
-conda activate python36 
+cd /data/rec_project/OffLineRec
 
 #1. download data
 nowday=`date  +"%Y%m%d" -d -0days`
@@ -15,27 +16,28 @@ python extract_share_log.py ${last7day} ${nowday}
 if [ $? -ne 0 ];
 then
     msg = "[ERROR] simrecall extract_share_log"
-    sh sendmsg.sh  $nowday  $msg
+    #sh sendmsg.sh  $nowday  $msg
     echo "[ERROR] echo 'extract_share_log"
     exit 255
 fi
 #nowday='20230505'
 #2. cal i2i result
 python calI2I.py ${nowday}
+if [ $? -ne 0 ];
 then
     msg = "[ERROR] simrecall calI2I.py"
-    sh sendmsg.sh $nowday $msg
+    #sh sendmsg.sh $nowday $msg
     echo $msg
     exit -1
 fi
 
 #3.import res
-python import_redist.py "./data/rec_result_"${nowday}".json"
+python import_redist.py "./data/rec_result_"${nowday}".json"  "./data/redis_cls_"${nowday}".json"
 if [ $? -ne 0 ];
 then
     msg = "[ERROR] simhot recall import_redist.py"
-    sh sendmsg.sh  $nowday  $msg
+    #sh sendmsg.sh  $nowday  $msg
     echo $msg
     exit -1
 fi
-echo "finish sorted"
+echo 'finish sorted'

run_ctr.sh

@@ -2,8 +2,8 @@
 source ~/.bash_profile
 source ~/.bashrc
 
-conda activate python36 
-
+conda activate base 
+cd /data/rec_project/OffLineRec
 #1. download data
 nowday=`date  +"%Y%m%d%H" -d -0days`
 echo ${nowday} 

test.py

@@ -0,0 +1,30 @@
+#coding utf-8
+import sys
+import json
+from db_help import RedisHelper
+
+if __name__=="__main__":
+     '''f2 = open(sys.argv[2], 'w')
+     with  open(sys.argv[1]) as f:
+         rec_json_list=json.load(f)
+         #print(rec_json_list)
+         import_data_dict = {}
+         for line in rec_json_list:
+             #print(line)
+             rec_list = line[1][:10]
+             rec_item_list = []
+             for rec_item in rec_list:
+                 rec_item_list.append((rec_item[0], round(rec_item[1],3)))
+             res_info = json.dumps(rec_item_list)
+             
+             f2.write(str(line[0])+"\t"+res_info+"\n")
+             key="sim_hot_"+line[0]
+             import_data_dict[key] = res_info'''
+     redis_helper = RedisHelper()
+         #redis_helper.update_batch_setnx_key(import_data_dict, 60*60*24*7)
+         #con = redis_helper.connect()
+     res = redis_helper.get_data_from_redis("sim_hot_14269966")
+     print(res)
+     #f2.close()
+     #f.close()
+   

test.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+source ~/.bash_profile
+source ~/.bashrc
+
+conda activate base 
+
+#1. download data
+nowday=`date  +"%Y%m%d" -d -0days`
+last7day=`date  +"%Y%m%d" -d -8days`
+echo ${nowday} 
+echo ${last7day}
+mkdir -p ./data/
+
+#python extract_share_log.py ${last7day} ${nowday}
+#if [ $? -ne 0 ];
+#then
+#    msg = "[ERROR] simrecall extract_share_log"
+#    sh sendmsg.sh  $nowday  $msg
+#    echo "[ERROR] echo 'extract_share_log"
+#    exit 255
+#fi
+#nowday='20230505'
+#2. cal i2i result
+#python calI2I.py ${nowday}
+#if [ $? -ne 0 ];
+#then
+#    msg = "[ERROR] simrecall calI2I.py"
+#    sh sendmsg.sh $nowday $msg
+#    echo $msg
+#    exit -1
+#fi
+
+#3.import res
+python import_redist.py "./data/rec_result_"${nowday}".json"  "./data/redis_cls_"${nowday}".json"
+#python test.py "./data/rec_result_"${nowday}".json"  './data/redis_cls_'${nowday}".json"
+#if [ $? -ne 0 ];
+#then
+#    msg = "[ERROR] simhot recall import_redist.py"
+#    sh sendmsg.sh  $nowday  $msg
+#    echo $msg
+    exit -1
+fi
+echo "finish sorted"