add speed

ad_out_v1_get_offline_score_item.py

@@ -1,5 +1,6 @@
 #coding utf-8
 import sys
+import json
 import datetime
 import traceback
 from threading import Timer
@@ -239,7 +240,7 @@ from candidate_item
             k = str(row['i_id'])
             item_features = get_item_features(row)
             item_h = lr_model.predict_h(item_features)
-            redis_helper.set_data_to_redis(f"{key_name_prefix}:{k}", item_h, expire_time)
+            #redis_helper.set_data_to_redis(f"{key_name_prefix}:{k}", item_h, expire_time)
             item_h_dict[k] = item_h
             mean_item_h += item_h
             count_item_h += 1
@@ -250,9 +251,9 @@ from candidate_item
     print(mean_item_h)
     print(count_item_h)
     k = 'mean'
-    redis_helper.set_data_to_redis(f"{key_name_prefix}:{k}", mean_item_h, expire_time)
-    # with open('{}.json'.format(key_name_prefix), 'w') as fout:
-    #     json.dump(item_h_dict, fout, indent=2, ensure_ascii=False, sort_keys=True)
+    #redis_helper.set_data_to_redis(f"{key_name_prefix}:{k}", mean_item_h, expire_time)
+    with open('{}.{}.json'.format(key_name_prefix, dt), 'w') as fout:
+         json.dump(item_h_dict, fout, indent=2, ensure_ascii=False, sort_keys=True)
 
 
 def timer_check(dt):

ad_out_v1_get_offline_score_item.sh

@@ -2,9 +2,9 @@ source /etc/profile
 echo $ROV_OFFLINE_ENV
 if [[ $ROV_OFFLINE_ENV == 'test' ]]; then
     cd /data2/rov-offline &&
-    /root/anaconda3/bin/python /data2/rov-offline/ad_out_v1_get_offline_score_item.py $*
+    /root/anaconda3/bin/python /data2/rov-offline/ad_out_v1_get_offline_score_item_new.py $*
 elif [[ $ROV_OFFLINE_ENV == 'pro' ]]; then
     cd /data/rov-offline &&
-    /root/anaconda3/bin/python /data/rov-offline/ad_out_v1_get_offline_score_item.py $*
+    /root/anaconda3/bin/python /data/rov-offline/ad_out_v1_get_offline_score_item_new.py $*
 fi
 

ad_out_v1_get_offline_score_item_new.py

@@ -29,7 +29,7 @@ def process_and_store(row):
     k = str(row['k'])
     features = get_features(row)
     h = lr_model.predict_h(features)
-    redis_helper.set_data_to_redis(f"{key_name_prefix}:{k}", h, expire_time)
+    redis_helper.set_data_to_redis(f"{key_name_prefix}:{k}", round(h, 6), expire_time)
 
 def update_offline_score_item(dt):
     project = 'loghubods'
@@ -240,7 +240,7 @@ from candidate_item
     # log_.info(sql)
     records = exe_sql(project, sql)
     log_.info('sql_done')
-    records_process(records, process_and_store, max_size=100, num_workers=10)
+    records_process(records, process_and_store, max_size=50, num_workers=10)
 
 def timer_check(dt):
     try:

ad_out_v1_get_offline_score_user.py

@@ -1,5 +1,6 @@
 #coding utf-8
 import sys
+import json
 import datetime
 import traceback
 from threading import Timer
@@ -229,7 +230,6 @@ from candidate_user
     lr_model = LrModel('model/{}.json'.format(model_key))
     user_h_dict = {}
     k_col = 'u_id'
-    dt = datetime
     key_name_prefix = f"{config_.KEY_NAME_PREFIX_AD_OUT_MODEL_SCORE_USER}{model_key}"
     print(key_name_prefix)
     mean_user_h = 0.0
@@ -241,7 +241,7 @@ from candidate_user
             k = str(row['u_id'])
             user_features = get_user_features(row)
             user_h = lr_model.predict_h(user_features)
-            redis_helper.set_data_to_redis(f"{key_name_prefix}:{k}", user_h, expire_time)
+            # redis_helper.set_data_to_redis(f"{key_name_prefix}:{k}", user_h, expire_time)
             user_h_dict[k] = user_h
             mean_user_h += user_h
             count_user_h += 1
@@ -252,9 +252,9 @@ from candidate_user
     print(mean_user_h)
     print(count_user_h)
     k = 'mean'
-    redis_helper.set_data_to_redis(f"{key_name_prefix}:{k}", mean_user_h, expire_time)
-    # with open('{}.json'.format(key_name_prefix), 'w') as fout:
-    #     json.dump(user_h_dict, fout, indent=2, ensure_ascii=False, sort_keys=True)
+    #redis_helper.set_data_to_redis(f"{key_name_prefix}:{k}", mean_user_h, expire_time)
+    with open('{}.{}.json'.format(key_name_prefix, dt), 'w') as fout:
+        json.dump(user_h_dict, fout, indent=2, ensure_ascii=False, sort_keys=True)
 
 
 def timer_check(dt):

ad_out_v1_get_offline_score_user.sh

@@ -2,9 +2,9 @@ source /etc/profile
 echo $ROV_OFFLINE_ENV
 if [[ $ROV_OFFLINE_ENV == 'test' ]]; then
     cd /data2/rov-offline &&
-    /root/anaconda3/bin/python /data2/rov-offline/ad_out_v1_get_offline_score_user.py $*
+    /root/anaconda3/bin/python /data2/rov-offline/ad_out_v1_get_offline_score_user_new.py $*
 elif [[ $ROV_OFFLINE_ENV == 'pro' ]]; then
     cd /data/rov-offline &&
-    /root/anaconda3/bin/python /data/rov-offline/ad_out_v1_get_offline_score_user.py $*
+    /root/anaconda3/bin/python /data/rov-offline/ad_out_v1_get_offline_score_user_new.py $*
 fi
 

ad_out_v1_get_offline_score_user_new.py

@@ -29,7 +29,7 @@ def process_and_store(row):
     k = str(row['k'])
     features = get_features(row)
     h = lr_model.predict_h(features)
-    redis_helper.set_data_to_redis(f"{key_name_prefix}:{k}", h, expire_time)
+    redis_helper.set_data_to_redis(f"{key_name_prefix}:{k}", round(h, 6), expire_time)
 
 def update_offline_score_user(dt):
     project = 'loghubods'