Bladeren bron

fix:模型训练与预测时的hdfs路径通配符

zhaohaipeng 9 maanden geleden
bovenliggende
commit
820bc9ea67
1 gewijzigde bestanden met toevoegingen van 5 en 5 verwijderingen
  1. 5 5
      ad/02_ad_model_update_twice_daily.sh

+ 5 - 5
ad/02_ad_model_update_twice_daily.sh

@@ -58,8 +58,8 @@ global_init() {
         trainBucketFeaturePath=${bucketFeatureSavePathHome}/${today_early_1}/train
         predictBucketFeaturePath=${bucketFeatureSavePathHome}/${today_early_1}/predict
 
-        local_model_file_path=${MODEL_HOME}/${train_end_str}.txt
-        local_change_model_file_path=${MODEL_HOME}/${train_end_str}_change.txt
+        local_model_file_path=${MODEL_HOME}/${model_name}_${train_end_str}.txt
+        local_change_model_file_path=${MODEL_HOME}/${model_name}_${train_end_str}_change.txt
         max_hour=12
     # elif [ $current_hour -ge 20 ]; then 
     #     train_begin_str=${today_early_1}22
@@ -193,7 +193,7 @@ make_bucket_feature() {
 # 模型训练
 model_train() {
     local step_start_time=$(date +%s)
-    $HADOOP fs -text ${trainBucketFeaturePath}/* | ${FM_HOME}/bin/fm_train -m ${local_model_file_path} -dim 1,1,8  -im ${LAST_MODEL_HOME}/model_online.txt -core 8
+    $HADOOP fs -text ${trainBucketFeaturePath}/*/* | ${FM_HOME}/bin/fm_train -m ${local_model_file_path} -dim 1,1,8  -im ${LAST_MODEL_HOME}/model_online.txt -core 8
 
     local return_code=$?
     check_run_status $return_code $step_start_time "模型训练"
@@ -205,7 +205,7 @@ auc_compare() {
 
     # 5.1 计算线上模型的AUC
     local step_start_time=$(date +%s)
-    $HADOOP fs -text ${predictBucketFeaturePath}/* | ${FM_HOME}/bin/fm_predict -m ${LAST_MODEL_HOME}/model_online.txt -dim 8 -core 8 -out ${PREDICT_PATH}/${model_name}_${train_end_str}_online.txt
+    $HADOOP fs -text ${predictBucketFeaturePath}/*/* | ${FM_HOME}/bin/fm_predict -m ${LAST_MODEL_HOME}/model_online.txt -dim 8 -core 8 -out ${PREDICT_PATH}/${model_name}_${train_end_str}_online.txt
     online_auc=`cat ${PREDICT_PATH}/${model_name}_${train_end_str}_online.txt | /root/sunmingze/AUC/AUC`
     
     local return_code=$?
@@ -213,7 +213,7 @@ auc_compare() {
 
     # 5.2 计算新模型的AUC
     step_start_time=$(date +%s)
-    $HADOOP fs -text ${predictBucketFeaturePath}/* | ${FM_HOME}/bin/fm_predict -m ${local_model_file_path} -dim 8 -core 8 -out ${PREDICT_PATH}/${model_name}_${train_end_str}_new.txt
+    $HADOOP fs -text ${predictBucketFeaturePath}/*/* | ${FM_HOME}/bin/fm_predict -m ${local_model_file_path} -dim 8 -core 8 -out ${PREDICT_PATH}/${model_name}_${train_end_str}_new.txt
     new_auc=`cat ${PREDICT_PATH}/${model_name}_${train_end_str}_new.txt | /root/sunmingze/AUC/AUC`
 
     return_code=$?