Bladeren bron

Update ad/01_ad_model_update: fix upload

root 1 maand geleden
bovenliggende
commit
c85fb6bbc4
1 gewijzigde bestanden met toevoegingen van 20 en 10 verwijderingen
  1. 20 10
      ad/01_ad_model_update.sh

+ 20 - 10
ad/01_ad_model_update.sh

@@ -365,20 +365,27 @@ model_upload_oss() {
 
     rm -rf ${model_name}.tar.gz.crc
 
-    # 从OSS中移除模型文件和校准文件
-    ${HADOOP} fs -rm -r -skipTrash ${MODEL_OSS_PATH}/${model_name}.tar.gz ${MODEL_OSS_PATH}/${OSS_CALIBRATION_FILE_NAME}.txt
-    
     # 将模型文件和校准文件推送到OSS上
-    ${HADOOP} fs -put ${model_name}.tar.gz ${OSS_CALIBRATION_FILE_NAME}.txt ${MODEL_OSS_PATH}
+    ${HADOOP} fs -put -f ${model_name}.tar.gz ${MODEL_OSS_PATH}/${model_name}_tmp.tar.gz
     local return_code=$?
     check_run_status ${return_code} ${step_start_time} "模型上传OSS任务" "模型上传OSS失败"
 
-    echo ${model_save_path} > ${model_path_file}
+    ${HADOOP} fs -put -f ${OSS_CALIBRATION_FILE_NAME}.txt ${MODEL_OSS_PATH}/${OSS_CALIBRATION_FILE_NAME}_tmp.txt
+    local return_code=$?
+    check_run_status ${return_code} ${step_start_time} "模型上传OSS任务" "模型上传OSS失败"
+
+    # 移动模型文件
+    # FIXME(zhoutian): 两步文件移动未保证事务
+    ${HADOOP} fs -mv ${MODEL_OSS_PATH}/${model_name}_tmp.tar.gz ${MODEL_OSS_PATH}/${model_name}.tar.gz
+    local return_code=$?
+    check_run_status ${return_code} ${step_start_time} "模型上传OSS任务" "模型OSS重命名失败"
+
+    ${HADOOP} fs -mv ${MODEL_OSS_PATH}/${OSS_CALIBRATION_FILE_NAME}_tmp.txt ${MODEL_OSS_PATH}/${OSS_CALIBRATION_FILE_NAME}.txt
+    local return_code=$?
+    check_run_status ${return_code} ${step_start_time} "模型上传OSS任务" "模型OSS重命名失败"
 
-    # 
-    rm -f ./${model_name}.tar.gz
-    rm -rf ./${model_name}
-    rm -rf ${OSS_CALIBRATION_FILE_NAME}.txt
+
+    echo ${model_save_path} > ${model_path_file}
   )
 
   local return_code=$?
@@ -389,6 +396,9 @@ model_upload_oss() {
   echo -e "${LOG_PREFIX} -- 模型更新完成 -- 模型更新成功: 耗时 ${elapsed}"
   
   send_success_upload_msg
+  rm -f ./${model_name}.tar.gz
+  rm -rf ./${model_name}
+  rm -f ${OSS_CALIBRATION_FILE_NAME}.txt
 }
 
 # 主方法
@@ -416,4 +426,4 @@ main() {
 }
 
 
-main
+main