|
@@ -365,20 +365,27 @@ model_upload_oss() {
|
|
|
|
|
|
rm -rf ${model_name}.tar.gz.crc
|
|
|
|
|
|
- # 从OSS中移除模型文件和校准文件
|
|
|
- ${HADOOP} fs -rm -r -skipTrash ${MODEL_OSS_PATH}/${model_name}.tar.gz ${MODEL_OSS_PATH}/${OSS_CALIBRATION_FILE_NAME}.txt
|
|
|
-
|
|
|
# 将模型文件和校准文件推送到OSS上
|
|
|
- ${HADOOP} fs -put ${model_name}.tar.gz ${OSS_CALIBRATION_FILE_NAME}.txt ${MODEL_OSS_PATH}
|
|
|
+ ${HADOOP} fs -put -f ${model_name}.tar.gz ${MODEL_OSS_PATH}/${model_name}_tmp.tar.gz
|
|
|
local return_code=$?
|
|
|
check_run_status ${return_code} ${step_start_time} "模型上传OSS任务" "模型上传OSS失败"
|
|
|
|
|
|
- echo ${model_save_path} > ${model_path_file}
|
|
|
+ ${HADOOP} fs -put -f ${OSS_CALIBRATION_FILE_NAME}.txt ${MODEL_OSS_PATH}/${OSS_CALIBRATION_FILE_NAME}_tmp.txt
|
|
|
+ local return_code=$?
|
|
|
+ check_run_status ${return_code} ${step_start_time} "模型上传OSS任务" "模型上传OSS失败"
|
|
|
+
|
|
|
+ # 移动模型文件
|
|
|
+ # FIXME(zhoutian): 两步文件移动未保证事务
|
|
|
+ ${HADOOP} fs -mv ${MODEL_OSS_PATH}/${model_name}_tmp.tar.gz ${MODEL_OSS_PATH}/${model_name}.tar.gz
|
|
|
+ local return_code=$?
|
|
|
+ check_run_status ${return_code} ${step_start_time} "模型上传OSS任务" "模型OSS重命名失败"
|
|
|
+
|
|
|
+ ${HADOOP} fs -mv ${MODEL_OSS_PATH}/${OSS_CALIBRATION_FILE_NAME}_tmp.txt ${MODEL_OSS_PATH}/${OSS_CALIBRATION_FILE_NAME}.txt
|
|
|
+ local return_code=$?
|
|
|
+ check_run_status ${return_code} ${step_start_time} "模型上传OSS任务" "模型OSS重命名失败"
|
|
|
|
|
|
- #
|
|
|
- rm -f ./${model_name}.tar.gz
|
|
|
- rm -rf ./${model_name}
|
|
|
- rm -rf ${OSS_CALIBRATION_FILE_NAME}.txt
|
|
|
+
|
|
|
+ echo ${model_save_path} > ${model_path_file}
|
|
|
)
|
|
|
|
|
|
local return_code=$?
|
|
@@ -389,6 +396,9 @@ model_upload_oss() {
|
|
|
echo -e "${LOG_PREFIX} -- 模型更新完成 -- 模型更新成功: 耗时 ${elapsed}"
|
|
|
|
|
|
send_success_upload_msg
|
|
|
+ rm -f ./${model_name}.tar.gz
|
|
|
+ rm -rf ./${model_name}
|
|
|
+ rm -f ${OSS_CALIBRATION_FILE_NAME}.txt
|
|
|
}
|
|
|
|
|
|
# 主方法
|
|
@@ -416,4 +426,4 @@ main() {
|
|
|
}
|
|
|
|
|
|
|
|
|
-main
|
|
|
+main
|