feat:修改广告脚本

.gitignore

@@ -41,4 +41,5 @@ xxl-job
 logs
 
 model
-predict
+predict
+.idea

.idea/.gitignore

@@ -1,8 +0,0 @@
-# Default ignored files
-/shelf/
-/workspace.xml
-# Editor-based HTTP Client requests
-/httpRequests/
-# Datasource local storage ignored files
-/dataSources/
-/dataSources.local.xml

.idea/codeStyles/Project.xml

@@ -1,7 +0,0 @@
-<component name="ProjectCodeStyleConfiguration">
-  <code_scheme name="Project" version="173">
-    <ScalaCodeStyleSettings>
-      <option name="MULTILINE_STRING_CLOSING_QUOTES_ON_NEW_LINE" value="true" />
-    </ScalaCodeStyleSettings>
-  </code_scheme>
-</component>

.idea/codeStyles/codeStyleConfig.xml

@@ -1,5 +0,0 @@
-<component name="ProjectCodeStyleConfiguration">
-  <state>
-    <option name="PREFERRED_PROJECT_CODE_STYLE" value="Default" />
-  </state>
-</component>

.idea/compiler.xml

@@ -1,18 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="CompilerConfiguration">
-    <annotationProcessing>
-      <profile name="Maven default annotation processors profile" enabled="true">
-        <sourceOutputDir name="target/generated-sources/annotations" />
-        <sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
-        <outputRelativeToContentRoot value="true" />
-        <module name="spark-examples" />
-      </profile>
-    </annotationProcessing>
-  </component>
-  <component name="JavacSettings">
-    <option name="ADDITIONAL_OPTIONS_OVERRIDE">
-      <module name="spark-examples" options="-parameters" />
-    </option>
-  </component>
-</project>

.idea/encodings.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="Encoding">
-    <file url="file://$PROJECT_DIR$/src/main/java" charset="UTF-8" />
-  </component>
-</project>

.idea/inspectionProfiles/profiles_settings.xml

@@ -1,6 +0,0 @@
-<component name="InspectionProjectProfileManager">
-  <settings>
-    <option name="USE_PROJECT_PROFILE" value="false" />
-    <version value="1.0" />
-  </settings>
-</component>

.idea/jarRepositories.xml

@@ -1,45 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="RemoteRepositoriesConfiguration">
-    <remote-repository>
-      <option name="id" value="public" />
-      <option name="name" value="public" />
-      <option name="url" value="http://nexus.stuuudy.com:9580/nexus/content/groups/public/" />
-    </remote-repository>
-    <remote-repository>
-      <option name="id" value="central" />
-      <option name="name" value="Central Repository" />
-      <option name="url" value="https://repo.maven.apache.org/maven2" />
-    </remote-repository>
-    <remote-repository>
-      <option name="id" value="public" />
-      <option name="name" value="Public Repositories" />
-      <option name="url" value="http://nexus.stuuudy.com:9580/nexus/content/groups/public/" />
-    </remote-repository>
-    <remote-repository>
-      <option name="id" value="central" />
-      <option name="name" value="Maven Central repository" />
-      <option name="url" value="https://repo1.maven.org/maven2" />
-    </remote-repository>
-    <remote-repository>
-      <option name="id" value="maven-net-cloudera" />
-      <option name="name" value="cloudera Mirror" />
-      <option name="url" value="https://repository.cloudera.com/content/repositories/releases/" />
-    </remote-repository>
-    <remote-repository>
-      <option name="id" value="jboss.community" />
-      <option name="name" value="JBoss Community repository" />
-      <option name="url" value="https://repository.jboss.org/nexus/content/repositories/public/" />
-    </remote-repository>
-    <remote-repository>
-      <option name="id" value="stuuudy" />
-      <option name="name" value="stuuudy" />
-      <option name="url" value="http://nexus.stuuudy.com:9580/nexus/content/repositories/stuuudy/" />
-    </remote-repository>
-    <remote-repository>
-      <option name="id" value="releases" />
-      <option name="name" value="Releases" />
-      <option name="url" value="http://nexus.stuuudy.com:9580/nexus/content/repositories/releases/" />
-    </remote-repository>
-  </component>
-</project>

.idea/recommend-emr-dataprocess.iml

@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module type="PYTHON_MODULE" version="4">
-  <component name="NewModuleRootManager">
-    <content url="file://$MODULE_DIR$" />
-    <orderEntry type="inheritedJdk" />
-    <orderEntry type="sourceFolder" forTests="false" />
-  </component>
-</module>

.idea/scala_compiler.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ScalaCompilerConfiguration">
-    <profile name="Maven 1" modules="spark-examples" />
-  </component>
-</project>

.idea/vcs.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="VcsDirectoryMappings">
-    <mapping directory="" vcs="Git" />
-  </component>
-</project>

ad/01_ad_model_update.sh

@@ -62,6 +62,7 @@ check_run_status() {
     local status=$1
     local step_start_time=$2
     local step_name=$3
+    local msg=$4
 
     local step_end_time=$(date +%s)
     local step_elapsed=$(($step_end_time - $step_start_time))
@@ -214,7 +215,7 @@ xgb_train() {
   eta:0.01 gamma:0.0 max_depth:5 num_round:1000 num_worker:30 repartition:20
 
   local return_code=$?
-  check_run_status $return_code $step_start_time "XGB模型训练任务"
+  check_run_status $return_code $step_start_time "XGB模型训练任务" "XGB模型训练失败"
 }
 
 calc_model_predict() {
@@ -244,14 +245,14 @@ calc_model_predict() {
   done < "${predict_analyse_file_path}"
 
   local return_code=$?
-  check_run_status $return_code $step_start_time "计算Top10差异"
+  check_run_status $return_code $step_start_time "计算Top10差异" "计算Top10差异异常"
 
   old_incr_rate_avg=$( echo "scale=6; ${old_total_diff} / ${count}" | bc -l )
-  check_run_status $? $step_start_time "计算老模型Top10差异"
+  check_run_status $? $step_start_time "计算老模型Top10差异" "计算老模型Top10差异异常"
 
 
   new_incr_rate_avg=$( echo "scale=6; ${new_total_diff} / ${count}" | bc -l )
-  check_run_status $? $step_start_time "计算新模型Top10差异"
+  check_run_status $? $step_start_time "计算新模型Top10差异" "计算新模型Top10差异异常"
 
   echo "老模型Top10差异平均值: ${old_incr_rate_avg}"
   echo "新模型Top10差异平均值: ${new_incr_rate_avg}"
@@ -283,17 +284,17 @@ model_predict() {
   modelPath:${online_model_path}
 
   local return_code=$?
-  check_run_status $return_code $step_start_time "线上模型评估${predict_date_path: -8}的数据"
+  check_run_status $return_code $step_start_time "线上模型评估${predict_date_path: -8}的数据" "线上模型评估${predict_date_path: -8}的数据失败"
 
   # 结果分析
   local python_return_code=$(python ${sh_path}/model_predict_analyse.py -p ${online_model_predict_result_path} ${new_model_predict_result_path} -f ${predict_analyse_file_path})
-  check_run_status $python_return_code $step_start_time "线上模型评估${predict_date_path: -8}的数据"
+  check_run_status $python_return_code $step_start_time "分析线上模型评估${predict_date_path: -8}的数据" "分析线上模型评估${predict_date_path: -8}的数据失败"
 
   calc_model_predict
 
   if (( $(echo "${new_incr_rate_avg} > 0.100000" | bc -l ) ));then 
-    check_run_status 1 $step_start_time "线上模型评估${predict_date_path: -8}的数据，绝对误差大于0.1，请检查"
     echo "线上模型评估${predict_date_path: -8}的数据，绝对误差大于0.1，请检查"
+    check_run_status 1 $step_start_time "${predict_date_path: -8}的数据，绝对误差大于0.1" "线上模型评估${predict_date_path: -8}的数据，绝对误差大于0.1，请检查"
     exit 1
   fi 
 }
@@ -307,7 +308,7 @@ model_upload_oss() {
     ${HADOOP} fs -get ${model_save_path} ${model_name}
     if [ ! -d ${model_name} ]; then
       echo "从HDFS下载模型失败"
-      check_run_status 1 $step_start_time "HDFS下载模型任务" 
+      check_run_status 1 $step_start_time "HDFS下载模型任务" "HDFS下载模型失败" 
       exit 1 
     fi
 
@@ -319,7 +320,7 @@ model_upload_oss() {
     
     ${HADOOP} fs -put ${model_name}.tar.gz ${MODEL_OSS_PATH}
     local return_code=$?
-    check_run_status $return_code $step_start_time "模型上传OSS任务"
+    check_run_status $return_code $step_start_time "模型上传OSS任务" "模型上传OSS失败"
 
 
     echo ${model_save_path} > ${model_path_file}
@@ -329,11 +330,10 @@ model_upload_oss() {
   )
 
   local return_code=$?
-  check_run_status $return_code $step_start_time "模型上传OSS任务"
+  check_run_status $return_code $step_start_time "模型上传OSS任务" "模型上传OSS失败"
 
   local step_end_time=$(date +%s)
   local elapsed=$(($step_end_time - $start_time))
-
   echo -e "$LOG_PREFIX -- 模型更新完成 -- 模型更新成功: 耗时 $elapsed"
   
   send_success_upload_msg

ad/ad_monitor_util.py

@@ -69,7 +69,6 @@ def _monitor(level, msg: str, start, elapsed, top10):
         return
 
     msg = msg.replace("\\n", "\n").replace("\\t", "\t")
-    top10 = top10.replace("\\n", "\n").replace("\\t", "\t")
     mgs_text = f"- 当前时间: {now.strftime('%Y-%m-%d %H:%M:%S')}" \
                f"\n- 任务开始时间: {timestamp_format(start)}" \
                f"\n- 任务状态: {level_task_status_map[level]}" \
@@ -96,31 +95,31 @@ def _monitor(level, msg: str, start, elapsed, top10):
             ]
         }
     }
-
-    collapsible_panel = {
-        "tag": "collapsible_panel",
-        "header": {
-            "title": {
-                "tag": "markdown",
-                "content": "**Top10差异详情**"
+    if top10 is not None:
+        collapsible_panel = {
+            "tag": "collapsible_panel",
+            "header": {
+                "title": {
+                    "tag": "markdown",
+                    "content": "**Top10差异详情**"
+                },
+                "vertical_align": "center",
+                "padding": "4px 0px 4px 8px"
             },
-            "vertical_align": "center",
-            "padding": "4px 0px 4px 8px"
-        },
-        "border": {
-            "color": "grey",
-            "corner_radius": "5px"
-        },
-        "element_id": "detail",
-        "elements": [
-            {
-                "tag": "markdown",
-                "content": top10,
-                "element_id": "Top10CID"
-            }
-        ]
-    }
-    card_json['body']['elements'].append(collapsible_panel)
+            "border": {
+                "color": "grey",
+                "corner_radius": "5px"
+            },
+            "element_id": "detail",
+            "elements": [
+                {
+                    "tag": "markdown",
+                    "content": top10.replace("\\n", "\n").replace("\\t", "\t"),
+                    "element_id": "Top10CID"
+                }
+            ]
+        }
+        card_json['body']['elements'].append(collapsible_panel)
 
     send_card_msg_to_feishu(
         webhook=server_robot.get('webhook'),
@@ -128,6 +127,7 @@ def _monitor(level, msg: str, start, elapsed, top10):
     )
 
 
+
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='告警Utils')
     parser.add_argument('--level', type=str, help='通知级别, info, warn, error', required=True)