add

qiaojialiang/checkHiveDataUtil.py

@@ -34,7 +34,7 @@ def check_origin_hive(args):
         bot = FeishuBot()
         msg = (
             f'推荐模型数据更新 \n【任务名称】:step1校验hive数据源\n【是否成功】:success\n【信息】:table:{table},beginStr:{beginStr},endStr:{endStr}\n【详细日志】:{exist_partition}')
-        bot.send_message(msg)
+        # bot.send_message(msg)
         print('0')
 
 

qiaojialiang/handle_rov.sh

@@ -2,7 +2,8 @@
 set -ex
 
 # 原始数据table name
-table='alg_recsys_sample_all'
+#table='alg_recsys_sample_all'
+table='alg_recsys_sample_all_test'
 # 处理分区配置
 beginStr="$(date -d '1 days ago' +%Y%m%d)"
 endStr="$(date -d '1 days ago' +%Y%m%d)"
@@ -36,9 +37,9 @@ echo "----------step2------------开始根据${table}生产原始数据"
 /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
 --class com.aliyun.odps.spark.examples.makedata_qiao.makedata_13_originData_20240705 \
 --master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
-./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
 tablePart:64 repartition:32 \
-beginStr:${beginStr} endStr:${endStr} \
+beginStr:${beginStr}${beginHhStr} endStr:${endStr}${endHhStr} \
 savePath:/dw/recommend/model/13_sample_data/ \
 table:${table}
 if [ $? -ne 0 ]; then
@@ -53,7 +54,7 @@ fi
 /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
 --class com.aliyun.odps.spark.examples.makedata_qiao.makedata_14_valueData_20240705 \
 --master yarn --driver-memory 1G --executor-memory 3G --executor-cores 1 --num-executors 32 \
-./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
 readPath:/dw/recommend/model/13_sample_data/ \
 savePath:/dw/recommend/model/14_feature_data/ \
 beginStr:${beginStr} endStr:${endStr} repartition:1000
@@ -69,7 +70,7 @@ echo "----------step3------------根据特征分桶生产重打分特征数据"
 /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
 --class com.aliyun.odps.spark.examples.makedata_qiao.makedata_16_bucketData_20240705 \
 --master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
-./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
 beginStr:${beginStr} endStr:${endStr} repartition:1000
 if [ $? -ne 0 ]; then
    echo "Spark特征分桶处理任务执行失败"