feat:合并recommend

qiaojialiang/test/demo01.py

@@ -1,41 +0,0 @@
-import subprocess
-from datetime import datetime
-
-# 获取当前日期，并格式化为字符串
-now = datetime.now()
-date_str = now.strftime("%Y-%m-%d")
-
-# 定义日志文件名
-log_file = f"script_output_{date_str}.log"
-
-n1 = 1
-n2 = 5
-
-# 定义Shell脚本的路径
-script1_path = "script1.sh"
-script2_path = "script2.sh"
-
-# 打开日志文件准备写入
-with open(log_file, 'w') as f:
-    # 调用script1.sh，不使用shell=True
-    result1 = subprocess.run([script1_path, str(n1), str(n2)], stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                             text=True)
-
-    # 检查script1.sh的返回码，并写入输出到日志文件
-    if result1.returncode == 0 and not result1.stderr:
-        print("script1.sh 执行成功")
-        f.write(f"--- script1.sh 输出 ---\n{result1.stdout}\n")
-
-        # 调用script2.sh，同样不使用shell=True
-        result2 = subprocess.run([script2_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
-
-        # 检查script2.sh的返回码，并写入输出到日志文件
-        if result2.returncode == 0 and not result2.stderr:
-            print("script2.sh 执行成功")
-            f.write(f"--- script2.sh 输出 ---\n{result2.stdout}\n")
-        else:
-            print("script2.sh 执行失败")
-            f.write(f"--- script2.sh 错误输出 ---\n{result2.stderr}\n")
-    else:
-        print("script1.sh 执行失败")
-        f.write(f"--- script1.sh 错误输出 ---\n{result1.stderr}\n")

qiaojialiang/test/handle_rov_bak.sh

@@ -1,202 +0,0 @@
-#!/bin/sh
-set -ex
-
-source /root/anaconda3/bin/activate py37
-
-#  nohup sh handle_rov.sh > "$(date +%Y%m%d_%H%M%S)_handle_rov.log" 2>&1 &
-
-# 原始数据table name
-table='alg_recsys_sample_all'
-today="$(date +%Y%m%d)"
-today_early_3="$(date -d '3 days ago' +%Y%m%d)"
-#table='alg_recsys_sample_all_test'
-# 处理分区配置 推荐数据间隔一天生产，所以5日0点使用3日0-23点数据生产new模型数据
-begin_early_2_Str="$(date -d '2 days ago' +%Y%m%d)"
-end_early_2_Str="$(date -d '2 days ago' +%Y%m%d)"
-beginHhStr=00
-endHhStr=23
-max_hour=05
-max_minute=00
-# 各节点产出hdfs文件绝对路径
-# 源数据文件
-originDataPath=/dw/recommend/model/13_sample_data/
-# 特征值
-valueDataPath=/dw/recommend/model/14_feature_data/
-# 特征分桶
-bucketDataPath=/dw/recommend/model/16_train_data/
-# 模型数据路径
-MODEL_PATH=/root/joe/recommend-emr-dataprocess/model
-# 预测路径
-PREDICT_PATH=/root/joe/recommend-emr-dataprocess/predict
-# 历史线上正在使用的模型数据路径
-LAST_MODEL_HOME=/root/joe/model_online
-# 模型数据文件前缀
-model_name=aka8
-# fm模型
-FM_HOME=/root/sunmingze/alphaFM/bin
-# hadoop
-HADOOP=/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop
-OSS_PATH=oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/zhangbo/
-
-# 0 判断上游表是否生产完成，最长等待到max_hour点
-# shellcheck disable=SC2154
-echo "$(date +%Y-%m-%d_%H-%M-%S)----------step0------------开始校验是否生产完数据，分区信息：beginStr:${begin_early_2_Str}${beginHhStr},endStr:${end_early_2_Str}${endHhStr}"
-while true; do
-  python_return_code=$(python /root/joe/recommend-emr-dataprocess/qiaojialiang/checkHiveDataUtil.py --table ${table} --beginStr ${begin_early_2_Str}${beginHhStr} --endStr ${end_early_2_Str}${endHhStr})
-  echo "python 返回值：${python_return_code}"
-  if [ $python_return_code -eq 0 ]; then
-    echo "Python程序返回0，校验存在数据，退出循环。"
-    break
-  fi
-  echo "Python程序返回非0值，不存在数据，等待五分钟后再次调用。"
-  sleep 300
-  current_hour=$(date +%H)
-  current_minute=$(date +%M)
-  # shellcheck disable=SC2039
-  if (( current_hour > max_hour || (current_hour == max_hour && current_minute >= max_minute) )); then
-    echo "最长等待时间已到，失败:${current_hour}-${current_minute}"
-    python FeishuBot.py "荐模型数据更新 \n【任务名称】:step0校验是否生产完数据\n【是否成功】:error\n【信息】:最长等待时间已到，失败:${current_hour}-${current_minute}"
-    exit 1
-  fi
-done
-
-# 1 生产原始数据
-echo "$(date +%Y-%m-%d_%H-%M-%S)----------step1------------开始根据${table}生产原始数据"
-/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
---class com.aliyun.odps.spark.examples.makedata_qiao.makedata_13_originData_20240705 \
---master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
-../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
-tablePart:64 repartition:32 \
-beginStr:${begin_early_2_Str}${beginHhStr} endStr:${end_early_2_Str}${endHhStr} \
-savePath:${originDataPath} \
-table:${table}
-if [ $? -ne 0 ]; then
-   echo "Spark原始样本生产任务执行失败"
-   python FeishuBot.py "荐模型数据更新 \n【任务名称】:step1根据${table}生产原始数据\n【是否成功】:error\n【信息】:Spark原始样本生产任务执行失败"
-   exit 1
-else
-   echo "spark原始样本生产执行成功"
-fi
-
-
-# 2 特征值拼接
-echo "$(date +%Y-%m-%d_%H-%M-%S)----------step2------------开始特征值拼接"
-/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
---class com.aliyun.odps.spark.examples.makedata_qiao.makedata_14_valueData_20240705 \
---master yarn --driver-memory 1G --executor-memory 3G --executor-cores 1 --num-executors 32 \
-../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
-readPath:${originDataPath} \
-savePath:${valueDataPath} \
-beginStr:${begin_early_2_Str} endStr:${end_early_2_Str} repartition:1000
-if [ $? -ne 0 ]; then
-   echo "Spark特征值拼接处理任务执行失败"
-   python FeishuBot.py "荐模型数据更新 \n【任务名称】:step2特征值拼接\n【是否成功】:error\n【信息】:Spark特征值拼接处理任务执行失败"
-   exit 1
-else
-   echo "spark特征值拼接处理执行成功"
-fi
-
-# 3 特征分桶
-echo "$(date +%Y-%m-%d_%H-%M-%S)----------step3------------根据特征分桶生产重打分特征数据"
-/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
---class com.aliyun.odps.spark.examples.makedata_qiao.makedata_16_bucketData_20240705 \
---master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
-../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
-readPath:${valueDataPath} \
-savePath:${bucketDataPath} \
-beginStr:${begin_early_2_Str} endStr:${end_early_2_Str} repartition:1000
-if [ $? -ne 0 ]; then
-   echo "Spark特征分桶处理任务执行失败"
-   python FeishuBot.py "荐模型数据更新 \n【任务名称】:step3训练数据产出\n【是否成功】:error\n【信息】:Spark特征分桶处理任务执行失败"
-   exit 1
-else
-   echo "spark特征分桶处理执行成功"
-fi
-
-
-# 4 对比AUC 前置对比3日模型数据 与 线上模型数据效果对比，如果3日模型优于线上，更新线上模型
-echo "$(date +%Y-%m-%d_%H-%M-%S)----------step4------------开始对比，新：${MODEL_PATH}/${model_name}_${today_early_3}.txt，与线上online模型数据auc效果"
-$HADOOP fs -text ${bucketDataPath}/${begin_early_2_Str}/* | ${FM_HOME}/fm_predict -m ${LAST_MODEL_HOME}/model_online.txt -dim 8 -core 8 -out ${PREDICT_PATH}/${model_name}_${today}_online.txt
-$HADOOP fs -text ${bucketDataPath}/${begin_early_2_Str}/* | ${FM_HOME}/fm_predict -m ${MODEL_PATH}/${model_name}_${today_early_3}.txt -dim 8 -core 8 -out ${PREDICT_PATH}/${model_name}_${today}_new.txt
-
-online_auc=`cat ${PREDICT_PATH}/${model_name}_${today}_online.txt | /root/sunmingze/AUC/AUC`
-if [ $? -ne 0 ]; then
-   echo "推荐线上模型AUC计算失败"
-   python FeishuBot.py "荐模型数据更新 \n【任务名称】:step4新旧模型AUC对比\n【是否成功】:error\n【信息】:推荐线上模型AUC计算失败"
-   exit 1
-fi
-
-new_auc=`cat ${PREDICT_PATH}/${model_name}_${today}_new.txt | /root/sunmingze/AUC/AUC`
-if [ $? -ne 0 ]; then
-   echo "推荐新模型AUC计算失败"
-   python FeishuBot.py "荐模型数据更新 \n【任务名称】:step4新旧模型AUC对比\n【是否成功】:error\n【信息】:推荐新模型AUC计算失败${PREDICT_PATH}/${model_name}_${today}_new.txt"
-   exit 1
-fi
-
-
-# 4.1 对比auc数据判断是否更新线上模型
-if [ "$online_auc" \< "$new_auc" ]; then
-    echo "新模型优于线上模型: 线上模型AUC: ${online_auc}, 新模型AUC: ${new_auc}"
-    # 4.1.1 模型格式转换
-    cat ${MODEL_PATH}/${model_name}_${today_early_3}.txt |
-    awk -F " " '{
-         if (NR == 1) {
-              print $1"\t"$2
-         } else {
-              split($0, fields, " ");
-              OFS="\t";
-              line=""
-              for (i = 1; i <= 10 && i <= length(fields); i++) {
-                  line = (line ? line "\t" : "") fields[i];
-              }
-              print line
-         }
-    }' > ${MODEL_PATH}/${model_name}_${today_early_3}_change.txt
-    if [ $? -ne 0 ]; then
-       echo "新模型文件格式转换失败"
-       python FeishuBot.py "荐模型数据更新 \n【任务名称】:step4模型格式转换\n【是否成功】:error\n【信息】:新模型文件格式转换失败${MODEL_PATH}/${model_name}_${today_early_3}.txt"
-       exit 1
-    fi
-    # 4.1.2 模型文件上传OSS
-    online_model_path=${OSS_PATH}/${model_name}.txt
-    $HADOOP fs -test -e ${online_model_path}
-    if [ $? -eq 0 ]; then
-        echo "数据存在, 先删除。"
-        $HADOOP fs -rm -r -skipTrash ${online_model_path}
-    else
-        echo "数据不存在"
-    fi
-
-    $HADOOP fs -put ${MODEL_PATH}/${model_name}_${today_early_3}_change.txt ${online_model_path}
-    if [ $? -eq 0 ]; then
-       echo "推荐模型文件至OSS成功"
-    else
-       echo "推荐模型文件至OSS失败"
-       python FeishuBot.py "荐模型数据更新 \n【任务名称】:step4模型推送oss\n【是否成功】:error\n【信息】:推荐模型文件至OSS失败${MODEL_PATH}/${model_name}_${today_early_3}_change.txt --- ${online_model_path}"
-       exit 1
-    fi
-    # 4.1.3 本地保存最新的线上使用的模型，用于下一次的AUC验证
-    cp -f ${LAST_MODEL_HOME}/model_online.txt ${LAST_MODEL_HOME}/model_online_$(date +\%Y\%m\%d).txt
-    cp -f ${MODEL_PATH}/${model_name}_${today_early_3}.txt ${LAST_MODEL_HOME}/model_online.txt
-    if [ $? -ne 0 ]; then
-       echo "模型备份失败"
-       exit 1
-    fi
-    python FeishuBot.py "荐模型数据更新 \n【任务名称】:step4模型更新\n【是否成功】:success\n【信息】:新模型优于线上模型: 线上模型AUC: ${online_auc}, 新模型AUC: ${new_auc}，已更新${model_name}_${today_early_3}.txt模型}"
-else
-    echo "新模型不如线上模型: 线上模型AUC: ${online_auc}, 新模型AUC: ${new_auc}"
-    python FeishuBot.py "荐模型数据更新 \n【任务名称】:step4模型更新\n【是否成功】:success\n【信息】:新模型不如线上模型: 线上模型AUC: ${online_auc}, 新模型AUC: ${new_auc}}"
-fi
-
-
-
-# 5 模型训练
-echo "$(date +%Y-%m-%d_%H-%M-%S)----------step5------------开始模型训练"
-$HADOOP fs -text ${bucketDataPath}/${begin_early_2_Str}/* | ${FM_HOME}/fm_train -m ${MODEL_PATH}/${model_name}_${begin_early_2_Str}.txt -dim 1,1,8 -im ${LAST_MODEL_HOME}/model_online.txt -core 8
-if [ $? -ne 0 ]; then
-   echo "模型训练失败"
-   python FeishuBot.py "荐模型数据更新 \n【任务名称】:step5模型训练\n【是否成功】:error\n【信息】:${bucketDataPath}/${begin_early_2_Str}训练失败"
-   exit 1
-fi
-
-echo "$(date +%Y-%m-%d_%H-%M-%S)----------step6------------模型训练完成：${MODEL_PATH}/${model_name}_${begin_early_2_Str}.txt"

qiaojialiang/test/multiTest.sh

@@ -1,28 +0,0 @@
-#!/bin/sh
-set -x
-
-source /root/anaconda3/bin/activate py37
-
-export SPARK_HOME=/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8
-export PATH=$SPARK_HOME/bin:$PATH
-export HADOOP_CONF_DIR=/etc/taihao-apps/hadoop-conf
-export JAVA_HOME=/usr/lib/jvm/java-1.8.0
-
-
-# 启动第一个命令并放入后台
-echo "Starting command 1"
-sleep 5 &
-
-# 启动第二个命令并放入后台
-echo "Starting command 2"
-sleep 3 &
-
-# 启动第三个命令并放入后台
-echo "Starting command 3"
-sleep 4 &
-
-# 使用wait等待所有后台命令完成
-wait
-
-# 所有命令执行完毕后执行的命令
-echo "All commands have completed successfully."

qiaojialiang/test/script1.sh

@@ -1,32 +0,0 @@
-#!/bin/bash
-
-# 检查是否提供了两个参数
-if [ "$#" -ne 2 ]; then
-    echo "用法: $0 n1 n2"
-    echo "此脚本输出从n1到n2（包含）的所有数字。"
-    exit 1
-fi
-
-n1=$1
-n2=$2
-
-# 检查n1和n2是否为整数
-if ! [[ "$n1" =~ ^[0-9]+$ ]] || ! [[ "$n2" =~ ^[0-9]+$ ]]; then
-    echo "错误：n1和n2都必须是正整数。"
-    exit 1
-fi
-
-# 确保n1不大于n2
-if [ "$n1" -gt "$n2" ]; then
-    echo "错误：n1必须小于或等于n2。"
-    exit 1
-fi
-
-# 输出从n1到n2的数字
-for (( i=$n1; i<=$n2; i++ ))
-do
-    echo $i
-    sleep 1
-done
-
-exit 0

qiaojialiang/test/script2.sh

@@ -1,4 +0,0 @@
-#!/bin/bash
-echo "执行第二个脚本"
-# 这里添加你的脚本逻辑
-exit 0