行为预测性别

profile_scripts/README.md

@@ -0,0 +1,32 @@
+# 1. 总体
+- 根据用户行为数据，进行行为画像。
+- 使用时，需将脚本中的jar文件路径，替换成自己的路径
+- [数据生成code](https://git.yishihui.com/algorithm/recommend-emr-dataprocess/src/feature/jch_makedata)
+- [性别模型训练code: spark_xgboost](https://git.yishihui.com/algorithm/recommend-model/src/feature/jch)
+
+# 2. 基础数据表
+## 2.1 用户点击分享行为
+- loghubods.alg_recsys_user_share_click_event
+## 2.2 视频标签&关键词
+- loghubods.video_merge_tag
+- loghubods.alg_recsys_profile_user_behavior_video
+## 2.3 微信性别标签
+- loghubods.alg_recsys_user_wx_gender
+## 2.4 样本数据
+- loghubods.alg_recsys_feature_behavior_profile
+
+# 3. 生成特征数据
+- make_data.sh 生成特征
+- stat_freq.sh 统计特征频次
+
+# 4. 性别模型
+## 4.1 生成训练数据
+- make_train_sample.sh
+## 4.2 生成预测数据
+- make_predict_sample.sh
+## 4.3 评估模型
+- xgb_model_eval.sh
+## 4.4 预测
+- xgb_model_predict.sh
+## 4.5 结果写入hive
+- parse_to_hive.sh

profile_scripts/gender/make_predict_sample.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+feature_file=""
+year=""
+if(($#==2))
+then
+    feature_file=$1
+    year=$2
+else
+    exit -1
+fi
+
+set -x
+export SPARK_HOME=/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8
+export PATH=$SPARK_HOME/bin:$PATH
+export HADOOP_CONF_DIR=/etc/taihao-apps/hadoop-conf
+export JAVA_HOME=/usr/lib/jvm/java-1.8.0
+
+# params
+readPath=/dw/recommend/model/user_profile/data/
+suffix="y,8,4,0,e,a,c,k,o,w,g,s,u,q,i,m"
+whatLabel=gender
+class="0"
+minCnt=10
+savePath=/dw/recommend/model/user_profile/gender/sample/predict/
+
+# 1 生产数据
+/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+--class com.aliyun.odps.spark.examples.makedata_recsys_r_rate.makedata_profile_gender_sample_20251114 \
+--master yarn --driver-memory 4G --executor-memory 6G --executor-cores 1 --num-executors 8 \
+--conf spark.yarn.executor.memoryoverhead=2048 \
+--files ${feature_file} \
+/mnt/disk1/jch/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-jar-with-dependencies.jar \
+readPath:${readPath} \
+year:${year} \
+suffix:${suffix} \
+whatLabel:${whatLabel} \
+class:${class} \
+featureFile:${feature_file} \
+minCnt:${minCnt} \
+repartition:32 \
+savePath:${savePath} \

profile_scripts/gender/make_train_sample.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+feature_file=""
+year=""
+if(($#==2))
+then
+    feature_file=$1
+    year=$2
+else
+    exit -1
+fi
+
+set -x
+export SPARK_HOME=/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8
+export PATH=$SPARK_HOME/bin:$PATH
+export HADOOP_CONF_DIR=/etc/taihao-apps/hadoop-conf
+export JAVA_HOME=/usr/lib/jvm/java-1.8.0
+
+# params
+readPath=/dw/recommend/model/user_profile/data/
+suffix="y,8,4,0,e,a,c,k,o,w,g,s,u,q,i,m"
+whatLabel=gender
+class="1,2"
+minCnt=10
+savePath=/dw/recommend/model/user_profile/gender/sample/train/
+
+# 1 生产数据
+/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+--class com.aliyun.odps.spark.examples.makedata_recsys_r_rate.makedata_profile_gender_sample_20251114 \
+--master yarn --driver-memory 4G --executor-memory 6G --executor-cores 1 --num-executors 8 \
+--conf spark.yarn.executor.memoryoverhead=2048 \
+--files ${feature_file} \
+/mnt/disk1/jch/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-jar-with-dependencies.jar \
+readPath:${readPath} \
+year:${year} \
+suffix:${suffix} \
+whatLabel:${whatLabel} \
+class:${class} \
+featureFile:${feature_file} \
+minCnt:${minCnt} \
+repartition:1 \
+savePath:${savePath} \

profile_scripts/gender/parse_to_hive.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+year=""
+version=""
+if(($#==2))
+then
+    year=$1
+    version=$2
+else
+    exit -1
+fi
+
+set -x
+export SPARK_HOME=/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8
+export PATH=$SPARK_HOME/bin:$PATH
+export HADOOP_CONF_DIR=/etc/taihao-apps/hadoop-conf
+export JAVA_HOME=/usr/lib/jvm/java-1.8.0
+
+# params
+input_path=/dw/recommend/model/user_profile/gender/result/
+suffix="y,8,4,0,e,a,c,k,o,w,g,s,u,q,i,m"
+#suffix="y"
+project=loghubods
+output_table=alg_recsys_user_behavior_gender_tmp
+
+# to hive
+/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+--class com.aliyun.odps.spark.examples.makedata_recsys_r_rate.parse_gender_to_hive \
+--master yarn --driver-memory 4G --executor-memory 6G --executor-cores 1 --num-executors 8 \
+--conf spark.yarn.executor.memoryoverhead=2048 \
+/mnt/disk1/jch/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-jar-with-dependencies.jar \
+inputPath:${input_path} \
+year:${year} \
+suffix:${suffix} \
+project:${project} \
+outputTable:${output_table} \
+version:${version} \

profile_scripts/gender/train_xgb_model.sh

@@ -0,0 +1,55 @@
+#!/bin/sh
+
+feature_file=""
+year=""
+if(($#==2))
+then
+    feature_file=$1
+    year=$2
+else
+    exit -1
+fi
+
+# env
+export HADOOP_CONF_DIR=/etc/taihao-apps/hadoop-conf
+export JAVA_HOME=/usr/lib/jvm/java-1.8.0
+
+# params
+workers=16
+minCnt=10
+BASE_TRAIN_DATA_PATH=/dw/recommend/model/user_profile/gender/sample/train
+MODEL_SAVE_PATH=/dw/recommend/model/user_profile/gender/model/model_xgb
+
+train_data_path=""
+suffix_array=(y 8 4 0 e a c k o w g s u q)
+for suffix in "${suffix_array[@]}"
+do
+    one_day_data_path="${BASE_TRAIN_DATA_PATH}/${year}_$suffix"
+    if [[ -z $train_data_path ]]
+    then
+        train_data_path=$one_day_data_path
+    else
+        train_data_path="$train_data_path,$one_day_data_path"
+    fi
+done
+
+## ******* train *******
+/opt/apps/SPARK3/spark-3.3.1-hadoop3.2-1.0.5/bin/spark-class org.apache.spark.deploy.SparkSubmit \
+--class com.tzld.piaoquan.recommend.model.train_profile_gender_xgb_20251114 \
+--master yarn --driver-memory 6G --executor-memory 10G --executor-cores 1 --num-executors ${workers} \
+--conf spark.yarn.executor.memoryoverhead=2048 \
+--conf spark.shuffle.service.enabled=true \
+--conf spark.shuffle.service.port=7337 \
+--conf spark.shuffle.consolidateFiles=true \
+--conf spark.shuffle.manager=sort \
+--conf spark.storage.memoryFraction=0.4 \
+--conf spark.shuffle.memoryFraction=0.5 \
+--conf spark.default.parallelism=200 \
+--conf spark.sql.debug.maxToStringFields=100 \
+--files ${feature_file} \
+/mnt/disk1/jch/recommend-model/recommend-model-produce/target/recommend-model-produce-jar-with-dependencies.jar \
+trainPath:${train_data_path} \
+featureFile:${feature_file} \
+minCnt:${minCnt} \
+modelPath:${MODEL_SAVE_PATH} \
+eta:0.06 gamma:0.0 max_depth:4 num_round:1000 num_worker:${workers}

profile_scripts/gender/xgb_model_eval.sh

@@ -0,0 +1,58 @@
+#!/bin/sh
+set -x
+
+feature_file=""
+year=""
+if(($#==2))
+then
+    feature_file=$1
+    year=$2
+else
+    exit -1
+fi
+
+# env
+export HADOOP_CONF_DIR=/etc/taihao-apps/hadoop-conf
+export JAVA_HOME=/usr/lib/jvm/java-1.8.0
+
+# params
+base_data_path=/dw/recommend/model/user_profile/gender/sample/train
+model_path=/dw/recommend/model/user_profile/gender/model/model_xgb
+minCnt=10
+save_path=/dw/recommend/model/user_profile/gender/eval
+repartition=4
+
+test_data_path=""
+suffix_array=(i m)
+for suffix in "${suffix_array[@]}"
+do
+    one_day_data_path="${base_data_path}/${year}_$suffix"
+    if [[ -z $test_data_path ]]
+    then
+        test_data_path=$one_day_data_path
+    else
+        test_data_path="$test_data_path,$one_day_data_path"
+    fi
+done
+
+echo `date` "predict gender sample"
+/opt/apps/SPARK3/spark-3.3.1-hadoop3.2-1.0.5/bin/spark-class org.apache.spark.deploy.SparkSubmit \
+--class com.tzld.piaoquan.recommend.model.pred_profile_gender_xgb_20251114 \
+--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 8 \
+--conf spark.yarn.executor.memoryoverhead=1024 \
+--conf spark.shuffle.service.enabled=true \
+--conf spark.shuffle.service.port=7337 \
+--conf spark.shuffle.consolidateFiles=true \
+--conf spark.shuffle.manager=sort \
+--conf spark.storage.memoryFraction=0.4 \
+--conf spark.shuffle.memoryFraction=0.5 \
+--conf spark.default.parallelism=200 \
+--conf spark.debug.maxToStringFields=100 \
+--files ${feature_file} \
+/mnt/disk1/jch/recommend-model/recommend-model-produce/target/recommend-model-produce-jar-with-dependencies.jar \
+modelPath:${model_path} \
+testPath:${test_data_path} \
+featureFile:${feature_file} \
+minCnt:${minCnt} \
+savePath:${save_path} \
+repartition:${repartition} \

profile_scripts/gender/xgb_model_predict.sh

@@ -0,0 +1,50 @@
+#!/bin/sh
+set -x
+
+feature_file=""
+year=""
+suffix=""
+if(($#==3))
+then
+    feature_file=$1
+    year=$2
+    suffix=$3
+else
+    exit -1
+fi
+
+# env
+export HADOOP_CONF_DIR=/etc/taihao-apps/hadoop-conf
+export JAVA_HOME=/usr/lib/jvm/java-1.8.0
+
+# params
+base_data_path=/dw/recommend/model/user_profile/gender/sample/predict
+model_path=/dw/recommend/model/user_profile/gender/model/model_xgb
+base_save_path=/dw/recommend/model/user_profile/gender/result
+minCnt=10
+repartition=4
+
+test_data_path="${base_data_path}/${year}_${suffix}"
+suffix_save_path="${base_save_path}/${year}_${suffix}"
+
+echo `date` "predict gender sample"
+/opt/apps/SPARK3/spark-3.3.1-hadoop3.2-1.0.5/bin/spark-class org.apache.spark.deploy.SparkSubmit \
+--class com.tzld.piaoquan.recommend.model.pred_profile_gender_xgb_20251114 \
+--master yarn --driver-memory 4G --executor-memory 10G --executor-cores 1 --num-executors 16 \
+--conf spark.yarn.executor.memoryoverhead=1024 \
+--conf spark.shuffle.service.enabled=true \
+--conf spark.shuffle.service.port=7337 \
+--conf spark.shuffle.consolidateFiles=true \
+--conf spark.shuffle.manager=sort \
+--conf spark.storage.memoryFraction=0.4 \
+--conf spark.shuffle.memoryFraction=0.5 \
+--conf spark.default.parallelism=200 \
+--conf spark.debug.maxToStringFields=100 \
+--files ${feature_file} \
+/mnt/disk1/jch/recommend-model/recommend-model-produce/target/recommend-model-produce-jar-with-dependencies.jar \
+modelPath:${model_path} \
+testPath:${test_data_path} \
+featureFile:${feature_file} \
+minCnt:${minCnt} \
+savePath:${suffix_save_path} \
+repartition:${repartition} \

profile_scripts/make_data.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+year=""
+table=""
+if(($#==2))
+then
+    year=$1
+    table=$2
+else
+    year=2025
+    table=alg_recsys_feature_behavior_profile
+fi
+
+set -x
+export SPARK_HOME=/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8
+export PATH=$SPARK_HOME/bin:$PATH
+export HADOOP_CONF_DIR=/etc/taihao-apps/hadoop-conf
+export JAVA_HOME=/usr/lib/jvm/java-1.8.0
+
+# params
+suffix="y,8,4,0,e,a,c,k,o,w,g,s,u,q,i,m"
+savePath=/dw/recommend/model/user_profile/data/
+
+# 1 生产原始数据
+echo "$(date +%Y-%m-%d_%H-%M-%S)----------step1------------开始根据${table}生产原始数据"
+
+/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+--class com.aliyun.odps.spark.examples.makedata_recsys_r_rate.makedata_profile_20251114 \
+--master yarn --driver-memory 4G --executor-memory 6G --executor-cores 1 --num-executors 16 \
+--conf spark.yarn.executor.memoryoverhead=2048 \
+/mnt/disk1/jch/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-jar-with-dependencies.jar \
+table:${table} tablePart:48 \
+year:${year} \
+suffix:${suffix} \
+repartition:32 \
+savePath:${savePath} \

profile_scripts/stat_freq.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+year=""
+if(($#==1))
+then
+    year=$1
+else
+    year=2025
+fi
+
+set -x
+export SPARK_HOME=/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8
+export PATH=$SPARK_HOME/bin:$PATH
+export HADOOP_CONF_DIR=/etc/taihao-apps/hadoop-conf
+export JAVA_HOME=/usr/lib/jvm/java-1.8.0
+
+# params
+readPath=/dw/recommend/model/user_profile/data/
+suffix="y,8,4,0,e,a,c,k,o,w,g,s,u,q"
+whatLabel=gender
+class="1,2"
+featureIndex=2
+repartition=10
+savePath=/dw/recommend/model/832_recsys_analysis_data
+
+# 1 统计特征
+/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+--class com.aliyun.odps.spark.examples.makedata_recsys_r_rate.stat_origin_feature \
+--master yarn --driver-memory 4G --executor-memory 6G --executor-cores 1 --num-executors 16 \
+--conf spark.yarn.executor.memoryoverhead=2048 \
+/mnt/disk1/jch/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-jar-with-dependencies.jar \
+readPath:${readPath} \
+year:${year} \
+suffix:${suffix} \
+whatLabel:${whatLabel} \
+class:${class} \
+featureIndex:${featureIndex} \
+repartition:${repartition} \
+savePath:${savePath} \