recommend-emr-dataprocess/profile_scripts/stat_freq.sh

#!/bin/bash

year=""
if(($#==1))
then
    year=$1
else
    year=2025
fi

set -x
export SPARK_HOME=/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8
export PATH=$SPARK_HOME/bin:$PATH
export HADOOP_CONF_DIR=/etc/taihao-apps/hadoop-conf
export JAVA_HOME=/usr/lib/jvm/java-1.8.0

# params
readPath=/dw/recommend/model/user_profile/data/
suffix="y,8,4,0,e,a,c,k,o,w,g,s,u,q"
whatLabel=gender
class="1,2"
featureIndex=2
repartition=10
savePath=/dw/recommend/model/832_recsys_analysis_data

# 1 统计特征
/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata_recsys_r_rate.stat_origin_feature \
--master yarn --driver-memory 4G --executor-memory 6G --executor-cores 1 --num-executors 16 \
--conf spark.yarn.executor.memoryoverhead=2048 \
/mnt/disk1/jch/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-jar-with-dependencies.jar \
readPath:${readPath} \
year:${year} \
suffix:${suffix} \
whatLabel:${whatLabel} \
class:${class} \
featureIndex:${featureIndex} \
repartition:${repartition} \
savePath:${savePath} \