recommend-emr-dataprocess/rec_scripts/rov/train/scripts/train_model_data.sh

#!/bin/bash

abs_path=$(cd `dirname $0`; pwd)
sample_path="${abs_path}/../data"
model_path="${abs_path}/../model"

start_date=20250721
end_date=20250727
model_file="${model_path}/str_fm.txt"
feature_file="${sample_path}/feature.csv"
if(($#==4))
then
    start_date=$1
    end_date=$2
    model_file=$3
    feature_file=$4
else
    start_date=$(date +%Y%m%d -d "-15 $days day")
    end_date=$(date +%Y%m%d -d "-2 $days day")
fi

sample_file_list=""
for((i=0; i<=21; i++))
do
  data_date=$(date -d "$start_date $i day" +"%Y%m%d")
  if [ "$data_date" -le "$end_date" ]
  then
    tmp_path=$sample_path
    test_file="${tmp_path}/${data_date}_00000.csv"
    if [ ! -f $test_file ]
    then
      tmp_path="/data2/jch/project/return_of_view_v6/data"
      test_file="${tmp_path}/${data_date}_00000.csv"
    fi

    if [ -f $test_file ]
    then
      sample_file="${tmp_path}/${data_date}_00*.csv"
      if [[ -z $sample_file_list ]]
      then
        sample_file_list=$sample_file
      else
        sample_file_list="$sample_file_list $sample_file"
      fi
    fi
  fi
done

# filter feature
filter_feature="${abs_path}/../data/filter_feature.csv"
if [ -f $filter_feature ]
then
    local_feature="${feature_file}.tmp"
    awk '{if(NR==FNR){a[$1]=1}else{if(!($1 in a)){print}}}' $filter_feature $feature_file > $local_feature
    wait
    sleep 20s
    feature_file=$local_feature
fi

# model name
model_prefix=`echo $model_file | awk -F".txt" '{print $1}'`
off_model_file="${model_prefix}_offline.txt"

# train
#trainTools="/data2/jch/package/alphaFM/bin/fm_train"
trainTools="/data2/jch/package/onlyTest/alphaFM/bin/fm_train"
trainParam="-dim 1,1,8 -core 24"

echo `date` "cat $sample_file_list | $trainTools -m $off_model_file $trainParam"
cat $sample_file_list | $trainTools -m $off_model_file $trainParam -feature $feature_file -min_feature_cnt 200 &
wait
sleep 30s

# transform model
trans_py="${abs_path}/../src/tools/offline2online_fm.py"
echo `date` "python3 $trans_py --input $off_model_file --output $model_file"
python3 $trans_py --input $off_model_file --output $model_file &
wait
sleep 30s