recommend-emr-dataprocess/rec_scripts/rov/train/scripts/download_data.sh

#!/bin/bash

abs_path=$(cd `dirname $0`; pwd)
start_date=""
end_date=""
if(($#==2))
then
    start_date=$1
    end_date=$2
else
    start_date=$(date +%Y%m%d -d "-2 $days day")
    end_date=$start_date
fi

partition=64
hdfs_url="hdfs://192.168.141.208:9000"
hdfs_path="/dw/recommend/model/831_recsys_rov_train_data"
local_path="${abs_path}/../data"
for((i=0; i<=21; i++))
do
  data_date=$(date -d "$start_date $i day" +"%Y%m%d")
  if [ "$data_date" -le "$end_date" ]
  then
    for((j=0; j<$partition; j++))
    do
      hdfs_file=$(printf "%s/%s/part-%05d.gz" $hdfs_path $data_date $j)
      local_file=$(printf "%s/%s_%05d.csv" $local_path $data_date $j)
      echo `date` " hdfs dfs -fs $hdfs_url -text $hdfs_file > $local_file"
      hdfs dfs -fs $hdfs_url -text $hdfs_file > $local_file
      sleep 1s
    done
  fi
done