package com.aliyun.odps.spark.examples.myUtils

/**
 * Author: zhangbo58
 * Description:
 *
 */
import org.apache.commons.lang.time.DateUtils
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.permission.FsPermission
import org.apache.hadoop.fs.{FileSystem, FileUtil, Path}

import scala.collection.mutable.ArrayBuffer

object MyHdfsUtils {
  def main(args: Array[String]): Unit = {
    val path = "zhangbo58/"
    //生成FileSystem
    println("获取目录下的一级文件和目录")
    getFilesAndDirs(path).foreach(println)
    println("获取目录下的一级文件")
    getFiles(path).foreach(println)
    println("获取目录下的一级目录")
    getDirs(path).foreach(println)
    println("获取目录下所有文件")
    getAllFiles(path).foreach(println)
  }

  def getHdfs(path: String): FileSystem = {
    val conf = new Configuration()
    //    FileSystem.get(URI.create(path), conf)
    val fs = org.apache.hadoop.fs.FileSystem.get(new org.apache.hadoop.conf.Configuration())
    fs
  }

  //获取目录下的一级文件和目录
  def getFilesAndDirs(path: String): Array[Path] = {
    val fs = getHdfs(path).listStatus(new Path(path))
    FileUtil.stat2Paths(fs)
  }
  //获取目录下的一级文件
  def getFiles(path: String): Array[String] = {
    getFilesAndDirs(path).filter(getHdfs(path).getFileStatus(_).isFile())
      .map(_.toString)
  }
  //获取目录下的一级目录
  def getDirs(path: String): Array[String] = {
    getFilesAndDirs(path).filter(getHdfs(path).getFileStatus(_).isDirectory)
      .map(_.toString)
  }
  //获取目录下的所有文件
  def getAllFiles(path: String): ArrayBuffer[String] = {
    val arr = ArrayBuffer[String]()
    val hdfs = getHdfs(path)
    val getPath = getFilesAndDirs(path)
    getPath.foreach(patha => {
      if (hdfs.getFileStatus(patha).isFile())
        arr += patha.toString
      else {
        arr ++= getAllFiles(patha.toString())
      }
    })
    arr
  }
  def ifHDFSHasData(path: String): Boolean = {
    val hdfs_path = new org.apache.hadoop.fs.Path(path.toString)
    val hdfs = org.apache.hadoop.fs.FileSystem.get(new org.apache.hadoop.conf.Configuration())

    var rst = false
    if (hdfs.exists(hdfs_path)) {
      //路径存在且不为空
      val statusList = hdfs.listStatus(hdfs_path)
      for (status <- statusList if !rst && (status.getPath.toString.contains("part-") || status.getPath.toString.contains("_SUCCESS"))) {
        if (status.getLen > 0) {
          rst = true
        }
      }
    }
    rst
  }

  def delete_hdfs_path(path: String): Unit = {
    val hdfs_path = new org.apache.hadoop.fs.Path(path.toString)
    val hdfs = org.apache.hadoop.fs.FileSystem.get(new org.apache.hadoop.conf.Configuration())

    if (hdfs.exists(hdfs_path)) {
      hdfs.delete(hdfs_path, true)
    }
  }

  def hdfs_exits(path:String): Boolean = {
    val hdfs_path = new org.apache.hadoop.fs.Path(path.toString)
    val hdfs = org.apache.hadoop.fs.FileSystem.get(new org.apache.hadoop.conf.Configuration())

    hdfs.exists(hdfs_path)
  }


  /* 删除某路径下，不在某段时间内的所有数据。
  * 举例：keepDays=2 dateStr=20191015 => 保留20191015和20191014两天的数据
  */
  def hdfs_delete_not_keep_days(
                                 path:String,
                                 keepDays:Int,
                                 dateStr:String,
                                 pattern:String = "yyyyMMdd"
                               ): Unit ={
    val file_list = this.getFiles(path)
    println("hdfs_delete_not_keep_days-file_list")
    file_list.foreach(println)

    for (file <- file_list){
      var flag = true
      val date_early = MyDateUtils.getNumDaysBefore(dateStr, keepDays, pattern)
      try{
        val file_split_strs = file.split("/")
        val len = file_split_strs.length
        var file_date = file_split_strs(len-1)
        if (file_date.equals("")){
          file_date = file_split_strs(len-2)
        }
        var date1 = DateUtils.parseDate(file_date, Array[String](pattern)) // 文件中的日期
        var date2 = DateUtils.parseDate(date_early, Array[String](pattern)) // 请求的日期前keepDays的日期
        if (date1.compareTo(date2) >= 0){ // 这个日期之前的数据 全部删除
          flag = false
        }
      }catch {
        case e:Exception =>
          flag = false
      }

      if (flag){
        MyHdfsUtils.delete_hdfs_path(file.toString)
      }
    }
  }

  /**
   * @Author: zhangbo
   * @Description: 给某hdfs路径加权限
   *
   */

  def give_hdfs_permission(path:String): Unit ={
    getHdfs(path).setPermission(new Path(path), new FsPermission("777"))
  }
}

