package examples.sparksql;

import com.aliyun.odps.TableSchema;
import com.aliyun.odps.data.Record;
import org.apache.spark.SparkConf;
import org.apache.spark.aliyun.odps.OdpsOps;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function2;


public class SparkGrapDataLoader {

    public static void main(String[] args) {

        String partition = "dt=20231204";
        String accessId = "LTAIWYUujJAm7CbH";
        String accessKey = "RfSjdiWwED1sGFlsjXv0DlfTnZTG1P";
        String odpsUrl = "http://service.odps.aliyun.com/api";
        String tunnelUrl = "http://dt.cn-hangzhou.maxcompute.aliyun-inc.com";
        String project = "loghubods";
        String table = "share_machinecode";
        String hdfsPath = "/dw/recommend/model/graph_data/" + partition;

        SparkConf sparkConf = new SparkConf().setAppName("E-MapReduce Demo 3-2: Spark MaxCompute Demo (Java)");
        JavaSparkContext jsc = new JavaSparkContext(sparkConf);
        OdpsOps odpsOps = new OdpsOps(jsc.sc(), accessId, accessKey, odpsUrl, tunnelUrl);
        System.out.println("Read odps table...");

        JavaRDD<String> readData = odpsOps.readTableWithJava(project, table, partition, new RecordsToSamples(), Integer.valueOf(50));
        readData.filter(line-> line.split("\t")[2].equals("21")).saveAsTextFile(hdfsPath);
    }


    static class RecordsToSamples implements Function2<Record, TableSchema, String>  {
        @Override
        public String call(Record record, TableSchema schema) throws Exception {
            String sourceUser = record.getString("machinecode_share");
            String destUser = record.getString("machinecode_click");
            String appType = record.getString("apptype");
            return sourceUser + "\t" + destUser + "\t" + appType;
        }
    }


}