package examples.sparksql;

import com.aliyun.odps.TableSchema;
import com.aliyun.odps.data.Record;
import org.apache.spark.SparkConf;
import org.apache.spark.aliyun.odps.OdpsOps;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function2;

import java.util.ArrayList;
import java.util.List;


public class SparkEMRLoaderDemo {

    public static void main(String[] args) {

        String partition = "dt=20231123";
        String accessId = "LTAIWYUujJAm7CbH";
        String accessKey = "RfSjdiWwED1sGFlsjXv0DlfTnZTG1P";
        String odpsUrl = "http://service.cn.maxcompute.aliyun.com/api";
        String tunnelUrl = "http://dt.cn-hangzhou.maxcompute.aliyun-inc.com";
        String project = "loghubods";
        String table = "alg_recsys_video_info";

        SparkConf sparkConf = new SparkConf().setAppName("E-MapReduce Demo 3-2: Spark MaxCompute Demo (Java)");
        JavaSparkContext jsc = new JavaSparkContext(sparkConf);

        OdpsOps odpsOps = new OdpsOps(jsc.sc(), accessId, accessKey, odpsUrl, tunnelUrl);

        System.out.println("Read odps table...");
        JavaRDD<List<String>> readData = odpsOps.readTableWithJava(project, table, partition, new RecordToLongs(), Integer.valueOf(10));

        System.out.println("counts: ");
        System.out.println(readData.count());
    }

    static class RecordToLongs implements Function2<Record, TableSchema, List<String>> {
        @Override
        public List<String> call(Record record, TableSchema schema) throws Exception {
            List<String> ret = new ArrayList<String>();
            for (int i = 0; i < schema.getColumns().size(); i++) {
                if (record.get(i) != null)
                    ret.add(record.get(i).toString());
                else
                    ret.add("0.0");
            }
            return ret;
        }
    }


}
