|
@@ -34,13 +34,13 @@ import java.util.Map;
|
|
public class ODPSService {
|
|
public class ODPSService {
|
|
private final String accessId = "LTAIWYUujJAm7CbH";
|
|
private final String accessId = "LTAIWYUujJAm7CbH";
|
|
private final String accessKey = "RfSjdiWwED1sGFlsjXv0DlfTnZTG1P";
|
|
private final String accessKey = "RfSjdiWwED1sGFlsjXv0DlfTnZTG1P";
|
|
- // private final String odpsUrl = "http://service.odps.aliyun.com/api";
|
|
|
|
|
|
+// private final String odpsUrl = "http://service.odps.aliyun.com/api";
|
|
// private final String tunnelUrl = "http://dt.cn-hangzhou.maxcompute.aliyun.com";
|
|
// private final String tunnelUrl = "http://dt.cn-hangzhou.maxcompute.aliyun.com";
|
|
private final String odpsUrl = "http://service.cn-hangzhou-vpc.maxcompute.aliyun-inc.com/api";
|
|
private final String odpsUrl = "http://service.cn-hangzhou-vpc.maxcompute.aliyun-inc.com/api";
|
|
private final String tunnelUrl = "http://dt.cn-hangzhou-vpc.maxcompute.aliyun-inc.com";
|
|
private final String tunnelUrl = "http://dt.cn-hangzhou-vpc.maxcompute.aliyun-inc.com";
|
|
private final String sqlFormat = "select %s from %s where 1=1 %s ;";
|
|
private final String sqlFormat = "select %s from %s where 1=1 %s ;";
|
|
private final String countSqlFormat = "select count(1) as count from %s where 1=1 %s ;";
|
|
private final String countSqlFormat = "select count(1) as count from %s where 1=1 %s ;";
|
|
- private final String latestPartitionSqlFormat = "select %s from %s where 0=1 OR %s ;";
|
|
|
|
|
|
+ private final String latestPartitionSqlFormat = "select %s from %s where 0=1 %s ;";
|
|
|
|
|
|
|
|
|
|
public JavaRDD<Map<String, String>> read(JavaSparkContext jsc, DTSConfig config, Map<String, String> argMap) {
|
|
public JavaRDD<Map<String, String>> read(JavaSparkContext jsc, DTSConfig config, Map<String, String> argMap) {
|
|
@@ -173,6 +173,7 @@ public class ODPSService {
|
|
odps.setDefaultProject(project);
|
|
odps.setDefaultProject(project);
|
|
|
|
|
|
String sql = String.format(latestPartitionSqlFormat, cols, table, condition);
|
|
String sql = String.format(latestPartitionSqlFormat, cols, table, condition);
|
|
|
|
+ log.info(sql);
|
|
List<Record> records;
|
|
List<Record> records;
|
|
try {
|
|
try {
|
|
Instance i = SQLTask.run(odps, sql);
|
|
Instance i = SQLTask.run(odps, sql);
|
|
@@ -189,6 +190,7 @@ public class ODPSService {
|
|
for (int i = 0; i < record.getColumnCount(); i++) {
|
|
for (int i = 0; i < record.getColumnCount(); i++) {
|
|
map.put(record.getColumns()[i].getName(), record.getString(i));
|
|
map.put(record.getColumns()[i].getName(), record.getString(i));
|
|
}
|
|
}
|
|
|
|
+ return map;
|
|
}
|
|
}
|
|
return Collections.emptyMap();
|
|
return Collections.emptyMap();
|
|
|
|
|