|
@@ -30,7 +30,7 @@ public class SparkAdFeaToRedisLoader {
|
|
RedisStandaloneConfiguration rsc = new RedisStandaloneConfiguration();
|
|
RedisStandaloneConfiguration rsc = new RedisStandaloneConfiguration();
|
|
rsc.setPort(6379);
|
|
rsc.setPort(6379);
|
|
rsc.setPassword("Wqsd@2019");
|
|
rsc.setPassword("Wqsd@2019");
|
|
- rsc.setHostName("r-bp1ps6my7lzg8rdhwx682.redis.rds.aliyuncs.com");
|
|
|
|
|
|
+ rsc.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
|
|
RedisTemplate<String, String> template = new RedisTemplate<>();
|
|
RedisTemplate<String, String> template = new RedisTemplate<>();
|
|
JedisConnectionFactory fac = new JedisConnectionFactory(rsc);
|
|
JedisConnectionFactory fac = new JedisConnectionFactory(rsc);
|
|
fac.afterPropertiesSet();
|
|
fac.afterPropertiesSet();
|
|
@@ -80,7 +80,7 @@ public class SparkAdFeaToRedisLoader {
|
|
|
|
|
|
// load Ad features
|
|
// load Ad features
|
|
JavaRDD<Record> readAdData = odpsOps.readTableWithJava(project, tableAdInfo, partition, null, Integer.valueOf(10));
|
|
JavaRDD<Record> readAdData = odpsOps.readTableWithJava(project, tableAdInfo, partition, null, Integer.valueOf(10));
|
|
- readAdData.sample(false, 0.0001).foreachPartition(
|
|
|
|
|
|
+ readAdData.foreachPartition(
|
|
rowIterator -> {
|
|
rowIterator -> {
|
|
RedisTemplate<String, String> redisTemplate = buildRedisTemplate();
|
|
RedisTemplate<String, String> redisTemplate = buildRedisTemplate();
|
|
rowIterator.forEachRemaining(line -> loadAdFeatureToRedis(redisTemplate, line));
|
|
rowIterator.forEachRemaining(line -> loadAdFeatureToRedis(redisTemplate, line));
|
|
@@ -90,7 +90,7 @@ public class SparkAdFeaToRedisLoader {
|
|
|
|
|
|
// load user features
|
|
// load user features
|
|
JavaRDD<Record> readUserData = odpsOps.readTableWithJava(project, tableUserInfo, partition, null, Integer.valueOf(10));
|
|
JavaRDD<Record> readUserData = odpsOps.readTableWithJava(project, tableUserInfo, partition, null, Integer.valueOf(10));
|
|
- readUserData.sample(false, 0.0001).foreachPartition(
|
|
|
|
|
|
+ readUserData.foreachPartition(
|
|
rowIterator -> {
|
|
rowIterator -> {
|
|
RedisTemplate<String, String> redisTemplate = buildRedisTemplate();
|
|
RedisTemplate<String, String> redisTemplate = buildRedisTemplate();
|
|
rowIterator.forEachRemaining(line -> loadUserAdFeatureToRedis(redisTemplate, line));
|
|
rowIterator.forEachRemaining(line -> loadUserAdFeatureToRedis(redisTemplate, line));
|