youhua

recommend-feature-produce/src/main/java/com/tzld/piaoquan/recommend/feature/produce/ODPSToRedis.java

@@ -71,7 +71,7 @@ public class ODPSToRedis {
         }
 
         log.info("odps count {}", count);
-        argMap.put("partitionNum", String.valueOf(count / 100000 + 1));
+        argMap.put("partitionNum", String.valueOf(count / 50000 + 1));
 
         JavaRDD<Map<String, String>> fieldValues = odpsService.read(jsc, config, argMap);
         if (fieldValues == null) {

recommend-feature-produce/src/main/java/com/tzld/piaoquan/recommend/feature/produce/service/RedisService.java

@@ -58,18 +58,19 @@ public class RedisService implements Serializable {
         Map<String, String> batch = new HashMap<>();
         Jedis jedis = new Jedis(hostName, port);
         jedis.auth(password);
+        long expire = config.getRedis().getExpire();
         while (dataIte.hasNext()) {
             Map<String, String> record = dataIte.next();
             String redisKey = redisKey(record, config);
             String value = JSONUtils.toJson(record);
             batch.put(redisKey, value);
-            if (batch.size() > 2000) {
-                mSet(jedis, batch, config.getRedis().getExpire(), TimeUnit.SECONDS);
+            if (batch.size() >= 5000) {
+                mSet(jedis, batch, expire, TimeUnit.SECONDS);
                 batch.clear();
             }
         }
         if (MapUtils.isNotEmpty(batch)) {
-            mSet(jedis, batch, config.getRedis().getExpire(), TimeUnit.SECONDS);
+            mSet(jedis, batch, expire, TimeUnit.SECONDS);
         }
         jedis.close();
     }