特征验证

recommend-feature-produce/src/main/java/com/tzld/piaoquan/recommend/feature/produce/FeatureDiff.java

@@ -78,38 +78,43 @@ public class FeatureDiff {
             return;
         }
 
-        long diffCount = fieldValues.repartition(partitionNum).filter(new Function<Map<String, String>, Boolean>() {
-            @Override
-            public Boolean call(Map<String, String> map) throws Exception {
-
-                if (StringUtils.isNotBlank(map.get("flowpool"))) {
-                    return false;
-                }
-
-                Map<String, String> metaFeatureMap = JSONUtils.fromJson(map.get("metafeaturemap"), new TypeToken<Map<String, String>>() {
-                }, Collections.emptyMap());
-                for (Map.Entry<String, String> e : tableToCol.entrySet()) {
-                    Map<String, String> offline = JSONUtils.fromJson(map.get(e.getValue()), new TypeToken<Map<String, String>>() {
-                    }, Collections.emptyMap());
-
-                    Map<String, String> online = JSONUtils.fromJson(metaFeatureMap.get(e.getKey()), new TypeToken<Map<String,
-                            String>>() {
-                    }, Collections.emptyMap());
-
-
-                    if (offline.size() != online.size()) {
-                        return true;
-                    }
+        long diffCount =
+                fieldValues.repartition(partitionNum).sample(false, 0.001d).filter(new Function<Map<String, String>,
+                        Boolean>() {
+                    @Override
+                    public Boolean call(Map<String, String> map) throws Exception {
+
+                        if (StringUtils.isNotBlank(map.get("flowpool"))) {
+                            return false;
+                        }
 
-                    for (Map.Entry<String, String> offlineE : offline.entrySet()) {
-                        if (!StringUtils.equals(online.get(offlineE.getKey()), offlineE.getValue())) {
-                            return true;
+                        Map<String, String> metaFeatureMap = JSONUtils.fromJson(map.get("metafeaturemap"), new TypeToken<Map<String, String>>() {
+                        }, Collections.emptyMap());
+                        for (Map.Entry<String, String> e : tableToCol.entrySet()) {
+                            Map<String, String> offline = JSONUtils.fromJson(map.get(e.getValue()), new TypeToken<Map<String, String>>() {
+                            }, Collections.emptyMap());
+
+                            Map<String, String> online = JSONUtils.fromJson(metaFeatureMap.get(e.getKey()), new TypeToken<Map<String,
+                                    String>>() {
+                            }, Collections.emptyMap());
+
+
+                            if (offline.size() != online.size()) {
+                                log.info("size diff");
+                                return true;
+                            }
+
+                            for (Map.Entry<String, String> offlineE : offline.entrySet()) {
+                                if (!StringUtils.equals(online.get(offlineE.getKey()), offlineE.getValue())) {
+                                    log.info("offline {}, online {}", JSONUtils.toJson(offline),
+                                            JSONUtils.toJson(online));
+                                    return true;
+                                }
+                            }
                         }
+                        return false;
                     }
-                }
-                return false;
-            }
-        }).count();
+                }).count();
 
         log.info("diff count {}", diffCount);