特征验证

recommend-feature-produce/src/main/java/com/tzld/piaoquan/recommend/feature/produce/FeatureDiff.java

@@ -88,15 +88,14 @@ public class FeatureDiff {
                             return false;
                         }
 
-                        Map<String, String> metaFeatureMap = JSONUtils.fromJson(map.get("metafeaturemap"), new TypeToken<Map<String, String>>() {
-                        }, Collections.emptyMap());
+                        Map<String, Map<String, String>> metaFeatureMap = JSONUtils.fromJson(map.get("metafeaturemap"),
+                                new TypeToken<Map<String, Map<String, String>>>() {
+                                }, Collections.emptyMap());
                         for (Map.Entry<String, String> e : tableToCol.entrySet()) {
                             Map<String, String> offline = JSONUtils.fromJson(map.get(e.getValue()), new TypeToken<Map<String, String>>() {
                             }, Collections.emptyMap());
 
-                            Map<String, String> online = JSONUtils.fromJson(metaFeatureMap.get(e.getKey()), new TypeToken<Map<String,
-                                    String>>() {
-                            }, Collections.emptyMap());
+                            Map<String, String> online = metaFeatureMap.getOrDefault(e.getKey(), new HashMap<>());
 
 
                             if (offline.size() != online.size()) {

recommend-feature-produce/src/main/java/com/tzld/piaoquan/recommend/feature/produce/Test.java

@@ -1,23 +1,143 @@
 package com.tzld.piaoquan.recommend.feature.produce;
 
+import com.aliyun.odps.Instance;
+import com.aliyun.odps.Odps;
+import com.aliyun.odps.OdpsException;
+import com.aliyun.odps.account.Account;
+import com.aliyun.odps.account.AliyunAccount;
+import com.aliyun.odps.data.Record;
+import com.aliyun.odps.data.SimpleJsonValue;
+import com.aliyun.odps.task.SQLTask;
+import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.feature.produce.util.CommonCollectionUtils;
+import com.tzld.piaoquan.recommend.feature.produce.util.JSONUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang.StringUtils;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 /**
  * @author dyp
  */
+@Slf4j
 public class Test {
+    private static Map<String, String> tableToCol;
+
+    static {
+        tableToCol = new HashMap<>();
+        tableToCol.put("alg_vid_feature_all_exp", "b1_feature");
+        tableToCol.put("alg_vid_feature_all_share", "b2_feature");
+        tableToCol.put("alg_vid_feature_all_return", "b3_feature");
+//        tableToCol.put("alg_vid_feature_head_play", "b4_feature");
+//        tableToCol.put("alg_vid_feature_feed_play", "b5_feature");
+        tableToCol.put("alg_vid_feature_exp2share", "b6_feature");
+        tableToCol.put("alg_vid_feature_share2return", "b7_feature");
+        tableToCol.put("alg_vid_feature_feed_noflow_exp", "b8_feature");
+        tableToCol.put("alg_vid_feature_feed_noflow_root_share", "b9_feature");
+        tableToCol.put("alg_vid_feature_feed_noflow_root_return", "b10_feature");
+        tableToCol.put("alg_vid_feature_feed_flow_exp", "b11_feature");
+        tableToCol.put("alg_vid_feature_feed_flow_root_share", "b12_feature");
+        tableToCol.put("alg_vid_feature_feed_flow_root_return", "b13_feature");
+//        tableToCol.put("alg_vid_feature_feed_apptype_exp", "b14_feature");
+//        tableToCol.put("alg_vid_feature_feed_apptype_root_share", "b15_feature");
+//        tableToCol.put("alg_vid_feature_feed_apptype_root_return", "b16_feature");
+        tableToCol.put("alg_vid_feature_feed_province_exp", "b17_feature");
+        tableToCol.put("alg_vid_feature_feed_province_root_share", "b18_feature");
+        tableToCol.put("alg_vid_feature_feed_province_root_return", "b19_feature");
+//        tableToCol.put("alg_mid_feature_play", "c1_feature");
+//        tableToCol.put("alg_mid_feature_share_and_return", "c2_feature");
+//        tableToCol.put("alg_mid_feature_play_tags", "c3_feature");
+//        tableToCol.put("alg_mid_feature_return_tags", "c4_feature");
+//        tableToCol.put("alg_mid_feature_share_tags", "c5_feature");
+//        tableToCol.put("alg_mid_feature_feed_exp_share_tags", "c6_feature");
+//        tableToCol.put("alg_mid_feature_feed_exp_return_tags", "c7_feature");
+//        tableToCol.put("alg_mid_feature_sharecf", "c8_feature");
+//        tableToCol.put("alg_mid_feature_returncf", "c9_feature");
+
+
+    }
+
     public static void main(String[] args) {
+//        args = new String[10];
+//        args[0] = "-project";
+//        args[1] = "loghubods";
+//        args[2] = "-table";
+//        args[3] = "alg_mid_feature_play";
+//        args[4] = "-dt";
+//        args[5] = "20240612";
+//        args[6] = "-hh";
+//        args[7] = "15";
+//        args[8] = "-env";
+//        args[9] = "test";
+//        ODPSToRedis.main(args);
+
+        final String accessId = "LTAIWYUujJAm7CbH";
+        final String accessKey = "RfSjdiWwED1sGFlsjXv0DlfTnZTG1P";
+        final String odpsUrl = "http://service.odps.aliyun.com/api";
+        final String tunnelUrl = "http://dt.cn-hangzhou.maxcompute.aliyun.com";
+        Account account = new AliyunAccount(accessId, accessKey);
+        Odps odps = new Odps(account);
+        odps.setEndpoint(odpsUrl);
+        odps.setDefaultProject("loghubods");
+
+        String sql = "select * from loghubods.alg_recsys_sample_all_new where dt=20240616 and hh=18 and flowpool = '' \n" +
+                "and vid = 21281133 AND abcode='ab0' order by uid desc limit 10;";
+
+        List<Record> records = null;
+        try {
+            Instance i = SQLTask.run(odps, sql);
+            i.waitForSuccess();
+            records = SQLTask.getResult(i);
+        } catch (OdpsException e) {
+            log.error("request odps error", e);
+        }
+
+        List<Map<String, String>> fieldValues = CommonCollectionUtils.toList(records, r -> {
+            Map<String, String> map = new HashMap<>();
+            for (int i = 0; i < r.getColumnCount(); i++) {
+                Object obj = r.get(i);
+                if (obj instanceof SimpleJsonValue) {
+                    map.put(r.getColumns()[i].getName(), ((SimpleJsonValue) obj).toString());
+                } else {
+                    map.put(r.getColumns()[i].getName(), r.getString(i));
+                }
+            }
+            return map;
+        });
+
+
+        int diff = 0;
+        A:for (Map<String, String> map : fieldValues) {
+            Map<String, Map<String, String>> metaFeatureMap = JSONUtils.fromJson(map.get("metafeaturemap"),
+                    new TypeToken<Map<String, Map<String, String>>>() {
+                    }, Collections.emptyMap());
+            for (Map.Entry<String, String> e : tableToCol.entrySet()) {
+                Map<String, String> offline = JSONUtils.fromJson(map.get(e.getValue()), new TypeToken<Map<String, String>>() {
+                }, Collections.emptyMap());
+
+                Map<String, String> online = metaFeatureMap.getOrDefault(e.getKey(), new HashMap<>());
+
+
+                if (offline.size() != online.size()) {
+                    diff++;
+                    continue A;
+                }
+
+                for (Map.Entry<String, String> offlineE : offline.entrySet()) {
+                    if (!StringUtils.equals(online.get(offlineE.getKey()), offlineE.getValue())) {
+                        diff++;
+                        continue A;
+                    }
+                }
+            }
+        }
+
+        log.info("diff {}", diff);
 
-        args = new String[10];
-        args[0] = "-project";
-        args[1] = "loghubods";
-        args[2] = "-table";
-        args[3] = "alg_mid_feature_play";
-        args[4] = "-dt";
-        args[5] = "20240612";
-        args[6] = "-hh";
-        args[7] = "15";
-        args[8] = "-env";
-        args[9] = "test";
-        ODPSToRedis.main(args);
     }
 
+
 }