|
|
@@ -1,9 +1,5 @@
|
|
|
package com.tzld.piaoquan.recommend.feature.service;
|
|
|
|
|
|
-import com.alibaba.fastjson.JSON;
|
|
|
-import com.ctrip.framework.apollo.model.ConfigChange;
|
|
|
-import com.ctrip.framework.apollo.model.ConfigChangeEvent;
|
|
|
-import com.ctrip.framework.apollo.spring.annotation.ApolloConfigChangeListener;
|
|
|
import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
|
|
|
import com.google.common.base.Strings;
|
|
|
import com.tzld.piaoquan.recommend.feature.common.ThreadPoolFactory;
|
|
|
@@ -21,9 +17,14 @@ import org.springframework.beans.factory.annotation.Value;
|
|
|
import org.springframework.data.redis.core.RedisTemplate;
|
|
|
import org.springframework.stereotype.Service;
|
|
|
|
|
|
-import javax.annotation.PostConstruct;
|
|
|
-import java.util.*;
|
|
|
-import java.util.concurrent.*;
|
|
|
+import java.util.ArrayList;
|
|
|
+import java.util.Collections;
|
|
|
+import java.util.List;
|
|
|
+import java.util.Optional;
|
|
|
+import java.util.concurrent.Future;
|
|
|
+import java.util.concurrent.TimeUnit;
|
|
|
+import java.util.concurrent.TimeoutException;
|
|
|
+import java.util.stream.Collectors;
|
|
|
|
|
|
/**
|
|
|
* @author dyp
|
|
|
@@ -35,78 +36,24 @@ public class FeatureV2Service {
|
|
|
@Autowired
|
|
|
private RedisTemplate<String, String> redisTemplate;
|
|
|
|
|
|
+ @Qualifier("byteRedisTemplate")
|
|
|
+ @Autowired
|
|
|
+ private RedisTemplate<String, byte[]> byteRedisTemplate;
|
|
|
+
|
|
|
@ApolloJsonValue("${dts.config:}")
|
|
|
private List<DTSConfig> newDtsConfigs;
|
|
|
|
|
|
- @Value("${feature.mget.batch.size:200}")
|
|
|
- private Integer batchSize;
|
|
|
-
|
|
|
- /**
|
|
|
- * DTSConfig 缓存,按 tableName 索引,避免每次 redisKey() 都遍历列表
|
|
|
- * 启动时通过 @PostConstruct 初始化,配置变更时通过 @ApolloConfigChangeListener 更新
|
|
|
- */
|
|
|
- private volatile Map<String, DTSConfig> dtsConfigCache = Collections.emptyMap();
|
|
|
+ @ApolloJsonValue("${dts.config.v2:}")
|
|
|
+ private List<DTSConfig> dtsConfigsV2;
|
|
|
|
|
|
- @PostConstruct
|
|
|
- public void init() {
|
|
|
- rebuildDtsConfigCache();
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 重建 DTSConfig 缓存(从 newDtsConfigs 字段)
|
|
|
- */
|
|
|
- private void rebuildDtsConfigCache() {
|
|
|
- rebuildDtsConfigCacheFromList(newDtsConfigs);
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 重建 DTSConfig 缓存
|
|
|
- * @param configs 配置列表
|
|
|
- */
|
|
|
- private void rebuildDtsConfigCacheFromList(List<DTSConfig> configs) {
|
|
|
- if (configs == null || configs.isEmpty()) {
|
|
|
- dtsConfigCache = Collections.emptyMap();
|
|
|
- log.info("DTSConfig cache rebuilt, size=0");
|
|
|
- return;
|
|
|
- }
|
|
|
- Map<String, DTSConfig> cache = new HashMap<>(configs.size());
|
|
|
- for (DTSConfig config : configs) {
|
|
|
- if (config.getOdps() != null && StringUtils.isNotBlank(config.getOdps().getTable())) {
|
|
|
- cache.put(config.getOdps().getTable(), config);
|
|
|
- }
|
|
|
- }
|
|
|
- dtsConfigCache = cache;
|
|
|
- log.info("DTSConfig cache rebuilt, size={}", cache.size());
|
|
|
- }
|
|
|
+ @Value("${feature.mget.batch.size:300}")
|
|
|
+ private Integer batchSize;
|
|
|
|
|
|
- /**
|
|
|
- * 监听 Apollo 配置变更,自动重建缓存
|
|
|
- * 手动解析新配置值,避免与 @ApolloJsonValue 自动注入产生竞态条件
|
|
|
- */
|
|
|
- @ApolloConfigChangeListener
|
|
|
- public void onConfigChange(ConfigChangeEvent changeEvent) {
|
|
|
- if (changeEvent.isChanged("dts.config")) {
|
|
|
- ConfigChange change = changeEvent.getChange("dts.config");
|
|
|
- String newValue = change.getNewValue();
|
|
|
- log.info("dts.config changed, old={}, new={}", change.getOldValue(), newValue);
|
|
|
-
|
|
|
- if (StringUtils.isNotBlank(newValue)) {
|
|
|
- try {
|
|
|
- List<DTSConfig> newConfigs = JSON.parseArray(newValue, DTSConfig.class);
|
|
|
- rebuildDtsConfigCacheFromList(newConfigs);
|
|
|
- } catch (Exception e) {
|
|
|
- log.error("Failed to parse dts.config: {}", newValue, e);
|
|
|
- }
|
|
|
- } else {
|
|
|
- dtsConfigCache = Collections.emptyMap();
|
|
|
- log.info("dts.config is empty, cache cleared");
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
+ @Value("${feature.service.optimize.switch:false}")
|
|
|
+ private Boolean optimizeV3Switch;
|
|
|
|
|
|
public MultiGetFeatureResponse multiGetFeature(MultiGetFeatureRequest request) {
|
|
|
int keyCount = request.getFeatureKeyCount();
|
|
|
-
|
|
|
if (keyCount == 0) {
|
|
|
return MultiGetFeatureResponse.newBuilder()
|
|
|
.setResult(Result.newBuilder().setCode(1))
|
|
|
@@ -115,32 +62,41 @@ public class FeatureV2Service {
|
|
|
|
|
|
long startTime = System.currentTimeMillis();
|
|
|
|
|
|
- // 1. 生成Redis Key列表
|
|
|
- List<String> redisKeys = CommonCollectionUtils.toList(request.getFeatureKeyList(), this::redisKey);
|
|
|
+ List<String> redisKeys;
|
|
|
+ if (Boolean.TRUE.equals(optimizeV3Switch)) {
|
|
|
+ redisKeys = request.getFeatureKeyList().stream()
|
|
|
+ .map(this::buildRedisKeyV2)
|
|
|
+ .collect(Collectors.toList());
|
|
|
+ } else {
|
|
|
+ redisKeys = request.getFeatureKeyList().stream()
|
|
|
+ .map(this::redisKey)
|
|
|
+ .map(s -> String.format("%s:v2", s))
|
|
|
+ .collect(Collectors.toList());
|
|
|
+ }
|
|
|
|
|
|
int safeBatchSize = (batchSize == null || batchSize <= 0) ? 300 : batchSize;
|
|
|
|
|
|
// 2. 分批并行查询(必须保证返回值与 key 顺序一致)
|
|
|
- List<BatchFuture> futures = new ArrayList<>();
|
|
|
+ List<BatchFuture<byte[]>> futures = new ArrayList<>();
|
|
|
for (int start = 0; start < redisKeys.size(); start += safeBatchSize) {
|
|
|
int end = Math.min(start + safeBatchSize, redisKeys.size());
|
|
|
List<String> batchKeys = redisKeys.subList(start, end);
|
|
|
- Future<List<String>> future = ThreadPoolFactory.multiGetFeaturePool()
|
|
|
- .submit(() -> redisTemplate.opsForValue().multiGet(batchKeys));
|
|
|
- futures.add(new BatchFuture(start, end - start, future));
|
|
|
+ Future<List<byte[]>> future = ThreadPoolFactory.multiGetFeaturePool()
|
|
|
+ .submit(() -> byteRedisTemplate.opsForValue().multiGet(batchKeys));
|
|
|
+ futures.add(new BatchFuture<byte[]>(start, end - start, future));
|
|
|
}
|
|
|
|
|
|
// 3. 收集结果(整体最多等待约 2s;超时/异常的批次用 null 填充,避免错位)
|
|
|
- List<String> allValues = new ArrayList<>(Collections.nCopies(keyCount, null));
|
|
|
+ List<byte[]> allValues = new ArrayList<>(Collections.nCopies(keyCount, null));
|
|
|
long deadlineNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(2000);
|
|
|
- for (BatchFuture bf : futures) {
|
|
|
+ for (BatchFuture<byte[]> bf : futures) {
|
|
|
long remainingNanos = deadlineNanos - System.nanoTime();
|
|
|
if (remainingNanos <= 0) {
|
|
|
bf.future.cancel(true);
|
|
|
continue;
|
|
|
}
|
|
|
try {
|
|
|
- List<String> batchValues = bf.future.get(remainingNanos, TimeUnit.NANOSECONDS);
|
|
|
+ List<byte[]> batchValues = bf.future.get(remainingNanos, TimeUnit.NANOSECONDS);
|
|
|
if (batchValues == null) {
|
|
|
continue;
|
|
|
}
|
|
|
@@ -150,14 +106,14 @@ public class FeatureV2Service {
|
|
|
}
|
|
|
} catch (TimeoutException e) {
|
|
|
bf.future.cancel(true);
|
|
|
- log.warn("Batch mGet timeout, startIndex={}, size={}", bf.startIndex, bf.size);
|
|
|
+ log.warn("Batch mGetV2 timeout, startIndex={}, size={}", bf.startIndex, bf.size);
|
|
|
} catch (Exception e) {
|
|
|
- log.error("Batch mGet failed, startIndex={}, size={}", bf.startIndex, bf.size, e);
|
|
|
+ log.error("Batch mGetV2 failed, startIndex={}, size={}", bf.startIndex, bf.size, e);
|
|
|
}
|
|
|
}
|
|
|
|
|
|
// 4. 解压缩
|
|
|
- List<String> values = CommonCollectionUtils.toList(allValues, CompressionUtil::snappyDecompress);
|
|
|
+ List<String> values = CommonCollectionUtils.toList(allValues, CompressionUtil::snappyDecompressV2);
|
|
|
|
|
|
// 5. 构建响应
|
|
|
MultiGetFeatureResponse.Builder builder = MultiGetFeatureResponse.newBuilder();
|
|
|
@@ -167,31 +123,40 @@ public class FeatureV2Service {
|
|
|
Strings.nullToEmpty(values.get(i)));
|
|
|
}
|
|
|
MultiGetFeatureResponse build = builder.build();
|
|
|
- log.info("multiGetFeature, cost={}ms", System.currentTimeMillis() - startTime);
|
|
|
+ log.info("multiGetFeature, cost={}ms, keyCount={}", System.currentTimeMillis() - startTime, keyCount);
|
|
|
return build;
|
|
|
}
|
|
|
|
|
|
- private static final class BatchFuture {
|
|
|
+ private static final class BatchFuture<T> {
|
|
|
private final int startIndex;
|
|
|
private final int size;
|
|
|
- private final Future<List<String>> future;
|
|
|
+ private final Future<List<T>> future;
|
|
|
|
|
|
- private BatchFuture(int startIndex, int size, Future<List<String>> future) {
|
|
|
+ private BatchFuture(int startIndex, int size, Future<List<T>> future) {
|
|
|
this.startIndex = startIndex;
|
|
|
this.size = size;
|
|
|
this.future = future;
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- // Note:写入和读取的key生成规则应保持一致
|
|
|
private String redisKey(FeatureKeyProto fk) {
|
|
|
- // 使用缓存查找配置,O(1) 复杂度
|
|
|
- // 缓存在启动时初始化,配置变更时通过 Apollo 监听器更新
|
|
|
- DTSConfig config = dtsConfigCache.get(fk.getTableName());
|
|
|
- if (config == null) {
|
|
|
+ return this.buildRedisKey(fk, newDtsConfigs);
|
|
|
+ }
|
|
|
+
|
|
|
+ private String buildRedisKeyV2(FeatureKeyProto fk) {
|
|
|
+ return this.buildRedisKey(fk, dtsConfigsV2);
|
|
|
+ }
|
|
|
+
|
|
|
+ // Note:写入和读取的key生成规则应保持一致
|
|
|
+ private String buildRedisKey(FeatureKeyProto fk, List<DTSConfig> dtsConfigs){
|
|
|
+ Optional<DTSConfig> optional = dtsConfigs.stream()
|
|
|
+ .filter(c -> c.getOdps() != null && StringUtils.equals(c.getOdps().getTable(), fk.getTableName()))
|
|
|
+ .findFirst();
|
|
|
+ if (!optional.isPresent()) {
|
|
|
log.error("table {} not config", fk.getTableName());
|
|
|
return "";
|
|
|
}
|
|
|
+ DTSConfig config = optional.get();
|
|
|
|
|
|
// Note:写入和读取的key生成规则应保持一致
|
|
|
List<String> fields = config.getRedis().getKey();
|
|
|
@@ -208,5 +173,4 @@ public class FeatureV2Service {
|
|
|
}
|
|
|
return sb.toString();
|
|
|
}
|
|
|
-
|
|
|
}
|