|
|
@@ -1,301 +0,0 @@
|
|
|
-package com.tzld.piaoquan.recommend.server.framework.recaller.provider;
|
|
|
-
|
|
|
-
|
|
|
-import com.google.common.base.Optional;
|
|
|
-import com.google.common.cache.CacheBuilder;
|
|
|
-import com.google.common.cache.CacheLoader;
|
|
|
-import com.google.common.cache.LoadingCache;
|
|
|
-
|
|
|
-import com.tzld.piaoquan.recommend.server.framework.utils.FixedThreadPoolHelper;
|
|
|
-
|
|
|
-import com.tzld.piaoquan.recommend.server.framework.utils.IndexUtils;
|
|
|
-import com.tzld.piaoquan.recommend.server.framework.utils.RedisSmartClient;
|
|
|
-import org.apache.commons.lang.exception.ExceptionUtils;
|
|
|
-import org.apache.commons.lang.math.RandomUtils;
|
|
|
-import org.slf4j.Logger;
|
|
|
-import org.slf4j.LoggerFactory;
|
|
|
-
|
|
|
-import java.util.concurrent.BlockingQueue;
|
|
|
-import java.util.concurrent.ExecutionException;
|
|
|
-import java.util.concurrent.ExecutorService;
|
|
|
-import java.util.concurrent.Executors;
|
|
|
-import java.util.concurrent.LinkedBlockingQueue;
|
|
|
-import java.util.concurrent.ScheduledExecutorService;
|
|
|
-import java.util.concurrent.TimeUnit;
|
|
|
-
|
|
|
-/**
|
|
|
- * Generalized item access to Redis.
|
|
|
- */
|
|
|
-public class ItemProvider<InMemoryItem> {
|
|
|
- private static final Logger logger = LoggerFactory.getLogger(ItemProvider.class);
|
|
|
-
|
|
|
- private static final int REDIS_DEFAULT_TTL = 7 * 24 * 60 * 60;
|
|
|
- private static final long CACHE_TIMEOUT_MS = 15 * 60 * 1000L;
|
|
|
- private static final long CACHE_MAXIMUMSIZE = 80 * 10000; // default 80w
|
|
|
-
|
|
|
- private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
|
|
|
- private final ExecutorService itemRefreshExecutorService = new FixedThreadPoolHelper(4, "itemRefresh").getThreadPoolExecutor();
|
|
|
- // 不能为 static
|
|
|
- private final BlockingQueue<String> asyncRefreshQueue = new LinkedBlockingQueue<String>(100000);
|
|
|
- private final RedisSmartClient client;
|
|
|
- private final String prefix;
|
|
|
- private final LoadingCache<String, CacheEntry<InMemoryItem>> cache;
|
|
|
- private final long cacheMaximumSize;
|
|
|
- private int redisTtl = REDIS_DEFAULT_TTL;
|
|
|
- private long cacheTimeout = CACHE_TIMEOUT_MS;
|
|
|
- private long refreshDynamicInterval = CACHE_TIMEOUT_MS;
|
|
|
-
|
|
|
-
|
|
|
- /**
|
|
|
- * 构造带有本地 cache 的 item provider, 使用redis作为底层存储,
|
|
|
- * 本地cache基于 Guava Cache 构建, 配置失效时间与异步刷新时间;
|
|
|
- *
|
|
|
- * @param client redis-cluster client
|
|
|
- * @param prefix 业务标志
|
|
|
- * @param cacheTimeOutMs 缓存失效时间
|
|
|
- */
|
|
|
- public ItemProvider(RedisSmartClient client,
|
|
|
- String prefix,
|
|
|
- long cacheTimeOutMs,
|
|
|
- long refreshDynamicInterval) {
|
|
|
- this(client,
|
|
|
- prefix,
|
|
|
- cacheTimeOutMs,
|
|
|
- refreshDynamicInterval,
|
|
|
- CACHE_MAXIMUMSIZE,
|
|
|
- REDIS_DEFAULT_TTL,
|
|
|
- TimeUnit.SECONDS);
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 构造带有本地 cache 的 item provider, 使用redis作为底层存储,
|
|
|
- * 本地cache基于 Guava Cache 构建, 配置失效时间与异步刷新时间;
|
|
|
- *
|
|
|
- * @param client redis-cluster client
|
|
|
- * @param prefix 业务标志
|
|
|
- * @param cacheTimeOutMs 缓存失效时间
|
|
|
- * @param redisKeyTtl
|
|
|
- * @param redisTtlUnit
|
|
|
- */
|
|
|
- public ItemProvider(RedisSmartClient client,
|
|
|
- String prefix,
|
|
|
- long cacheTimeOutMs,
|
|
|
- long refreshDynamicInterval,
|
|
|
- final long cacheMaximumSize,
|
|
|
- long redisKeyTtl,
|
|
|
- TimeUnit redisTtlUnit) {
|
|
|
-
|
|
|
- this.cacheTimeout = cacheTimeOutMs;
|
|
|
- this.refreshDynamicInterval = refreshDynamicInterval;
|
|
|
- this.redisTtl = (int) redisTtlUnit.toSeconds(redisKeyTtl);
|
|
|
- this.cacheMaximumSize = cacheMaximumSize;
|
|
|
-
|
|
|
- this.client = client;
|
|
|
- this.prefix = prefix;
|
|
|
- this.cache = CacheBuilder.newBuilder()
|
|
|
- .expireAfterWrite((long) (this.cacheTimeout * 1.4), TimeUnit.MILLISECONDS)
|
|
|
- .maximumSize(cacheMaximumSize)
|
|
|
- .build(new CacheLoader<String, CacheEntry<InMemoryItem>>() {
|
|
|
- @Override
|
|
|
- public CacheEntry<InMemoryItem> load(String key) throws Exception {
|
|
|
- if (key == null) {
|
|
|
- return null;
|
|
|
- } else {
|
|
|
- return loadItem(key);
|
|
|
- }
|
|
|
- }
|
|
|
- });
|
|
|
-
|
|
|
- // cache refresh
|
|
|
- for (int i = 0; i < 4; i++) {
|
|
|
- itemRefreshExecutorService.execute(new AsyncRefreshCacheRunnable());
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- public ItemProvider(RedisSmartClient client, String prefix) {
|
|
|
- this(client, prefix, CACHE_TIMEOUT_MS, CACHE_TIMEOUT_MS);
|
|
|
- }
|
|
|
-
|
|
|
- public RedisSmartClient getClient() {
|
|
|
- return client;
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- public InMemoryItem deserialize(byte[] bytes){
|
|
|
- return null;
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- /**
|
|
|
- * 初始时定义 expire time = System.currentTimeMillis()+cacheTimeout
|
|
|
- * @param id
|
|
|
- * @return
|
|
|
- */
|
|
|
- private CacheEntry<InMemoryItem> loadItem(String id) throws Exception {
|
|
|
- long start = System.nanoTime();
|
|
|
- byte[] bytes = client.get(IndexUtils.convertKey(prefix + id));
|
|
|
- CacheEntry<InMemoryItem> result;
|
|
|
- if (bytes == null) {
|
|
|
- result = null;
|
|
|
- } else {
|
|
|
- result = makeCacheEntry(id, Optional.of(deserialize(bytes)));
|
|
|
- }
|
|
|
-
|
|
|
- return result;
|
|
|
- }
|
|
|
-
|
|
|
- public long dbSize() {
|
|
|
- return this.cache.size();
|
|
|
- }
|
|
|
-
|
|
|
- private CacheEntry<InMemoryItem> makeCacheEntry(String itemId, Optional<InMemoryItem> t) {
|
|
|
- long expireTime = System.currentTimeMillis() + cacheTimeout + RandomUtils.nextInt(10 * 60 * 1000);
|
|
|
- long dynamicExpireTime = System.currentTimeMillis() + refreshDynamicInterval;
|
|
|
- return new CacheEntry<InMemoryItem>(itemId, expireTime, dynamicExpireTime, t);
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- public CacheEntry<InMemoryItem> getCacheEntry(String id) throws ExecutionException {
|
|
|
- return cache.get(id);
|
|
|
-
|
|
|
- }
|
|
|
-
|
|
|
- public Optional<InMemoryItem> getDirect(String id) throws Exception {
|
|
|
- CacheEntry<InMemoryItem> cacheEntry = loadItem(id);
|
|
|
- return cacheEntry.getItem();
|
|
|
- }
|
|
|
-
|
|
|
- public Optional<InMemoryItem> get(String id) throws ExecutionException {
|
|
|
- CacheEntry<InMemoryItem> result = cache.get(id);
|
|
|
-
|
|
|
- if (result == null || !result.getItem().isPresent()) {
|
|
|
- cache.invalidate(id);
|
|
|
- return null;
|
|
|
- }
|
|
|
-
|
|
|
- // minimum expire time, update key expire time
|
|
|
- if (result.getExpireTime() <= 0) {
|
|
|
- result = makeCacheEntry(id, result.getItem());
|
|
|
- cache.put(id, result);
|
|
|
- }
|
|
|
-
|
|
|
- // check expire and refresh
|
|
|
- long currentTime = System.currentTimeMillis();
|
|
|
- if ((result.getDynamicFeatureExpireTime() > 0 && currentTime > result.getDynamicFeatureExpireTime()) ||
|
|
|
- (result.getExpireTime() > 0 && currentTime > result.getExpireTime())) {
|
|
|
- boolean ret = asyncRefreshQueue.offer(id);
|
|
|
- if (logger.isDebugEnabled()) {
|
|
|
- logger.debug("offer async queue itemid [{}], result [{}]", id, ret);
|
|
|
- }
|
|
|
-
|
|
|
- if (asyncRefreshQueue.size() > CACHE_MAXIMUMSIZE) {
|
|
|
- logger.warn("item backgroud blocking queue length [{}], it is danger", asyncRefreshQueue.size());
|
|
|
- }
|
|
|
-
|
|
|
- if (!ret) {
|
|
|
- logger.error("item backgroud offer blocking queue error, itemid [{}], with queue length [{}]", id, asyncRefreshQueue.size());
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- return result.getItem();
|
|
|
- }
|
|
|
-
|
|
|
- public String setLocalCache(String id, InMemoryItem t) throws ExecutionException {
|
|
|
- long start = System.nanoTime();
|
|
|
-
|
|
|
- // put to local cache
|
|
|
- CacheEntry<InMemoryItem> result = makeCacheEntry(id, Optional.of(t));
|
|
|
- cache.put(id, result);
|
|
|
-
|
|
|
- return "";
|
|
|
- }
|
|
|
-
|
|
|
- public String set(String id, InMemoryItem t) throws Exception {
|
|
|
- long start = System.nanoTime();
|
|
|
-
|
|
|
- // set redis
|
|
|
- String strKey = prefix + id;
|
|
|
- String ret = client.setex(IndexUtils.convertKey(strKey), this.redisTtl, t.toString().getBytes());
|
|
|
-
|
|
|
- // put to cache
|
|
|
- CacheEntry<InMemoryItem> result = makeCacheEntry(id, Optional.of(t));
|
|
|
- cache.put(id, result);
|
|
|
-
|
|
|
- return ret;
|
|
|
- }
|
|
|
-
|
|
|
- public long remove(String id) throws Exception {
|
|
|
- long start = System.nanoTime();
|
|
|
-
|
|
|
- // del redis
|
|
|
- long affectRow = client.del(IndexUtils.convertKey(prefix + id));
|
|
|
-
|
|
|
- // del cache
|
|
|
- cache.invalidate(id);
|
|
|
- return affectRow;
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- /**
|
|
|
- * 通过 blocking queue 异步刷新即将到期的item
|
|
|
- */
|
|
|
- public class AsyncRefreshCacheRunnable implements Runnable {
|
|
|
-
|
|
|
- @Override
|
|
|
- public void run() {
|
|
|
- while (true) {
|
|
|
- String itemId = "";
|
|
|
- try {
|
|
|
- itemId = asyncRefreshQueue.take(); // take async item, blocked when no item
|
|
|
-
|
|
|
- final long currentTimeMillis = System.currentTimeMillis();
|
|
|
- if (logger.isDebugEnabled()) {
|
|
|
- logger.debug("async refresh thread get itemid [{}]", itemId); // get itemid
|
|
|
- }
|
|
|
-
|
|
|
- CacheEntry<InMemoryItem> entry = cache.get(itemId);
|
|
|
- // 全量刷新
|
|
|
- if (currentTimeMillis > entry.getExpireTime()) {
|
|
|
- refreshStaticFeatures(entry);
|
|
|
- }
|
|
|
- // 刷新动态特征
|
|
|
- if (currentTimeMillis > entry.getDynamicFeatureExpireTime()) {
|
|
|
- refreshDynamicFeatures(entry);
|
|
|
- }
|
|
|
-
|
|
|
- long endTime = System.currentTimeMillis();
|
|
|
- if (logger.isDebugEnabled()) {
|
|
|
- logger.debug("async refresh items spenttime [{}]", endTime - currentTimeMillis);
|
|
|
- }
|
|
|
- } catch (Exception e) {
|
|
|
- logger.error("async refresh item error, itemId [{}], [{}]", itemId, ExceptionUtils.getFullStackTrace(e));
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- public void refreshStaticFeatures(CacheEntry<InMemoryItem> entry) {
|
|
|
- try {
|
|
|
- cache.refresh(entry.getItemId());
|
|
|
- if (logger.isDebugEnabled()) {
|
|
|
- logger.debug("async refresh item [{}]", entry.getItemId());
|
|
|
- }
|
|
|
- } catch (Exception e) {
|
|
|
- logger.error("async refresh item error, itemId [{}], exception [{}]", entry.getItemId(), ExceptionUtils.getFullStackTrace(e));
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- public void refreshDynamicFeatures(CacheEntry<InMemoryItem> entry) {
|
|
|
- try {
|
|
|
- // reload dynamic items and update expireTime
|
|
|
- entry.setItem(Optional.of(entry.getItem().get()));
|
|
|
- entry.setDynamicFeatureExpireTime(System.currentTimeMillis() + refreshDynamicInterval);
|
|
|
- if (logger.isDebugEnabled()) {
|
|
|
- logger.debug("async refresh items dynamic features [{}]", entry.getItemId());
|
|
|
- }
|
|
|
- } catch (Exception e) {
|
|
|
- logger.error("async refresh item dynamic features error, itemId [{}], exception [{}]",
|
|
|
- entry.getItemId(), ExceptionUtils.getFullStackTrace(e));
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-}
|
sunmingze