|
@@ -1,11 +1,16 @@
|
|
|
package com.tzld.videoVector.service.impl;
|
|
package com.tzld.videoVector.service.impl;
|
|
|
|
|
|
|
|
|
|
+import com.alibaba.fastjson.JSON;
|
|
|
|
|
+import com.alibaba.fastjson.TypeReference;
|
|
|
import com.tzld.videoVector.api.DashScopeEmbeddingApiService;
|
|
import com.tzld.videoVector.api.DashScopeEmbeddingApiService;
|
|
|
import com.tzld.videoVector.model.po.pgVector.DeconstructVectorConfig;
|
|
import com.tzld.videoVector.model.po.pgVector.DeconstructVectorConfig;
|
|
|
import com.tzld.videoVector.service.EmbeddingService;
|
|
import com.tzld.videoVector.service.EmbeddingService;
|
|
|
|
|
+import com.tzld.videoVector.util.Md5Util;
|
|
|
|
|
+import com.tzld.videoVector.util.RedisUtils;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
|
import org.springframework.stereotype.Service;
|
|
import org.springframework.stereotype.Service;
|
|
|
|
|
+import org.springframework.util.StringUtils;
|
|
|
|
|
|
|
|
import javax.annotation.PostConstruct;
|
|
import javax.annotation.PostConstruct;
|
|
|
import javax.annotation.Resource;
|
|
import javax.annotation.Resource;
|
|
@@ -26,15 +31,26 @@ public class EmbeddingServiceImpl implements EmbeddingService {
|
|
|
@Value("${embedding.mode:dashscope}")
|
|
@Value("${embedding.mode:dashscope}")
|
|
|
private String embeddingMode;
|
|
private String embeddingMode;
|
|
|
|
|
|
|
|
|
|
+ /**
|
|
|
|
|
+ * embedding 缓存过期时间(秒),默认 1 小时
|
|
|
|
|
+ */
|
|
|
|
|
+ @Value("${embedding.cache.expire:3600}")
|
|
|
|
|
+ private long cacheExpireSeconds;
|
|
|
|
|
+
|
|
|
|
|
+ private static final String CACHE_KEY_PREFIX = "embedding:cache:";
|
|
|
|
|
+
|
|
|
@Resource
|
|
@Resource
|
|
|
private com.tzld.videoVector.api.EmbeddingApiService embeddingApiService;
|
|
private com.tzld.videoVector.api.EmbeddingApiService embeddingApiService;
|
|
|
|
|
|
|
|
@Resource
|
|
@Resource
|
|
|
private DashScopeEmbeddingApiService dashScopeEmbeddingApiService;
|
|
private DashScopeEmbeddingApiService dashScopeEmbeddingApiService;
|
|
|
|
|
|
|
|
|
|
+ @Resource
|
|
|
|
|
+ private RedisUtils redisUtils;
|
|
|
|
|
+
|
|
|
@PostConstruct
|
|
@PostConstruct
|
|
|
public void init() {
|
|
public void init() {
|
|
|
- log.info("向量化服务初始化完成,模式: {}", embeddingMode);
|
|
|
|
|
|
|
+ log.info("向量化服务初始化完成,模式: {}, 缓存过期: {}s", embeddingMode, cacheExpireSeconds);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
@Override
|
|
@@ -44,17 +60,49 @@ public class EmbeddingServiceImpl implements EmbeddingService {
|
|
|
return Collections.emptyList();
|
|
return Collections.emptyList();
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- // 根据配置选择向量化方式
|
|
|
|
|
|
|
+ // 构建缓存 key:基于文本MD5 + 模型 + 维度
|
|
|
|
|
+ String model = config != null ? config.getEmbeddingModel() : null;
|
|
|
|
|
+ Integer dim = config != null ? config.getDimension() : null;
|
|
|
|
|
+ String cacheKey = buildCacheKey(text, model, dim);
|
|
|
|
|
+
|
|
|
|
|
+ // 查询 Redis 缓存
|
|
|
|
|
+ if (cacheKey != null) {
|
|
|
|
|
+ try {
|
|
|
|
|
+ String cached = redisUtils.get(cacheKey);
|
|
|
|
|
+ if (StringUtils.hasText(cached)) {
|
|
|
|
|
+ List<Float> vector = JSON.parseObject(cached, new TypeReference<List<Float>>() {});
|
|
|
|
|
+ if (vector != null && !vector.isEmpty()) {
|
|
|
|
|
+ log.debug("命中 embedding Redis 缓存,key={}", cacheKey);
|
|
|
|
|
+ return vector;
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ } catch (Exception e) {
|
|
|
|
|
+ log.warn("读取 embedding 缓存失败,key={},继续调用 API", cacheKey, e);
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ // 调用 embedding API
|
|
|
|
|
+ List<Float> vector;
|
|
|
if ("dashscope".equalsIgnoreCase(embeddingMode)) {
|
|
if ("dashscope".equalsIgnoreCase(embeddingMode)) {
|
|
|
- String model = config != null ? config.getEmbeddingModel() : null;
|
|
|
|
|
- Integer dim = config != null ? config.getDimension() : null;
|
|
|
|
|
- return dashScopeEmbeddingApiService.embed(text, model, dim);
|
|
|
|
|
|
|
+ vector = dashScopeEmbeddingApiService.embed(text, model, dim);
|
|
|
} else if ("api".equalsIgnoreCase(embeddingMode)) {
|
|
} else if ("api".equalsIgnoreCase(embeddingMode)) {
|
|
|
- return embeddingApiService.embed(text);
|
|
|
|
|
|
|
+ vector = embeddingApiService.embed(text);
|
|
|
} else {
|
|
} else {
|
|
|
log.error("不支持的 embedding 模式: {}", embeddingMode);
|
|
log.error("不支持的 embedding 模式: {}", embeddingMode);
|
|
|
return Collections.emptyList();
|
|
return Collections.emptyList();
|
|
|
}
|
|
}
|
|
|
|
|
+
|
|
|
|
|
+ // 写入 Redis 缓存
|
|
|
|
|
+ if (vector != null && !vector.isEmpty() && cacheKey != null) {
|
|
|
|
|
+ try {
|
|
|
|
|
+ redisUtils.set(cacheKey, JSON.toJSONString(vector), cacheExpireSeconds);
|
|
|
|
|
+ log.debug("embedding 结果已缓存,key={}, 维度={}", cacheKey, vector.size());
|
|
|
|
|
+ } catch (Exception e) {
|
|
|
|
|
+ log.warn("写入 embedding 缓存失败,key={}", cacheKey, e);
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ return vector;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
@Override
|
|
@@ -63,17 +111,26 @@ public class EmbeddingServiceImpl implements EmbeddingService {
|
|
|
return Collections.emptyList();
|
|
return Collections.emptyList();
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- // 根据配置选择向量化方式
|
|
|
|
|
- if ("dashscope".equalsIgnoreCase(embeddingMode)) {
|
|
|
|
|
- String model = config != null ? config.getEmbeddingModel() : null;
|
|
|
|
|
- Integer dim = config != null ? config.getDimension() : null;
|
|
|
|
|
- return dashScopeEmbeddingApiService.batchEmbed(texts, model, dim);
|
|
|
|
|
- } else if ("api".equalsIgnoreCase(embeddingMode)) {
|
|
|
|
|
- return embeddingApiService.batchEmbed(texts);
|
|
|
|
|
- } else {
|
|
|
|
|
- log.error("不支持的 embedding 模式: {}", embeddingMode);
|
|
|
|
|
- return Collections.emptyList();
|
|
|
|
|
|
|
+ // 批量场景逐条走 embed 方法,复用缓存逻辑
|
|
|
|
|
+ List<List<Float>> results = new java.util.ArrayList<>(texts.size());
|
|
|
|
|
+ for (String text : texts) {
|
|
|
|
|
+ results.add(embed(text, config));
|
|
|
|
|
+ }
|
|
|
|
|
+ return results;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ /**
|
|
|
|
|
+ * 构建 embedding 缓存 key
|
|
|
|
|
+ * 格式:embedding:cache:{md5(text)}:{model}:{dimension}
|
|
|
|
|
+ */
|
|
|
|
|
+ private String buildCacheKey(String text, String model, Integer dimension) {
|
|
|
|
|
+ String textHash = Md5Util.encoderByMd5(text);
|
|
|
|
|
+ if (!StringUtils.hasText(textHash)) {
|
|
|
|
|
+ return null;
|
|
|
}
|
|
}
|
|
|
|
|
+ String modelPart = StringUtils.hasText(model) ? model : "default";
|
|
|
|
|
+ String dimPart = dimension != null ? String.valueOf(dimension) : "0";
|
|
|
|
|
+ return CACHE_KEY_PREFIX + textHash + ":" + modelPart + ":" + dimPart;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
}
|
|
}
|