|
@@ -9,12 +9,15 @@ import com.google.common.collect.Lists;
|
|
|
import com.google.common.util.concurrent.ThreadFactoryBuilder;
|
|
|
import com.tzld.longarticle.recommend.server.common.CommonThreadPoolExecutor;
|
|
|
import com.tzld.longarticle.recommend.server.common.HttpPoolFactory;
|
|
|
+import com.tzld.longarticle.recommend.server.common.constant.SceneConstants;
|
|
|
import com.tzld.longarticle.recommend.server.common.enums.FieshuTableColumnDataTypeEnum;
|
|
|
import com.tzld.longarticle.recommend.server.common.enums.aigc.ChannelEnum;
|
|
|
import com.tzld.longarticle.recommend.server.common.enums.aigc.MiniprogramUseTypeEnum;
|
|
|
import com.tzld.longarticle.recommend.server.common.enums.aigc.ProduceContentAuditStatusEnum;
|
|
|
+import com.tzld.longarticle.recommend.server.common.enums.aigc.PushTypeEnum;
|
|
|
import com.tzld.longarticle.recommend.server.common.enums.longArticle.ArticleVideoAuditStatusEnum;
|
|
|
import com.tzld.longarticle.recommend.server.common.enums.recommend.AccountBusinessTypeEnum;
|
|
|
+import com.tzld.longarticle.recommend.server.common.enums.recommend.ArticleTypeEnum;
|
|
|
import com.tzld.longarticle.recommend.server.common.enums.recommend.ContentPoolEnum;
|
|
|
import com.tzld.longarticle.recommend.server.common.enums.recommend.FeishuRobotIdEnum;
|
|
|
import com.tzld.longarticle.recommend.server.mapper.aigc.AigcBaseMapper;
|
|
@@ -31,13 +34,17 @@ import com.tzld.longarticle.recommend.server.model.entity.crawler.GetOffVideoCra
|
|
|
import com.tzld.longarticle.recommend.server.model.entity.crawler.LongArticlesVideo;
|
|
|
import com.tzld.longarticle.recommend.server.model.entity.longArticle.*;
|
|
|
import com.tzld.longarticle.recommend.server.model.param.ArticleFindSourceParam;
|
|
|
+import com.tzld.longarticle.recommend.server.model.param.RecommendParam;
|
|
|
+import com.tzld.longarticle.recommend.server.model.param.RecommendRequest;
|
|
|
import com.tzld.longarticle.recommend.server.model.param.TitleHisCacheParam;
|
|
|
+import com.tzld.longarticle.recommend.server.model.vo.ArticleSortResponseDataItem;
|
|
|
import com.tzld.longarticle.recommend.server.model.vo.FeishuTableDTO;
|
|
|
import com.tzld.longarticle.recommend.server.remote.ODPSManager;
|
|
|
import com.tzld.longarticle.recommend.server.remote.aigc.AIGCProduceContentAuditService;
|
|
|
import com.tzld.longarticle.recommend.server.remote.aigc.AIGCWaitingPublishContentService;
|
|
|
import com.tzld.longarticle.recommend.server.repository.aigc.ProducePlanRepository;
|
|
|
import com.tzld.longarticle.recommend.server.repository.aigc.PublishPlanRepository;
|
|
|
+import com.tzld.longarticle.recommend.server.repository.aigc.PublishPlanSettingRepository;
|
|
|
import com.tzld.longarticle.recommend.server.repository.crawler.GetOffVideoCrawlerRepository;
|
|
|
import com.tzld.longarticle.recommend.server.repository.crawler.LongArticlesVideoRepository;
|
|
|
import com.tzld.longarticle.recommend.server.repository.longArticle.*;
|
|
@@ -45,6 +52,10 @@ import com.tzld.longarticle.recommend.server.repository.model.PushMessageCallbac
|
|
|
import com.tzld.longarticle.recommend.server.service.recommend.ArticleCategoryService;
|
|
|
import com.tzld.longarticle.recommend.server.service.recommend.ArticlePromotionService;
|
|
|
import com.tzld.longarticle.recommend.server.service.recommend.ArticleService;
|
|
|
+import com.tzld.longarticle.recommend.server.service.recommend.RecommendService;
|
|
|
+import com.tzld.longarticle.recommend.server.service.recommend.filter.FilterParam;
|
|
|
+import com.tzld.longarticle.recommend.server.service.recommend.filter.FilterResult;
|
|
|
+import com.tzld.longarticle.recommend.server.service.recommend.filter.FilterService;
|
|
|
import com.tzld.longarticle.recommend.server.service.recommend.recall.RecallService;
|
|
|
import com.tzld.longarticle.recommend.server.util.DateUtils;
|
|
|
import com.tzld.longarticle.recommend.server.util.LarkRobotUtil;
|
|
@@ -60,11 +71,13 @@ import org.apache.http.client.methods.CloseableHttpResponse;
|
|
|
import org.apache.http.client.methods.HttpGet;
|
|
|
import org.apache.http.client.utils.URIBuilder;
|
|
|
import org.apache.http.impl.client.CloseableHttpClient;
|
|
|
+import org.springframework.beans.BeanUtils;
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
|
import org.springframework.dao.DuplicateKeyException;
|
|
|
import org.springframework.data.domain.Page;
|
|
|
import org.springframework.data.domain.PageRequest;
|
|
|
+import org.springframework.data.redis.core.RedisTemplate;
|
|
|
import org.springframework.stereotype.Service;
|
|
|
import org.springframework.util.StringUtils;
|
|
|
|
|
@@ -131,6 +144,15 @@ public class XxlJobService {
|
|
|
private PublishContentGzhWaitingRepository publishContentGzhWaitingRepository;
|
|
|
@Autowired
|
|
|
private LongArticleTitleAuditRepository titleAuditRepository;
|
|
|
+ @Autowired
|
|
|
+ private FilterService filterService;
|
|
|
+ @Autowired
|
|
|
+ private RecommendService recommendService;
|
|
|
+ @Autowired
|
|
|
+ private PublishPlanSettingRepository publishPlanSettingRepository;
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ private RedisTemplate<String, String> redisTemplate;
|
|
|
|
|
|
ExecutorService thread = new CommonThreadPoolExecutor(
|
|
|
5,
|
|
@@ -1095,4 +1117,83 @@ public class XxlJobService {
|
|
|
log.info("syncGzhWaitingPublishContent success planId: {} accountId: {}", planId, accountId);
|
|
|
}
|
|
|
|
|
|
+ @XxlJob("ContentPreFilterJob")
|
|
|
+ public ReturnT<String> gzhWaitingPublishContentPreFilter(String param) {
|
|
|
+ List<PublishPlanAccountDTO> planAccountList = longArticleBaseMapper.getGroupPublishPlanAccounts();
|
|
|
+ ExecutorService thread = new CommonThreadPoolExecutor(
|
|
|
+ syncPublishContentThreadPoolSize, syncPublishContentThreadPoolSize, 0L, TimeUnit.SECONDS,
|
|
|
+ new LinkedBlockingQueue<>(),
|
|
|
+ new ThreadFactoryBuilder().setNameFormat("ContentPreFilter-%d").build(),
|
|
|
+ new ThreadPoolExecutor.AbortPolicy());
|
|
|
+ CountDownLatch cdl = new CountDownLatch(planAccountList.size());
|
|
|
+ String dateStr = DateUtils.getCurrentDateStr("yyyyMMdd");
|
|
|
+ for (PublishPlanAccountDTO item : planAccountList) {
|
|
|
+ String redisKey = "ContentPreFilterJob:" + dateStr + ":" + item.getPlanId() + "-" + item.getAccountId();
|
|
|
+ thread.submit(() -> {
|
|
|
+ try {
|
|
|
+ String value = redisTemplate.opsForValue().get(redisKey);
|
|
|
+ if (StringUtils.hasText(value)) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ Account account = accountRepository.getById(item.getAccountId());
|
|
|
+ List<Content> contentList = longArticleBaseMapper.getPublishContentGzhWaiting(item.getPlanId(), item.getAccountId());
|
|
|
+ // 预处理过滤
|
|
|
+ if (Objects.isNull(account) || CollectionUtil.isEmpty(contentList)) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ FilterParam filterParam = buildFilterParam(item.getPlanId(), account, contentList);
|
|
|
+ FilterResult filterResult = filterService.filter(filterParam, true);
|
|
|
+ if (CollectionUtil.isNotEmpty(filterResult.getFilterContent())) {
|
|
|
+ List<ArticleSortResponseDataItem> filterContentList = new ArrayList<>();
|
|
|
+ for (Content filterContent : filterResult.getFilterContent()) {
|
|
|
+ ArticleSortResponseDataItem saveItem = new ArticleSortResponseDataItem();
|
|
|
+ BeanUtils.copyProperties(filterContent, saveItem);
|
|
|
+ filterContentList.add(saveItem);
|
|
|
+ }
|
|
|
+ recommendService.updateWaitingContentFilter(filterContentList);
|
|
|
+ }
|
|
|
+ } finally {
|
|
|
+ redisTemplate.opsForValue().set(redisKey, "1", 1, TimeUnit.DAYS);
|
|
|
+ cdl.countDown();
|
|
|
+ }
|
|
|
+ });
|
|
|
+ }
|
|
|
+ try {
|
|
|
+ cdl.await();
|
|
|
+ } catch (InterruptedException e) {
|
|
|
+ log.error("gzhWaitingPublishContentPreFilter error", e);
|
|
|
+ }
|
|
|
+ return ReturnT.SUCCESS;
|
|
|
+ }
|
|
|
+
|
|
|
+ private FilterParam buildFilterParam(String planId, Account account, List<Content> contentList) {
|
|
|
+ PublishPlanSetting publishPlanSetting = publishPlanSettingRepository.getByPlanId(planId);
|
|
|
+ String type = ArticleTypeEnum.QUNFA.getVal();
|
|
|
+ if (Objects.equals(publishPlanSetting.getPushType(), PushTypeEnum.AUTO_PUBLISH.getVal())
|
|
|
+ || Objects.equals(publishPlanSetting.getPushType(), PushTypeEnum.ROBOPOST.getVal())) {
|
|
|
+ type = ArticleTypeEnum.WUXIANLIU.getVal();
|
|
|
+ }
|
|
|
+ FilterParam filterParam = new FilterParam();
|
|
|
+ filterParam.setPlanId(planId);
|
|
|
+ filterParam.setAccountId(account.getId());
|
|
|
+ filterParam.setAccountName(account.getName());
|
|
|
+ filterParam.setGhId(account.getGhId());
|
|
|
+ filterParam.setType(type);
|
|
|
+ filterParam.setStrategy(getStrategy(planId, account.getGhId(), account.getName(), publishPlanSetting.getPushType()));
|
|
|
+ filterParam.setScene(SceneConstants.DEFAULT);
|
|
|
+ filterParam.setContents(contentList);
|
|
|
+ return filterParam;
|
|
|
+ }
|
|
|
+
|
|
|
+ private String getStrategy(String planId, String ghId, String accountName, Integer pushType) {
|
|
|
+ RecommendRequest request = new RecommendRequest();
|
|
|
+ request.setGhId(ghId);
|
|
|
+ request.setAccountName(accountName);
|
|
|
+ request.setPushType(pushType);
|
|
|
+ RecommendParam param = new RecommendParam();
|
|
|
+ param.setPlanId(planId);
|
|
|
+ recommendService.setStrategy(request, param);
|
|
|
+ return param.getStrategy();
|
|
|
+ }
|
|
|
+
|
|
|
}
|