Merge branch 'wyp/0313-videoPoolBatch' of Server/long-article-recommend into master

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/longArticle/VideoPoolAuditMapper.java

@@ -21,14 +21,15 @@ public interface VideoPoolAuditMapper {
                                                          ListItemFilterOrderParam auditTimestamp,
                                                          int offset, Integer pageSize, String poolLevelDesc);
 
-    PublishSingleVideoSource articleVideoAuditNext(List<String> contentId, List<Integer> status,
-                                                   List<String> title, List<String> auditAccount,
-                                                   Integer flowPoolLevel, List<String> excludeContentIds);
+    List<PublishSingleVideoSource> articleVideoAuditNext(List<String> contentId, List<Integer> status,
+                                                         List<String> title, List<String> auditAccount,
+                                                         Integer flowPoolLevel, List<String> excludeContentIds,
+                                                         Integer limit);
 
-    List<PublishSingleVideoSource> articleVideoWatingAuditList(List<Integer> status,
-                                                               List<Integer> flowPoolLevels,
-                                                               List<String> excludeContentIds,
-                                                               Integer size);
+    List<PublishSingleVideoSource> articleVideoWaitingAuditList(List<Integer> status,
+                                                                List<Integer> flowPoolLevels,
+                                                                List<String> excludeContentIds,
+                                                                Integer size);
 
     List<String> searchFilterValueByItemName(String itemName, String searchKeyword);
 
@@ -36,4 +37,7 @@ public interface VideoPoolAuditMapper {
 
     void updateVideoTitle(VideoTitleUpdateParam param, Long updateTime);
 
+    List<PublishSingleVideoSource> articleVideoAuditCount(String accountName, Long start, Long end);
+
+    int articleVideoWaitingAuditCount();
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/param/videoAudit/VideoPoolAuditDataParam.java

@@ -0,0 +1,10 @@
+package com.tzld.longarticle.recommend.server.model.param.videoAudit;
+
+import lombok.Data;
+
+@Data
+public class VideoPoolAuditDataParam {
+
+    private String auditAccountName;
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/param/videoAudit/VideoPoolAuditListParam.java

@@ -13,7 +13,8 @@ public class VideoPoolAuditListParam {
     private List<String> auditAccount;
     private List<Integer> flowPoolLevel;
     private ListItemFilterOrderParam auditTimestamp;
+    private String auditAccountName;
 
-    private Integer pageNum = 1;
-    private Integer pageSize = 50;
+    private Integer pageNum;
+    private Integer pageSize;
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/vo/VideoPoolAuditDataVO.java

@@ -0,0 +1,11 @@
+package com.tzld.longarticle.recommend.server.model.vo;
+
+import lombok.Data;
+
+@Data
+public class VideoPoolAuditDataVO {
+    private String auditAccount;
+    private Integer auditCount;
+    private Integer auditPassCount;
+    private Integer waitingAuditCount;
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/VideoPoolAuditService.java

@@ -12,6 +12,7 @@ import com.tzld.longarticle.recommend.server.mapper.longArticle.VideoPoolAuditMa
 import com.tzld.longarticle.recommend.server.model.entity.longArticle.PublishSingleVideoSource;
 import com.tzld.longarticle.recommend.server.model.entity.longArticle.VideoTitleReWrite;
 import com.tzld.longarticle.recommend.server.model.param.videoAudit.*;
+import com.tzld.longarticle.recommend.server.model.vo.VideoPoolAuditDataVO;
 import com.tzld.longarticle.recommend.server.model.vo.VideoPoolAuditListVO;
 import com.tzld.longarticle.recommend.server.repository.longArticle.PublishSingleVideoSourceRepository;
 import com.tzld.longarticle.recommend.server.repository.longArticle.VideoTitleReWriteRepository;
@@ -60,6 +61,10 @@ public class VideoPoolAuditService {
     private Map<String, Integer> dailyAuditPoolCount;
 
     public Page<VideoPoolAuditListVO> list(VideoPoolAuditListParam param) {
+        if (Objects.isNull(param.getPageNum()) || Objects.isNull(param.getPageSize())) {
+            param.setPageNum(1);
+            param.setPageSize(50);
+        }
         int offset = (param.getPageNum() - 1) * param.getPageSize();
         int count = videoPoolAuditMapper.articleVideoAuditListCount(param.getContentId(), param.getStatus(),
                 param.getTitle(), param.getAuditAccount(), param.getFlowPoolLevel(), param.getAuditTimestamp());
@@ -139,55 +144,80 @@ public class VideoPoolAuditService {
             return result; // 返回空结果
         }
         long now = System.currentTimeMillis();
-        String id;
-        PublishSingleVideoSource obj = null;
-        String inAuditListRedisKey = "video-pool-in-audit-list";
+        if (Objects.isNull(param.getPageSize())) {
+            param.setPageSize(1);
+        }
+        int limit = param.getPageSize();
+        List<PublishSingleVideoSource> objs = new ArrayList<>();
+        String inAuditListRedisKey = "video-pool-in-audit-list-" + param.getAuditAccountName();
         try {
             String dateStr = DateUtils.getCurrentDateStr("yyyyMMdd");
             String auditQueueRedisKey = "video-pool-audit-queue-" + dateStr;
             // 从待审核队列中获取数据，如未获取到则从数据库查询一条
             Long size = redisTemplate.opsForZSet().size(auditQueueRedisKey);
-            if (Objects.isNull(size) || size == 0) {
-                List<String> entries = redisTemplate.opsForList().range(inAuditListRedisKey, 0, -1);
-                List<String> excludeContentIds = new ArrayList<>();
-                if (CollectionUtils.isNotEmpty(entries)) {
-                    excludeContentIds = entries.stream().map(o -> {
-                        JSONObject json = JSONObject.parseObject(o);
-                        return json.getString("id");
-                    }).collect(Collectors.toList());
-                }
-                obj = videoPoolAuditMapper.articleVideoAuditNext(param.getContentId(),
-                        param.getStatus(), param.getTitle(), param.getAuditAccount(), null, excludeContentIds);
-            } else {
+            if (Objects.nonNull(size) && size > 0) {
                 // 从待审核队列中获取数据, 如获取到的内容已审核，则重新获取
-                while (true) {
-                    Set<String> ids = redisTemplate.opsForZSet().reverseRangeByScore(auditQueueRedisKey, 0, 100, 0, 1);
-                    if (CollectionUtils.isNotEmpty(ids)) {
-                        id = ids.iterator().next();
-                        redisTemplate.opsForZSet().remove(auditQueueRedisKey, id);
-                        obj = videoSourceRepository.getByContentTraceId(id);
-                        if (obj.getVideoPoolAuditStatus() == ArticleVideoAuditStatusEnum.WAITING.getCode()) {
-                            break;
-                        }
-                    } else {
+                while (limit > 0) {
+                    Set<String> ids = redisTemplate.opsForZSet().reverseRangeByScore(auditQueueRedisKey, 0, 100, 0, limit);
+                    if (CollectionUtils.isEmpty(ids)) {
+                        break;
+                    }
+                    List<String> contentTraceIdList = new ArrayList<>(ids);
+                    List<PublishSingleVideoSource> checkList = videoSourceRepository.getByContentTraceIdIn(contentTraceIdList);
+                    if (CollectionUtils.isEmpty(checkList)) {
                         break;
                     }
+                    for (PublishSingleVideoSource videoSource : checkList) {
+                        redisTemplate.opsForZSet().remove(auditQueueRedisKey, videoSource.getContentTraceId());
+                        if (videoSource.getVideoPoolAuditStatus() == ArticleVideoAuditStatusEnum.WAITING.getCode()) {
+                            objs.add(videoSource);
+                            limit--;
+                        }
+                    }
+                }
+            }
+            if (limit > 0) {
+                List<String> excludeContentIds = new ArrayList<>();
+                String auditAccountListKey = "video-pool-audit-account-list-" + dateStr;
+                Set<String> auditAccountList = redisTemplate.opsForSet().members(auditAccountListKey);
+                if (CollectionUtils.isNotEmpty(auditAccountList)) {
+                    for (String auditAccount : auditAccountList) {
+                        String inAuditAccountListRedisKey = "video-pool-in-audit-list-" + auditAccount;
+                        List<String> entries = redisTemplate.opsForList().range(inAuditAccountListRedisKey, 0, -1);
+                        if (CollectionUtils.isNotEmpty(entries)) {
+                            excludeContentIds.addAll(entries.stream().map(o -> {
+                                JSONObject json = JSONObject.parseObject(o);
+                                return json.getString("id");
+                            }).collect(Collectors.toList()));
+                        }
+                    }
+                }
+                objs.addAll(videoPoolAuditMapper.articleVideoAuditNext(param.getContentId(),
+                        param.getStatus(), param.getTitle(), param.getAuditAccount(), null,
+                        excludeContentIds, limit));
+            }
+            // 添加到超时队列
+            if (CollectionUtils.isNotEmpty(objs)) {
+                for (PublishSingleVideoSource obj : objs) {
+                    JSONObject json = new JSONObject();
+                    json.put("id", obj.getContentTraceId());
+                    json.put("timestamp", now + (120 * 60 * 1000));
+                    redisTemplate.opsForList().rightPush(inAuditListRedisKey, json.toJSONString());
+                    redisTemplate.expire(inAuditListRedisKey, 12, TimeUnit.HOURS);
                 }
+                String auditAccountListKey = "video-pool-audit-account-list-" + dateStr;
+                redisTemplate.opsForSet().add(auditAccountListKey, param.getAuditAccountName());
+                redisTemplate.expire(auditAccountListKey, 12, TimeUnit.HOURS);
             }
         } finally {
             // 释放锁
             redisUtil.releaseLock(lockKey, requestId);
         }
-        if (Objects.isNull(obj)) {
+        if (CollectionUtils.isEmpty(objs)) {
             return result;
         }
-        // 添加到超时队列
-        JSONObject json = new JSONObject();
-        json.put("id", obj.getContentTraceId());
-        json.put("timestamp", now + (10 * 60 * 1000));
-        redisTemplate.opsForList().rightPush(inAuditListRedisKey, json.toJSONString());
         // 填充数据
-        List<VideoPoolAuditListVO> list = buildVideoPoolAuditListVO(Collections.singletonList(obj));
+        List<VideoPoolAuditListVO> list = buildVideoPoolAuditListVO(objs);
         result.setObjs(list);
         return result;
     }
@@ -280,36 +310,42 @@ public class VideoPoolAuditService {
         try {
             Long now = System.currentTimeMillis();
             String dateStr = DateUtils.getCurrentDateStr("yyyyMMdd");
-            String inAuditListRedisKey = "video-pool-in-audit-list";
             String auditQueueRedisKey = "video-pool-audit-queue-" + dateStr;
             // 判断是否审核超时，重新加入待审核队列
-            while (true) {
-                List<String> inAuditList = redisTemplate.opsForList().range(inAuditListRedisKey, 0, 0);
-                if (CollectionUtils.isEmpty(inAuditList)) {
-                    break;
-                }
-                JSONObject firstObj = JSONObject.parseObject(inAuditList.get(0));
-                if (now > firstObj.getLong("timestamp")) {
-                    redisTemplate.opsForList().leftPop(inAuditListRedisKey);
-                    String id = firstObj.getString("id");
-                    PublishSingleVideoSource videoAudit = videoSourceRepository.getByContentTraceId(id);
-                    if (videoAudit.getVideoPoolAuditStatus() == ArticleVideoAuditStatusEnum.WAITING.getCode()) {
-                        ContentPoolEnum poolEnum = ContentPoolEnum.from(videoAudit.getFlowPoolLevel());
-                        redisTemplate.opsForZSet().add(auditQueueRedisKey, id, poolEnum.getWeight());
-                    }
-                } else {
-                    break;
-                }
-            }
+            String auditAccountListKey = "video-pool-audit-account-list-" + dateStr;
+            Set<String> auditAccountList = redisTemplate.opsForSet().members(auditAccountListKey);
             List<String> excludeContentIds = new ArrayList<>();
-            // 审核中列表获取，按内容池分组
-            List<String> inAuditList = redisTemplate.opsForList().range(inAuditListRedisKey, 0, -1);
             Map<String, List<PublishSingleVideoSource>> inAuditListPoolCountMap = new HashMap<>();
-            if (CollectionUtils.isNotEmpty(inAuditList)) {
-                List<String> inAuditListIds = inAuditList.stream().map(item -> JSONObject.parseObject(item).getString("id")).collect(Collectors.toList());
-                List<PublishSingleVideoSource> auditQueueList = videoSourceRepository.getByContentTraceIdIn(new ArrayList<>(inAuditListIds));
-                inAuditListPoolCountMap = auditQueueList.stream().collect(Collectors.groupingBy(item -> ContentPoolEnum.from(item.getFlowPoolLevel()).getContentPool()));
-                excludeContentIds.addAll(inAuditListIds);
+            if (CollectionUtils.isNotEmpty(auditAccountList)) {
+                for (String auditAccount : auditAccountList) {
+                    String inAuditListRedisKey = "video-pool-in-audit-list-" + auditAccount;
+                    while (true) {
+                        List<String> inAuditList = redisTemplate.opsForList().range(inAuditListRedisKey, 0, 0);
+                        if (CollectionUtils.isEmpty(inAuditList)) {
+                            break;
+                        }
+                        JSONObject firstObj = JSONObject.parseObject(inAuditList.get(0));
+                        if (now > firstObj.getLong("timestamp")) {
+                            redisTemplate.opsForList().leftPop(inAuditListRedisKey);
+                            String id = firstObj.getString("id");
+                            PublishSingleVideoSource videoAudit = videoSourceRepository.getByContentTraceId(id);
+                            if (videoAudit.getVideoPoolAuditStatus() == ArticleVideoAuditStatusEnum.WAITING.getCode()) {
+                                ContentPoolEnum poolEnum = ContentPoolEnum.from(videoAudit.getFlowPoolLevel());
+                                redisTemplate.opsForZSet().add(auditQueueRedisKey, id, poolEnum.getWeight());
+                            }
+                        } else {
+                            break;
+                        }
+                    }
+                    // 审核中列表获取，按内容池分组
+                    List<String> inAuditList = redisTemplate.opsForList().range(inAuditListRedisKey, 0, -1);
+                    if (CollectionUtils.isNotEmpty(inAuditList)) {
+                        List<String> inAuditListIds = inAuditList.stream().map(item -> JSONObject.parseObject(item).getString("id")).collect(Collectors.toList());
+                        List<PublishSingleVideoSource> auditQueueList = videoSourceRepository.getByContentTraceIdIn(new ArrayList<>(inAuditListIds));
+                        inAuditListPoolCountMap = auditQueueList.stream().collect(Collectors.groupingBy(item -> ContentPoolEnum.from(item.getFlowPoolLevel()).getContentPool()));
+                        excludeContentIds.addAll(inAuditListIds);
+                    }
+                }
             }
             // 待审核列表获取，按内容池分组
             Set<String> auditQueueIds = redisTemplate.opsForZSet().rangeByScore(auditQueueRedisKey, 0, 100);
@@ -329,7 +365,7 @@ public class VideoPoolAuditService {
                     List<PublishSingleVideoSource> inAuditListPool = inAuditListPoolCountMap.getOrDefault(poolEnum.getContentPool(), new ArrayList<>());
                     int needCount = entry.getValue() - (totalCount + auditQueueList.size() + inAuditListPool.size());
                     if (needCount > 0) {
-                        List<PublishSingleVideoSource> addList = videoPoolAuditMapper.articleVideoWatingAuditList(
+                        List<PublishSingleVideoSource> addList = videoPoolAuditMapper.articleVideoWaitingAuditList(
                                 Arrays.asList(ArticleVideoAuditStatusEnum.WAITING.getCode()),
                                 Arrays.asList(poolEnum.getValue()), excludeContentIds, needCount);
                         if (CollectionUtils.isNotEmpty(addList)) {
@@ -343,9 +379,9 @@ public class VideoPoolAuditService {
             }
             // 待发布内容不足添加
             Long auditQueueSize = redisTemplate.opsForZSet().size(auditQueueRedisKey);
-            if (Objects.isNull(auditQueueSize) || auditQueueSize < 20) {
-                List<PublishSingleVideoSource> list = videoPoolAuditMapper.articleVideoWatingAuditList(
-                        Arrays.asList(ArticleVideoAuditStatusEnum.WAITING.getCode()), null, excludeContentIds, 40);
+            if (Objects.isNull(auditQueueSize) || auditQueueSize < 100) {
+                List<PublishSingleVideoSource> list = videoPoolAuditMapper.articleVideoWaitingAuditList(
+                        Arrays.asList(ArticleVideoAuditStatusEnum.WAITING.getCode()), null, excludeContentIds, 100);
                 if (CollectionUtils.isNotEmpty(list)) {
                     for (PublishSingleVideoSource item : list) {
                         ContentPoolEnum poolEnum = ContentPoolEnum.from(item.getFlowPoolLevel());
@@ -359,4 +395,26 @@ public class VideoPoolAuditService {
         }
         return ReturnT.SUCCESS;
     }
+
+    public VideoPoolAuditDataVO auditData(VideoPoolAuditDataParam param) {
+        String accountName = param.getAuditAccountName();
+        Long start = DateUtils.getTodayStart();
+        Long end = System.currentTimeMillis();
+        List<PublishSingleVideoSource> auditCount = videoPoolAuditMapper.articleVideoAuditCount(accountName, start, end);
+        int auditPassCount = auditCount.stream()
+                .filter(item -> item.getVideoPoolAuditStatus() == ArticleVideoAuditStatusEnum.PASS.getCode())
+                .collect(Collectors.toList()).size();
+        int waitingAuditCount = videoPoolAuditMapper.articleVideoWaitingAuditCount();
+        VideoPoolAuditDataVO result = new VideoPoolAuditDataVO();
+        result.setAuditAccount(accountName);
+        result.setAuditCount(auditCount.size());
+        result.setAuditPassCount(auditPassCount);
+        result.setWaitingAuditCount(waitingAuditCount);
+        return result;
+    }
+
+    public void getOff(VideoPoolAuditDataParam param) {
+        String inAuditListRedisKey = "video-pool-in-audit-list-" + param.getAuditAccountName();
+        redisTemplate.delete(inAuditListRedisKey);
+    }
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/web/recommend/VideoPoolAuditController.java

@@ -2,6 +2,7 @@ package com.tzld.longarticle.recommend.server.web.recommend;
 
 import com.tzld.longarticle.recommend.server.common.response.CommonResponse;
 import com.tzld.longarticle.recommend.server.model.param.videoAudit.*;
+import com.tzld.longarticle.recommend.server.model.vo.VideoPoolAuditDataVO;
 import com.tzld.longarticle.recommend.server.model.vo.VideoPoolAuditListVO;
 import com.tzld.longarticle.recommend.server.service.recommend.VideoPoolAuditService;
 import com.tzld.longarticle.recommend.server.util.page.Page;
@@ -29,6 +30,17 @@ public class VideoPoolAuditController {
         return CommonResponse.success(service.next(param));
     }
 
+    @PostMapping("/auditData")
+    public CommonResponse<VideoPoolAuditDataVO> articleVideoAuditData(@RequestBody VideoPoolAuditDataParam param) {
+        return CommonResponse.success(service.auditData(param));
+    }
+
+    @PostMapping("/getOff")
+    public CommonResponse<Void> getOff(@RequestBody VideoPoolAuditDataParam param) {
+        service.getOff(param);
+        return CommonResponse.success();
+    }
+
     @PostMapping("/auditArticle")
     public CommonResponse<Void> auditArticle(@RequestBody ArticleAuditParam param) {
         service.auditArticle(param);

long-article-recommend-service/src/main/resources/mapper/longArticle/VideoPoolAuditMapper.xml

@@ -162,7 +162,7 @@
             and flow_pool_level = #{flowPoolLevel}
         </if>
         order by crawler_timestamp desc
-        limit 1
+        limit #{limit}
     </select>
 
     <select id="searchFilterValueByItemName" resultType="java.lang.String">
@@ -203,7 +203,7 @@
         </choose>
     </select>
 
-    <select id="articleVideoWatingAuditList"
+    <select id="articleVideoWaitingAuditList"
             resultType="com.tzld.longarticle.recommend.server.model.entity.longArticle.PublishSingleVideoSource">
         select *
         from publish_single_video_source
@@ -248,5 +248,18 @@
         where content_trace_id = #{param.contentId}
     </update>
 
+    <select id="articleVideoAuditCount" resultType="com.tzld.longarticle.recommend.server.model.entity.longArticle.PublishSingleVideoSource">
+        select *
+        from publish_single_video_source
+        where bad_status in (0, 5)
+        and video_pool_audit_account = #{accountName}
+        and video_pool_audit_timestamp between #{start} and #{end}
+    </select>
+
+    <select id="articleVideoWaitingAuditCount" resultType="java.lang.Integer">
+        select count(1)
+        from publish_single_video_source
+        where bad_status = 0 and audit_status = 1 and video_pool_audit_status = 0
+    </select>
 
 </mapper>