Merge branch 'wyp/0106-singleMiniprogramDelete' of Server/long-article-recommend into master

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/aigc/AigcBaseMapper.java

@@ -70,4 +70,6 @@ public interface AigcBaseMapper {
     List<String> getOnlyMiniprogramArticle(Integer offset, Integer pageSize);
 
     List<ProduceTaskAtom> getProduceScoreGreaterThan(Long startTimeStamp, Long endTimeStamp);
+
+    void updatePublishContentSingleMiniProgram(List<String> publishContentIds);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/longArticle/LongArticleBaseMapper.java

@@ -72,4 +72,6 @@ public interface LongArticleBaseMapper {
     void updateVideoPoolContentBad(String contentTraceId);
 
     void batchInsertArticleReMatchRecord(List<ArticleReMatchRecord> list);
+
+    List<String> getDisAuditDeleteGhid();
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/remote/WxArticleDeleteService.java

@@ -25,6 +25,7 @@ public class WxArticleDeleteService {
                 .build();
 
     }
+
     public RequestResult<String> deleteArticle(String token, String msgId, int index) {
         RequestResult<String> result = new RequestResult<>();
         JSONObject jsonBody = new JSONObject();
@@ -46,9 +47,13 @@ public class WxArticleDeleteService {
             log.info("wx delete api responseContent = {}", responseContent);
             if (response.isSuccessful()) {
                 JSONObject obj = JSONObject.parseObject(responseContent);
-                if (Objects.nonNull(obj) && obj.containsKey("errcode") && obj.getInteger("errcode") == 0) {
-                    result.setSuccess(true);
-                    result.setResponse(obj.toJSONString());
+                if (Objects.nonNull(obj) && obj.containsKey("errcode")) {
+                    if (obj.getInteger("errcode") == 0) {
+                        result.setSuccess(true);
+                        result.setResponse(obj.toJSONString());
+                    } else {
+                        result.setFailReason(obj.getString("errmsg"));
+                    }
                 } else {
                     result.setFailReason("response empty");
                 }
@@ -63,4 +68,46 @@ public class WxArticleDeleteService {
         }
         return result;
     }
+
+    public RequestResult<String> clearQuota(String token, String appid) {
+        RequestResult<String> result = new RequestResult<>();
+        JSONObject jsonBody = new JSONObject();
+        jsonBody.put("appid", appid);
+        try {
+            MediaType mediaType = MediaType.parse("application/json");
+            RequestBody body = RequestBody.create(mediaType, JSONObject.toJSONString(jsonBody));
+            Request request = new Request.Builder()
+                    .url("https://api.weixin.qq.com/cgi-bin/clear_quota?access_token=" + token)
+                    .method("POST", body)
+                    .addHeader("Content-Type", "application/json")
+                    .addHeader("Authorization", "Bearer sk-5DqYCa88kche6nwIWjLE1p4oMm8nXrR9kQMKbBolNAWERu7q")
+                    .build();
+            Response response = client.newCall(request).execute();
+
+            String responseContent = response.body().string();
+            result.setResponseStr(responseContent);
+            log.info("wx clearQuota api responseContent = {}", responseContent);
+            if (response.isSuccessful()) {
+                JSONObject obj = JSONObject.parseObject(responseContent);
+                if (Objects.nonNull(obj) && obj.containsKey("errcode")) {
+                    if (obj.getInteger("errcode") == 0) {
+                        result.setSuccess(true);
+                        result.setResponse(obj.toJSONString());
+                    } else {
+                        result.setFailReason(obj.getString("errmsg"));
+                    }
+                } else {
+                    result.setFailReason("response empty");
+                }
+            } else {
+                JSONObject json = JSONObject.parseObject(responseContent);
+                result.setFailReason("request error code:" + response.code() + " message:" + json.getString("error"));
+            }
+        } catch (Exception e) {
+            log.error("wx clearQuota api error", e);
+            result.setSuccess(false);
+            result.setFailReason("request error" + e.getMessage());
+        }
+        return result;
+    }
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/crawler/ArticleRepository.java

@@ -38,4 +38,6 @@ public interface ArticleRepository extends JpaRepository<Article, String> {
     List<Article> getByGhIdInAndUpdateTimeGreaterThanAndTypeEquals(List<String> ghIds, long l, String number);
 
     List<Article> getByTitleMd5(String titleMd5);
+
+    List<Article> getByPublishTimestampGreaterThan(Long publishTimestamp);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/ArticleAuditService.java

@@ -517,30 +517,38 @@ public class ArticleAuditService {
         CountDownLatch cdl = new CountDownLatch(dealMap.size());
         for (Map.Entry<String, List<LongArticleAuditDelete>> entry : dealMap.entrySet()) {
             pool.submit(() -> {
-                // 获取token
                 try {
+                    // 获取token
                     String token = wxAccessTokenRemoteService.getAccessToken(entry.getKey());
                     List<LongArticleAuditDelete> list = entry.getValue();
-                    for (LongArticleAuditDelete delete : list) {
-                        try {
-                            if (Objects.equals(delete.getPushType(), PushTypeEnum.AUTO_GROUP_PUBLISH.getVal())) {
-                                // 删除文章
-                                RequestResult<String> result = wxArticleDeleteService.deleteArticle(token, delete.getMsgId(), delete.getIndex());
-                                if (result.isSuccess()) {
-                                    delete.setStatus(ArticleDeleteStatusEnum.SUCCESS.getCode());
-                                } else {
-                                    delete.setStatus(ArticleDeleteStatusEnum.FAIL.getCode());
-                                    delete.setFailReason(result.getFailReason());
-                                }
+                    Map<String, List<LongArticleAuditDelete>> msgIdMap = list.stream()
+                            .collect(Collectors.groupingBy(LongArticleAuditDelete::getMsgId));
+                    for (Map.Entry<String, List<LongArticleAuditDelete>> msgIdEntry : msgIdMap.entrySet()) {
+                        String msgId = msgIdEntry.getKey();
+                        List<LongArticleAuditDelete> msgIdList = msgIdEntry.getValue();
+                        Integer pushType = msgIdList.get(0).getPushType();
+                        int status;
+                        String errMsg;
+                        if (Objects.equals(pushType, PushTypeEnum.AUTO_GROUP_PUBLISH.getVal())) {
+                            RequestResult<String> result = wxArticleDeleteService.deleteArticle(token, msgId, 0);
+                            if (result.isSuccess()) {
+                                errMsg = null;
+                                status = ArticleDeleteStatusEnum.SUCCESS.getCode();
                             } else {
-                                deleteFailAlarmAdd(alarmList, delete.getPublishContentId(), "非自动群发", delete.getIndex());
-                                delete.setStatus(ArticleDeleteStatusEnum.SUCCESS.getCode());
+                                status = ArticleDeleteStatusEnum.FAIL.getCode();
+                                errMsg = result.getFailReason();
                             }
+                        } else {
+                            errMsg = null;
+                            status = ArticleDeleteStatusEnum.SUCCESS.getCode();
+                            msgIdList.forEach(delete -> deleteFailAlarmAdd(alarmList, delete.getPublishContentId(), "非自动群发", delete.getIndex()));
+                        }
+                        msgIdList.forEach(delete -> {
+                            delete.setStatus(status);
+                            delete.setFailReason(errMsg);
                             delete.setFinishTimestamp(System.currentTimeMillis());
                             longArticleAuditDeleteRepository.save(delete);
-                        } catch (Exception e) {
-                            log.error("articleVideoDelete ghId:{} error", delete.getGhId(), e);
-                        }
+                        });
                     }
                 } finally {
                     cdl.countDown();
@@ -778,7 +786,12 @@ public class ArticleAuditService {
             if (CollectionUtils.isEmpty(publishContentIds)) {
                 break;
             }
-            buildArticleAuditDelete(publishContentIds);
+            List<LongArticleAuditDelete> exists = longArticleAuditDeleteRepository.getByPublishContentIdIn(publishContentIds);
+            List<String> existsIds = exists.stream().map(LongArticleAuditDelete::getPublishContentId).collect(Collectors.toList());
+            publishContentIds.removeAll(existsIds);
+            if (CollectionUtils.isNotEmpty(publishContentIds)) {
+                buildArticleAuditDelete(publishContentIds);
+            }
             pageNum++;
         }
     }

long-article-recommend-service/src/main/resources/mapper/aigc/AigcBaseMapper.xml

@@ -15,13 +15,9 @@
         join publish_content_output output
           on content.id = output.publish_content_id and output.content_type = 3 AND output.select_status = 1
         where content.status = 2
-          and content.publish_account_id in
+        and (content.publish_account_id, output.output) in
         <foreach collection="list" item="item" open="(" close=")" separator=",">
-            #{item.publishAccountId}
-        </foreach>
-          and output.output in
-        <foreach collection="list" item="item" open="(" close=")" separator=",">
-            #{item.title}
+            (#{item.publishAccountId}, #{item.title})
         </foreach>
     </select>
 
@@ -370,4 +366,11 @@
         order by record.audit_timestamp desc
     </select>
 
+    <update id="updatePublishContentSingleMiniProgram">
+        update publish_content set gzh_only_miniprogram_flag = 1 where id in
+        <foreach collection="publishContentIds" item="item" open="(" close=")" separator=",">
+            #{item}
+        </foreach>
+    </update>
+
 </mapper>

long-article-recommend-service/src/main/resources/mapper/longArticle/LongArticleBaseMapper.xml

@@ -267,4 +267,8 @@
         </foreach>
     </insert>
 
+    <select id="getDisAuditDeleteGhid" resultType="java.lang.String">
+        select distinct gh_id from long_article_audit_delete where create_timestamp > 1736157600000 and status = 2 and fail_reason like 'reach max api daily quota%'
+    </select>
+
 </mapper>

long-article-recommend-service/src/test/java/com/tzld/longarticle/recommend/server/RecommendTest.java

@@ -5,11 +5,16 @@ import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.google.common.collect.Lists;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.ArticleTypeEnum;
+import com.tzld.longarticle.recommend.server.mapper.aigc.AigcBaseMapper;
 import com.tzld.longarticle.recommend.server.mapper.crawler.CrawlerBaseMapper;
+import com.tzld.longarticle.recommend.server.model.dto.PublishContentDTO;
+import com.tzld.longarticle.recommend.server.model.entity.aigc.PublishAccount;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.AccountAvgInfo;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.Article;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.ArticleDetailInfo;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.PublishSortLog;
+import com.tzld.longarticle.recommend.server.model.param.PublishContentParam;
+import com.tzld.longarticle.recommend.server.repository.aigc.PublishAccountRepository;
 import com.tzld.longarticle.recommend.server.repository.crawler.AccountAvgInfoRepository;
 import com.tzld.longarticle.recommend.server.repository.crawler.ArticleDetailInfoRepository;
 import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
@@ -26,6 +31,7 @@ import org.apache.poi.ss.usermodel.Workbook;
 import org.apache.poi.xssf.usermodel.XSSFWorkbook;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.BeanUtils;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
 
 import javax.annotation.Resource;
@@ -55,6 +61,10 @@ public class RecommendTest {
     private CrawlerBaseMapper crawlerBaseMapper;
     @Resource
     private PublishSortLogRepository publishSortLogRepository;
+    @Autowired
+    private PublishAccountRepository publishAccountRepository;
+    @Autowired
+    private AigcBaseMapper aigcBaseMapper;
 
 //    @Test
 //    void recall() {
@@ -528,9 +538,9 @@ public class RecommendTest {
 
     @Test
     public void account() {
-        List<String> ghIds = Arrays.asList("gh_d7fa1998b4e1","gh_52100b6803fb","gh_8d7fc54d5026");
-        List<String> accountNames = Arrays.asList("生活超读","灵读生活","生活情感读");
-        List<Integer> fans = Arrays.asList(85759,103083,79214);
+        List<String> ghIds = Arrays.asList("gh_d7fa1998b4e1", "gh_52100b6803fb", "gh_8d7fc54d5026");
+        List<String> accountNames = Arrays.asList("生活超读", "灵读生活", "生活情感读");
+        List<Integer> fans = Arrays.asList(85759, 103083, 79214);
         List<Article> articleList = articleRepository.getByGhIdInAndPublishTimestampLessThanAndTypeEquals(
                 Arrays.asList("gh_02f5bca5b5d9"), 1729353600L, ArticleTypeEnum.QUNFA.getVal());
         for (int i = 0; i < ghIds.size(); i++) {
@@ -572,7 +582,7 @@ public class RecommendTest {
         List<Article> articleList = articleRepository.getByGhIdInAndUpdateTimeGreaterThanAndTypeEquals(ghIds, 1725638400L, "9");
         articleList = articleList.stream().filter(o -> o.getItemIndex() == 1).collect(Collectors.toList());
         Map<String, Map<String, Article>> articleMap = articleList.stream().collect(Collectors.groupingBy(Article::getGhId, Collectors.toMap(
-                o -> DateUtils.timestampToYMDStr(o.getUpdateTime(),"yyyyMMdd"), o -> o,
+                o -> DateUtils.timestampToYMDStr(o.getUpdateTime(), "yyyyMMdd"), o -> o,
                 (existing, replacement) -> replacement)));
         List<AccountAvgInfo> accountAvgInfoList = accountAvgInfoRepository.getAllByGhIdIn(new HashSet<>(ghIds));
         Map<String, Map<String, AccountAvgInfo>> accountAvgInfoMap = accountAvgInfoList.stream()
@@ -616,17 +626,17 @@ public class RecommendTest {
                 title = publishSortLog.getTitle();
                 scoreArr[i] = Double.parseDouble(publishSortLog.getScore());
                 JSONObject scoreMap = JSONObject.parseObject(publishSortLog.getScoreMap());
-                HisFissionFansRateRateStrategyArr[i] =  Double.parseDouble(String.format("%.3f", Optional.of(scoreMap.getDoubleValue("HisFissionFansRateRateStrategy")).orElse(0.0)));
-                HisFissionAvgReadRateRateStrategyArr[i] =  Double.parseDouble(String.format("%.3f", Optional.of(scoreMap.getDoubleValue("HisFissionAvgReadRateRateStrategy")).orElse(0.0)));
-                PublishTimesStrategyArr[i] =  Double.parseDouble(String.format("%.3f", Optional.of(scoreMap.getDoubleValue("PublishTimesStrategy")).orElse(0.0)));
-                ViewCountRateCorrelationStrategyArr[i] =  Double.parseDouble(String.format("%.3f", Optional.of(scoreMap.getDoubleValue("ViewCountRateCorrelationStrategy")).orElse(0.0)));
-                HisFissionAvgReadSumRateStrategyArr[i] =  Double.parseDouble(String.format("%.3f", Optional.of(scoreMap.getDoubleValue("HisFissionAvgReadSumRateStrategy")).orElse(0.0)));
-                HisFissionAvgReadRateCorrelationRateStrategyArr[i] =  Double.parseDouble(String.format("%.3f", Optional.of(scoreMap.getDoubleValue("HisFissionAvgReadRateCorrelationRateStrategy")).orElse(0.0)));
-                HisFissionFansSumRateStrategyArr[i] =  Double.parseDouble(String.format("%.3f", Optional.of(scoreMap.getDoubleValue("HisFissionFansSumRateStrategy")).orElse(0.0)));
-                SimilarityStrategyArr[i] =  Double.parseDouble(String.format("%.3f", Optional.of(scoreMap.getDoubleValue("SimilarityStrategy")).orElse(0.0)));
-                ViewCountStrategyArr[i] =  Double.parseDouble(String.format("%.3f", Optional.of(scoreMap.getDoubleValue("ViewCountStrategy")).orElse(0.0)));
-                ViewCountRateStrategyArr[i] =  Double.parseDouble(String.format("%.3f", Optional.of(scoreMap.getDoubleValue("ViewCountRateStrategy")).orElse(0.0)));
-                HisFissionDeWeightAvgReadSumRateStrategyArr[i] =  Double.parseDouble(String.format("%.3f", Optional.of(scoreMap.getDoubleValue("HisFissionDeWeightAvgReadSumRateStrategy")).orElse(0.0)));
+                HisFissionFansRateRateStrategyArr[i] = Double.parseDouble(String.format("%.3f", Optional.of(scoreMap.getDoubleValue("HisFissionFansRateRateStrategy")).orElse(0.0)));
+                HisFissionAvgReadRateRateStrategyArr[i] = Double.parseDouble(String.format("%.3f", Optional.of(scoreMap.getDoubleValue("HisFissionAvgReadRateRateStrategy")).orElse(0.0)));
+                PublishTimesStrategyArr[i] = Double.parseDouble(String.format("%.3f", Optional.of(scoreMap.getDoubleValue("PublishTimesStrategy")).orElse(0.0)));
+                ViewCountRateCorrelationStrategyArr[i] = Double.parseDouble(String.format("%.3f", Optional.of(scoreMap.getDoubleValue("ViewCountRateCorrelationStrategy")).orElse(0.0)));
+                HisFissionAvgReadSumRateStrategyArr[i] = Double.parseDouble(String.format("%.3f", Optional.of(scoreMap.getDoubleValue("HisFissionAvgReadSumRateStrategy")).orElse(0.0)));
+                HisFissionAvgReadRateCorrelationRateStrategyArr[i] = Double.parseDouble(String.format("%.3f", Optional.of(scoreMap.getDoubleValue("HisFissionAvgReadRateCorrelationRateStrategy")).orElse(0.0)));
+                HisFissionFansSumRateStrategyArr[i] = Double.parseDouble(String.format("%.3f", Optional.of(scoreMap.getDoubleValue("HisFissionFansSumRateStrategy")).orElse(0.0)));
+                SimilarityStrategyArr[i] = Double.parseDouble(String.format("%.3f", Optional.of(scoreMap.getDoubleValue("SimilarityStrategy")).orElse(0.0)));
+                ViewCountStrategyArr[i] = Double.parseDouble(String.format("%.3f", Optional.of(scoreMap.getDoubleValue("ViewCountStrategy")).orElse(0.0)));
+                ViewCountRateStrategyArr[i] = Double.parseDouble(String.format("%.3f", Optional.of(scoreMap.getDoubleValue("ViewCountRateStrategy")).orElse(0.0)));
+                HisFissionDeWeightAvgReadSumRateStrategyArr[i] = Double.parseDouble(String.format("%.3f", Optional.of(scoreMap.getDoubleValue("HisFissionDeWeightAvgReadSumRateStrategy")).orElse(0.0)));
                 Map<String, AccountAvgInfo> map = accountAvgInfoMap.get(article.getGhId());
                 if (Objects.nonNull(map)) {
                     List<String> avgMapDateList = new ArrayList<>(map.keySet());
@@ -688,7 +698,6 @@ public class RecommendTest {
     }
 
 
-
     @Test
     void getScoreFromLogFile() {
 
@@ -811,4 +820,43 @@ public class RecommendTest {
         }
     }
 
+    @Test
+    public void checkTest() {
+        List<Article> articleList = articleRepository.getByPublishTimestampGreaterThan(1732982400L);
+        List<Article> singleArticleList = new ArrayList<>();
+        for (Article article : articleList) {
+            if (Objects.nonNull(article.getRootSourceIdList())) {
+                try {
+                    List<String> rootSourceIdList = JSONArray.parseArray(article.getRootSourceIdList(), String.class);
+                    if (rootSourceIdList.size() == 1) {
+                        singleArticleList.add(article);
+                    }
+                } catch (Exception ignore) {
+                }
+            }
+        }
+        List<String> ghIds = singleArticleList.stream().map(Article::getGhId).distinct().collect(Collectors.toList());
+        List<PublishAccount> publishAccountList = publishAccountRepository.getAllByGhIdIn(ghIds);
+        log.info("newSortStrategyData publishAccountList finish");
+        Map<String, PublishAccount> publishAccountMap = publishAccountList.stream().collect(Collectors.toMap(PublishAccount::getGhId, o -> o));
+        // 获取发布内容
+        List<PublishContentParam> publishContentParamList = singleArticleList.stream().map(article -> {
+            PublishContentParam item = new PublishContentParam();
+            item.setTitle(article.getTitle());
+            PublishAccount account = publishAccountMap.get(article.getGhId());
+            if (Objects.nonNull(account)) {
+                item.setPublishAccountId(account.getId());
+                return item;
+            }
+            return null;
+        }).filter(Objects::nonNull).collect(Collectors.toList());
+        List<PublishContentDTO> publishContents = new ArrayList<>();
+        for (List<PublishContentParam> partitions : Lists.partition(publishContentParamList, 100)) {
+            publishContents.addAll(aigcBaseMapper.getPublishContentByTitle(partitions));
+        }
+        List<String> publishContentIds = publishContents.stream().map(PublishContentDTO::getId).collect(Collectors.toList());
+        for (List<String> partition : Lists.partition(publishContentIds, 500)) {
+            aigcBaseMapper.updatePublishContentSingleMiniProgram(partition);
+        }
+    }
 }

long-article-recommend-service/src/test/java/com/tzld/longarticle/recommend/server/WxApiTest.java

@@ -0,0 +1,36 @@
+package com.tzld.longarticle.recommend.server;
+
+import com.tzld.longarticle.recommend.server.mapper.aigc.AigcBaseMapper;
+import com.tzld.longarticle.recommend.server.mapper.longArticle.LongArticleBaseMapper;
+import com.tzld.longarticle.recommend.server.model.entity.aigc.PublishAccount;
+import com.tzld.longarticle.recommend.server.remote.WxAccessTokenRemoteService;
+import com.tzld.longarticle.recommend.server.remote.WxArticleDeleteService;
+import com.tzld.longarticle.recommend.server.repository.aigc.PublishAccountRepository;
+import org.junit.jupiter.api.Test;
+import org.springframework.boot.test.context.SpringBootTest;
+
+import javax.annotation.Resource;
+import java.util.List;
+
+@SpringBootTest(classes = Application.class)
+public class WxApiTest {
+
+    @Resource
+    private LongArticleBaseMapper longArticleBaseMapper;
+    @Resource
+    private PublishAccountRepository publishAccountRepository;
+    @Resource
+    private WxAccessTokenRemoteService wxAccessTokenRemoteService;
+    @Resource
+    private WxArticleDeleteService wxArticleDeleteService;
+
+    @Test
+    public void clearQuotaTest() {
+        List<String> ghIds = longArticleBaseMapper.getDisAuditDeleteGhid();
+        List<PublishAccount> publishAccountList = publishAccountRepository.getAllByGhIdIn(ghIds);
+        for (PublishAccount publishAccount : publishAccountList) {
+            String token = wxAccessTokenRemoteService.getAccessToken(publishAccount.getGhId());
+            wxArticleDeleteService.clearQuota(token, publishAccount.getAppId());
+        }
+    }
+}