Merge branch 'wyp/0207-badAccountContentReject' of Server/long-article-recommend into master

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/aigc/AigcBaseMapper.java

@@ -4,10 +4,8 @@ import com.tzld.longarticle.recommend.server.model.dto.ContentPoolTypeDTO;
 import com.tzld.longarticle.recommend.server.model.dto.CrawlerContent;
 import com.tzld.longarticle.recommend.server.model.dto.ProduceContentDTO;
 import com.tzld.longarticle.recommend.server.model.dto.ProducePlanAuditCheckDTO;
-import com.tzld.longarticle.recommend.server.model.entity.aigc.CrawlerPlan;
-import com.tzld.longarticle.recommend.server.model.entity.aigc.CrawlerPlanResultRel;
-import com.tzld.longarticle.recommend.server.model.entity.aigc.ProducePlanExeRecord;
-import com.tzld.longarticle.recommend.server.model.entity.aigc.ProduceTaskAtom;
+import com.tzld.longarticle.recommend.server.model.dto.aigc.BadCrawlerAccountDTO;
+import com.tzld.longarticle.recommend.server.model.entity.aigc.*;
 import org.apache.ibatis.annotations.Mapper;
 
 import java.util.List;
@@ -51,4 +49,13 @@ public interface AigcBaseMapper {
 
     Long getProduceAuditPassCountByProducePlanIds(List<String> producePlanIds, List<Integer> auditStatus, Long start, Long end);
 
+    Long getProduceAuditPassCountByProducePlanIds(List<String> producePlanIds, Long start, Long end);
+
+    List<String> getPlanExeIdByAccountName(List<String> accountNames);
+
+    List<ProduceReviewRecord> getReviewList(List<String> planExeIds);
+
+    List<BadCrawlerAccountDTO> getBadAccountList(List<String> producePlanIds);
+
+    void setCrawlerContentDisuseByAccountName(List<String> accountNames);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/dto/aigc/BadCrawlerAccountDTO.java

@@ -0,0 +1,10 @@
+package com.tzld.longarticle.recommend.server.model.dto.aigc;
+
+import lombok.Data;
+
+@Data
+public class BadCrawlerAccountDTO {
+    private String accountName;
+    private Double rate;
+    private Integer totalCount;
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/aigc/ProduceReviewRecord.java

@@ -0,0 +1,40 @@
+package com.tzld.longarticle.recommend.server.model.entity.aigc;
+
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.Table;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Entity
+@Table(name = "produce_review_record")
+public class ProduceReviewRecord {
+
+    @Id
+    @Column(name = "record_id")
+    private String recordId;
+    @Column(name = "plan_exe_id")
+    private String planExeId;
+    @Column(name = "task_id")
+    private String taskId;
+    @Column(name = "review_type")
+    private Integer reviewType;
+    @Column(name = "review_status")
+    private Integer reviewStatus;
+    @Column(name = "review_account")
+    private String reviewAccount;
+    @Column(name = "create_timestamp")
+    private Long createTimestamp;
+    @Column(name = "review_timestamp")
+    private Long reviewTimestamp;
+    @Column(name = "review_count")
+    private Integer reviewCount;
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/remote/aigc/AIGCProduceContentAuditService.java

@@ -0,0 +1,59 @@
+package com.tzld.longarticle.recommend.server.remote.aigc;
+
+import com.alibaba.fastjson.JSONObject;
+import lombok.extern.slf4j.Slf4j;
+import okhttp3.*;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.PostConstruct;
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+@Service
+@Slf4j
+public class AIGCProduceContentAuditService {
+
+    private OkHttpClient client;
+
+    @PostConstruct
+    public void init() {
+        client = new OkHttpClient().newBuilder()
+                .connectTimeout(15, TimeUnit.MINUTES)
+                .readTimeout(15, TimeUnit.MINUTES)
+                .writeTimeout(15, TimeUnit.MINUTES)
+                .build();
+    }
+
+    public void request(String reviewId, String planExeId) {
+        String str = "{\"params\":{\"postOutputReviewDetail\":{\"auditlayoutList\":null,\"bodyTextModuleOutputGroups\":[],\"coverModuleOutputGroups\":[],\"id\":\"20250111090400217509503\",\"imageModuleOutputGroups\":[],\"titleModuleOutputGroups\":[],\"animationModuleOutputGroups\":[],\"videoModuleOutputGroups\":[],\"bgmModuleOutputGroups\":[],\"miniProgramModuleOutputGroups\":[]},\"recordId\":\"20250111090709426752083\",\"reviewStatus\":2},\"baseInfo\":{\"token\":\"9ebfcb397e954c41986971f183eb1707\",\"appType\":9,\"platform\":\"pc\",\"appVersionCode\":1000,\"clientTimestamp\":1,\"fid\":1,\"loginUid\":1,\"pageSource\":1,\"requestId\":1,\"rid\":1,\"uid\":1}}";
+        JSONObject param = JSONObject.parseObject(str);
+        param.getJSONObject("params").put("recordId", reviewId);
+        param.getJSONObject("params").getJSONObject("postOutputReviewDetail").put("id", planExeId);
+        try {
+            MediaType mediaType = MediaType.parse("application/json");
+            RequestBody body = RequestBody.create(mediaType, param.toJSONString());
+            Request request = new Request.Builder()
+                    .url("http://aigc-api.cybertogether.net/aigc/produce/review/saveDetail")
+                    .method("POST", body)
+                    .addHeader("Accept", "application/json")
+                    .addHeader("Accept-Language", "zh-CN,zh;q=0.9")
+                    .addHeader("Cache-Control", "no-cache")
+                    .addHeader("Connection", "keep-alive")
+                    .addHeader("Content-Type", "application/json")
+                    .addHeader("Origin", "http://aigc-admin-test.cybertogether.net")
+                    .addHeader("Pragma", "no-cache")
+                    .addHeader("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36")
+                    .build();
+            Response response = client.newCall(request).execute();
+            if (response.isSuccessful()) {
+                String responseContent = response.body().string();
+                JSONObject obj = JSONObject.parseObject(responseContent);
+                log.info(obj.toJSONString());
+            } else {
+                log.error("request failed : {}", JSONObject.toJSONString(response));
+            }
+        } catch (IOException e) {
+            log.error("catch error");
+        }
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/XxlJobService.java

@@ -19,8 +19,10 @@ import com.tzld.longarticle.recommend.server.mapper.crawler.CrawlerBaseMapper;
 import com.tzld.longarticle.recommend.server.mapper.growth.NewPushMessageCallbackMapper;
 import com.tzld.longarticle.recommend.server.mapper.longArticle.LongArticleBaseMapper;
 import com.tzld.longarticle.recommend.server.model.dto.*;
+import com.tzld.longarticle.recommend.server.model.dto.aigc.BadCrawlerAccountDTO;
 import com.tzld.longarticle.recommend.server.model.entity.aigc.CrawlerPlan;
 import com.tzld.longarticle.recommend.server.model.entity.aigc.ProducePlan;
+import com.tzld.longarticle.recommend.server.model.entity.aigc.ProduceReviewRecord;
 import com.tzld.longarticle.recommend.server.model.entity.aigc.PublishAccount;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.AccountAvgInfo;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.GetOffVideoCrawler;
@@ -30,6 +32,7 @@ import com.tzld.longarticle.recommend.server.model.param.ArticleFindSourceParam;
 import com.tzld.longarticle.recommend.server.model.param.TitleHisCacheParam;
 import com.tzld.longarticle.recommend.server.model.vo.FeishuTableDTO;
 import com.tzld.longarticle.recommend.server.remote.ODPSManager;
+import com.tzld.longarticle.recommend.server.remote.aigc.AIGCProduceContentAuditService;
 import com.tzld.longarticle.recommend.server.repository.aigc.ProducePlanRepository;
 import com.tzld.longarticle.recommend.server.repository.crawler.GetOffVideoCrawlerRepository;
 import com.tzld.longarticle.recommend.server.repository.crawler.LongArticlesVideoRepository;
@@ -105,6 +108,8 @@ public class XxlJobService {
     private RecallService recallService;
     @Autowired
     private ProducePlanRepository producePlanRepository;
+    @Autowired
+    private AIGCProduceContentAuditService aigcProduceContentAuditService;
 
     ExecutorService thread = new CommonThreadPoolExecutor(
             5,
@@ -116,6 +121,10 @@ public class XxlJobService {
 
     @ApolloJsonValue("${touliu.account.ghIds:[\"gh_93e00e187787\", \"gh_ac43e43b253b\", \"gh_68e7fdc09fe4\",\"gh_77f36c109fb1\", \"gh_b181786a6c8c\", \"gh_1ee2e1b39ccf\"]}")
     private List<String> touliuAccountGhIds;
+    @ApolloJsonValue("${cold.pool.produce.planId:[\"20240802021606053813696\", \"20240802080355355308981\",\n" +
+            "\"20240805154433785506170\", \"20240805154359027876170\", \"20241024100016206421084\", " +
+            "\"20241030070010871546586\"]}")
+    private static List<String> producePlanIds;
 
     @XxlJob("checkPublishPlan")
     public ReturnT<String> checkPublishPlan(String param) {
@@ -807,4 +816,26 @@ public class XxlJobService {
         }
     }
 
+    @XxlJob("badAccountContentReject")
+    public ReturnT<String> badAccountContentReject(String param) {
+        List<BadCrawlerAccountDTO> badAccountList = aigcBaseMapper.getBadAccountList(producePlanIds);
+        List<String> accountNames = badAccountList.stream().map(BadCrawlerAccountDTO::getAccountName).collect(Collectors.toList());
+        // 抓取内容设置废弃
+        aigcBaseMapper.setCrawlerContentDisuseByAccountName(accountNames);
+
+        List<String> planExeIds = aigcBaseMapper.getPlanExeIdByAccountName(accountNames);
+        List<ProduceReviewRecord> reviewList = aigcBaseMapper.getReviewList(planExeIds);
+        Map<String, String> reviewMap = reviewList.stream().collect(
+                Collectors.toMap(ProduceReviewRecord::getPlanExeId, ProduceReviewRecord::getRecordId));
+        for (String planExeId : planExeIds) {
+            String reviewId = reviewMap.get(planExeId);
+            if (!StringUtils.hasText(reviewId)) {
+                continue;
+            }
+            // 已生成内容审核不通过
+            aigcProduceContentAuditService.request(reviewId, planExeId);
+        }
+        return ReturnT.SUCCESS;
+    }
+
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/web/XxlJobController.java

@@ -94,4 +94,9 @@ public class XxlJobController {
         service.refreshArticleHisCache(null);
     }
 
+    @GetMapping("/badAccountContentReject")
+    public void badAccountContentReject() {
+        service.badAccountContentReject(null);
+    }
+
 }

long-article-recommend-service/src/main/resources/mapper/aigc/AigcBaseMapper.xml

@@ -200,4 +200,60 @@
         and audit_timestamp between #{start} and #{end}
     </select>
 
+    <select id="getPlanExeIdByAccountName" resultType="java.lang.String">
+        select plan_exe_id
+        from produce_plan_exe_record
+        where channel_content_id in (
+        select channel_content_id
+        from crawler_content
+        where channel_account_id in (
+        select channel_account_id
+        from crawler_account
+        where account_name in
+        <foreach collection="accountNames" item="item" open="(" close=")" separator=",">
+            #{item}
+        </foreach>
+        )) and audit_status = 0 and status = 6
+    </select>
+
+    <select id="getReviewList" resultType="com.tzld.longarticle.recommend.server.model.entity.aigc.ProduceReviewRecord">
+        select * from produce_review_record where plan_exe_id in
+        <foreach collection="planExeIds" item="item" open="(" close=")" separator=",">
+            #{item}
+        </foreach>
+        and review_status = 0
+    </select>
+
+    <select id="getBadAccountList" resultType="com.tzld.longarticle.recommend.server.model.dto.aigc.BadCrawlerAccountDTO">
+        select account_name, count(if(audit_status = 1, 1, null)) / count(1) as rate, count(1) as total_count
+        from (
+             select  audit_status,
+                     e.account_name
+             from produce_plan_exe_record a
+                      join produce_plan b ON a.plan_id = b.id
+                      join crawler_content d on a.channel_content_id = d.channel_content_id
+                      join crawler_account e on d.channel_account_id = e.channel_account_id
+             where a.plan_id in
+                 <foreach collection="producePlanIds" item="item" open="(" close=")" separator=",">
+                     #{item}
+                 </foreach>
+               and a.audit_status in (1, 2)
+               and a.update_timestamp > unix_timestamp('2024-09-01')
+             ) t1
+        group by account_name
+        having rate = 0 and total_count >= 10
+        order by rate
+    </select>
+
+    <update id="setCrawlerContentDisuseByAccountName">
+        update crawler_content cc
+        join crawler_account ca on cc.channel_account_id = ca.channel_account_id
+        set cc.status = 3
+        where ca.account_name in
+        <foreach collection="accountNames" item="item" open="(" close=")" separator=",">
+            #{item}
+        </foreach>
+        and cc.status = 1
+    </update>
+
 </mapper>