Merge branch 'wyp/1018-accountAvgV3Job' of Server/long-article-recommend into master

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/enums/AccountBusinessTypeEnum.java

@@ -6,11 +6,11 @@ import java.util.Objects;
 
 @Getter
 public enum AccountBusinessTypeEnum {
-    changwen(1, "长文"),
-    touliu(2, "投流"),
-    qiwei(3, "企微"),
+    CHANGWEN(1, "长文"),
+    TOULIU(2, "投流"),
+    QIWEI(3, "企微"),
     
-    other(999, "其他"),
+    OTHER(999, "其他"),
 
     ;
 
@@ -28,6 +28,6 @@ public enum AccountBusinessTypeEnum {
                 return typeEnum;
             }
         }
-        return other;
+        return OTHER;
     }
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/aigc/AigcBaseMapper.java

@@ -1,5 +1,6 @@
 package com.tzld.longarticle.recommend.server.mapper.aigc;
 
+import com.tzld.longarticle.recommend.server.model.dto.AccountTypeFansDTO;
 import com.tzld.longarticle.recommend.server.model.dto.NotPublishPlan;
 import com.tzld.longarticle.recommend.server.model.dto.ProduceContentDTO;
 import com.tzld.longarticle.recommend.server.model.entity.aigc.*;
@@ -27,4 +28,6 @@ public interface AigcBaseMapper {
     List<PublishAccount> getPublishAccounts(String planId, Long todayStart);
 
     List<ProduceContentDTO> getSourceProduceContentByTitles(List<String> titleList);
+
+    List<AccountTypeFansDTO> getAccountTypeFans();
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/crawler/CrawlerBaseMapper.java

@@ -1,5 +1,6 @@
 package com.tzld.longarticle.recommend.server.mapper.crawler;
 
+import com.tzld.longarticle.recommend.server.model.entity.crawler.AccountAvgInfo;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.AccountCorrelation;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.LongArticlesRootSourceId;
 
@@ -11,4 +12,10 @@ public interface CrawlerBaseMapper {
 
     void batchInsertLongArticlesRootSourceId(List<LongArticlesRootSourceId> list);
 
+    void batchInsertAccountAvgInfo(List<AccountAvgInfo> list);
+
+    void deleteAccountAvgInfoByGhIdAndUpdateTime(String ghId, String date);
+
+    void updateAccountAvgInfoStatus(String ghId, String date);
+
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/dto/AccountTypeFansDTO.java

@@ -0,0 +1,14 @@
+package com.tzld.longarticle.recommend.server.model.dto;
+
+import lombok.Data;
+
+@Data
+public class AccountTypeFansDTO {
+    private String name;
+    private String ghId;
+    private Integer followerCount;
+    private String accountSourceName;
+    private String modeType;
+    private String accountType;
+    private String status;
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/longArticle/LongArticlesReadRate.java

@@ -0,0 +1,61 @@
+package com.tzld.longarticle.recommend.server.model.entity.longArticle;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import javax.persistence.*;
+import java.io.Serializable;
+import java.util.Date;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Entity
+@Table(name = "long_articles_read_rate")
+@IdClass(LongArticlesReadRate.PK.class)
+public class LongArticlesReadRate {
+    @Id
+    @Column(name = "gh_id")
+    private String ghId;
+
+    @Id
+    @Column(name = "position")
+    private int position;
+
+    @Column(name = "account_name")
+    private String accountName;
+
+    @Column(name = "fans")
+    private Integer fans;
+
+    @Column(name = "read_rate_avg")
+    private Double readRateAvg;
+
+    @Column(name = "remark")
+    private String remark;
+
+    @Column(name = "articles_count")
+    private Integer articlesCount;
+
+    @Column(name = "earliest_publish_time")
+    private Date earliestPublishTime;
+
+    @Column(name = "latest_publish_time")
+    private Date latestPublishTime;
+
+
+    @Data
+    public static class PK implements Serializable {
+
+        @Column(name = "gh_id", length = 32)
+        private String ghId;
+        @Column(name = "position")
+        private int position;
+
+        public PK() {
+        }
+
+
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/crawler/AccountAvgInfoRepository.java

@@ -23,4 +23,6 @@ public interface AccountAvgInfoRepository extends JpaRepository<AccountAvgInfo,
     List<AccountAvgInfo> getAllByUpdateTimeGreaterThanEqual(String updateTime);
 
     List<AccountAvgInfo> getAllByUpdateTimeLessThanEqual(String updateTime);
+
+    void deleteByGhIdAndUpdateTime(String ghId, String updateTime);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/longArticle/LongArticlesReadRateRepository.java

@@ -0,0 +1,9 @@
+package com.tzld.longarticle.recommend.server.repository.longArticle;
+
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.LongArticlesReadRate;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.stereotype.Repository;
+
+@Repository
+public interface LongArticlesReadRateRepository extends JpaRepository<LongArticlesReadRate, LongArticlesReadRate.PK> {
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/XxlJobService.java

@@ -3,21 +3,26 @@ package com.tzld.longarticle.recommend.server.service;
 import cn.hutool.core.collection.CollectionUtil;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.longarticle.recommend.server.common.enums.AccountBusinessTypeEnum;
 import com.tzld.longarticle.recommend.server.common.enums.FeishuRobotIdEnum;
 import com.tzld.longarticle.recommend.server.mapper.aigc.AigcBaseMapper;
 import com.tzld.longarticle.recommend.server.mapper.crawler.CrawlerBaseMapper;
+import com.tzld.longarticle.recommend.server.model.dto.AccountTypeFansDTO;
 import com.tzld.longarticle.recommend.server.model.dto.NotPublishPlan;
 import com.tzld.longarticle.recommend.server.model.entity.aigc.PublishAccount;
+import com.tzld.longarticle.recommend.server.model.entity.crawler.AccountAvgInfo;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.GetOffVideoCrawler;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.LongArticlesRootSourceId;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.LongArticlesVideo;
 import com.tzld.longarticle.recommend.server.model.entity.longArticle.GetOffVideoArticle;
 import com.tzld.longarticle.recommend.server.model.entity.longArticle.LongArticlesMatchVideo;
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.LongArticlesReadRate;
 import com.tzld.longarticle.recommend.server.repository.crawler.GetOffVideoCrawlerRepository;
-import com.tzld.longarticle.recommend.server.repository.crawler.LongArticlesRootSourceIdRepository;
 import com.tzld.longarticle.recommend.server.repository.crawler.LongArticlesVideoRepository;
 import com.tzld.longarticle.recommend.server.repository.longArticle.GetOffVideoArticleRepository;
 import com.tzld.longarticle.recommend.server.repository.longArticle.LongArticlesMatchVideoRepository;
+import com.tzld.longarticle.recommend.server.repository.longArticle.LongArticlesReadRateRepository;
 import com.tzld.longarticle.recommend.server.util.DateUtils;
 import com.tzld.longarticle.recommend.server.util.feishu.FeishuMessageSender;
 import com.xxl.job.core.biz.model.ReturnT;
@@ -30,6 +35,7 @@ import org.springframework.util.StringUtils;
 import java.time.LocalTime;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 @Service
@@ -49,7 +55,10 @@ public class XxlJobService {
     @Autowired
     private LongArticlesMatchVideoRepository longArticlesMatchVideoRepository;
     @Autowired
-    private LongArticlesRootSourceIdRepository longArticlesRootSourceIdRepository;
+    private LongArticlesReadRateRepository longArticlesReadRateRepository;
+
+    @ApolloJsonValue("${touliu.account.ghIds:[\"gh_93e00e187787\", \"gh_ac43e43b253b\", \"gh_68e7fdc09fe4\",\"gh_77f36c109fb1\", \"gh_b181786a6c8c\", \"gh_1ee2e1b39ccf\"]}")
+    private List<String> touliuAccountGhIds;
 
     @XxlJob("checkPublishPlan")
     public ReturnT<String> checkPublishPlan(String param) {
@@ -149,8 +158,8 @@ public class XxlJobService {
             log.error("Error processCrawlerEachData: {}", JSONObject.toJSONString(longArticlesVideo), e);
             FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.JOB.getRobotId(),
                     "processCrawlerEachData 异常\n"
-                    + "异常数据: " + longArticlesVideo.getTraceId() + "\n"
-                    + "错误信息：" + e.getMessage());
+                            + "异常数据: " + longArticlesVideo.getTraceId() + "\n"
+                            + "错误信息：" + e.getMessage());
         }
     }
 
@@ -205,4 +214,60 @@ public class XxlJobService {
         }
     }
 
+    @XxlJob("accountAvgInfoV3Job")
+    public ReturnT<String> accountAvgInfoV3Job(String param) {
+        long timeStamp = DateUtils.getBeforeDayStart(1);
+        if (StringUtils.hasText(param)) {
+            timeStamp = DateUtils.getStartOfDay(param, "yyyyMMdd");
+        }
+        String dateStr = DateUtils.timestampToYMDStr(timeStamp, "yyyy-MM-dd");
+        List<AccountTypeFansDTO> accountList = aigcBaseMapper.getAccountTypeFans();
+        List<LongArticlesReadRate> readRateList = longArticlesReadRateRepository.findAll();
+        Map<String, Double> rateDict = readRateList.stream().collect(Collectors.toMap(o -> o.getGhId() + "_" + o.getPosition(),
+                LongArticlesReadRate::getReadRateAvg));
+        for (AccountTypeFansDTO accountTypeFansDTO : accountList) {
+            saveAccountAvgInfo(accountTypeFansDTO, rateDict, dateStr);
+        }
+        return ReturnT.SUCCESS;
+    }
+
+    public void saveAccountAvgInfo(AccountTypeFansDTO account, Map<String, Double> rateDict, String date) {
+        int businessType = touliuAccountGhIds.contains(account.getGhId()) ?
+                AccountBusinessTypeEnum.TOULIU.getType() : AccountBusinessTypeEnum.CHANGWEN.getType();
+        if (account.getFollowerCount() > 0) {
+            List<AccountAvgInfo> saveList = new ArrayList<>();
+            for (int index = 1; index <= 8; index++) {
+                String ghIdPosition = account.getGhId() + "_" + index;
+                if (rateDict.containsKey(ghIdPosition)) {
+                    double rate = rateDict.get(ghIdPosition);
+                    double readAvg = account.getFollowerCount() * rate;
+                    AccountAvgInfo accountAvgInfo = new AccountAvgInfo();
+                    accountAvgInfo.setGhId(account.getGhId());
+                    accountAvgInfo.setPosition(String.valueOf(index));
+                    accountAvgInfo.setUpdateTime(date);
+                    accountAvgInfo.setAccountName(account.getName());
+                    accountAvgInfo.setFans(account.getFollowerCount());
+                    accountAvgInfo.setReadAvg(readAvg);
+                    accountAvgInfo.setStatus(1);
+                    accountAvgInfo.setAccountType(account.getAccountType());
+                    accountAvgInfo.setAccountMode(account.getModeType());
+                    accountAvgInfo.setAccountSource(account.getAccountSourceName());
+                    accountAvgInfo.setAccountStatus(account.getStatus());
+                    accountAvgInfo.setBusinessType(businessType);
+                    accountAvgInfo.setReadRateAvg(rate);
+                    saveList.add(accountAvgInfo);
+                }
+            }
+            if (CollectionUtil.isNotEmpty(saveList)) {
+                try {
+                    crawlerBaseMapper.batchInsertAccountAvgInfo(saveList);
+                } catch (Exception e) {
+                    crawlerBaseMapper.deleteAccountAvgInfoByGhIdAndUpdateTime(account.getGhId(), date);
+                    crawlerBaseMapper.batchInsertAccountAvgInfo(saveList);
+                }
+                crawlerBaseMapper.updateAccountAvgInfoStatus(account.getGhId(), date);
+            }
+        }
+    }
+
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/web/XxlJobController.java

@@ -28,4 +28,9 @@ public class XxlJobController {
     public void migrateArticleRootSourceId(String dateStr) {
         service.migrateArticleRootSourceId(dateStr);
     }
+
+    @GetMapping("/accountAvgInfoV3Job")
+    public void accountAvgInfoV3Job(String dateStr) {
+        service.accountAvgInfoV3Job(dateStr);
+    }
 }

long-article-recommend-service/src/main/resources/mapper/aigc/AigcBaseMapper.xml

@@ -112,5 +112,24 @@
         </foreach>
     </select>
 
+    <select id="getAccountTypeFans"
+            resultType="com.tzld.longarticle.recommend.server.model.dto.AccountTypeFansDTO">
+        SELECT distinct t3.`name`,
+                        t3.gh_id,
+                        t3.follower_count,
+                        t6.account_source_name,
+                        t6.mode_type,
+                        t6.account_type,
+                        t6.`status`
+        FROM publish_plan t1
+        JOIN publish_plan_account t2 ON t1.id = t2.plan_id
+        JOIN publish_account t3 ON t2.account_id = t3.id
+        LEFT JOIN publish_account_wx_type t4 on t3.id = t4.account_id
+        LEFT JOIN wx_statistics_group_source_account t5 on t3.id = t5.account_id
+        LEFT JOIN wx_statistics_group_source t6 on t5.group_source_name = t6.account_source_name
+        WHERE t1.plan_status = 1
+          AND t3.channel = 5
+    </select>
+
 
 </mapper>

long-article-recommend-service/src/main/resources/mapper/crawler/CrawlerBaseMapper.xml

@@ -20,4 +20,24 @@
         </foreach>
     </insert>
 
+    <insert id="batchInsertAccountAvgInfo">
+        INSERT INTO account_avg_info_v3
+        (gh_id, position, update_time, account_name, fans, read_avg, like_avg, status, account_type, account_mode,
+        account_source, account_status, business_type, read_rate_avg)
+        VALUES
+        <foreach collection="list" item="item" separator=",">
+            (#{item.ghId}, #{item.position}, #{item.updateTime}, #{item.accountName}, #{item.fans}, #{item.readAvg},
+            #{item.likeAvg}, #{item.status}, #{item.accountType}, #{item.accountMode}, #{item.accountSource},
+            #{item.accountStatus}, #{item.businessType}, #{item.readRateAvg})
+        </foreach>
+    </insert>
+
+    <delete id="deleteAccountAvgInfoByGhIdAndUpdateTime">
+        delete from account_avg_info_v3 where gh_id = #{ghId} and update_time = #{date}
+    </delete>
+
+    <update id="updateAccountAvgInfoStatus">
+        update account_avg_info_v3 set status = 0 where gh_id = #{ghId} and update_time != #{date}
+    </update>
+
 </mapper>