articlePromotion

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/longArticle/LongArticleBaseMapper.java

@@ -26,4 +26,7 @@ public interface LongArticleBaseMapper {
     void batchInsertDatastatScore(List<DatastatScore> list);
 
     void batchInsertArticleCategory(List<ArticleCategory> list);
+
+    List<DatastatSortStrategy> getArticlePromotion(Integer viewCount, Double viewCountRate,
+                                                   Integer fans, String dateStr, List<Integer> positions);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/vo/WxContentDetailResponse.java

@@ -0,0 +1,52 @@
+package com.tzld.longarticle.recommend.server.model.vo;
+
+import com.alibaba.fastjson.annotation.JSONField;
+import lombok.Getter;
+import lombok.Setter;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+@Getter
+@Setter
+public class WxContentDetailResponse {
+    private String channelContentId;
+    private String title;
+    private String bodyText;
+    private Long publishTimestamp;
+    private List<MiniProgram> miniProgram;
+    private AdInfo adInfo;
+    private Integer viewCount;
+    private Integer likeCount;
+    private Integer shareCount;
+
+    public AdInfo getAdInfo() {
+        if (Objects.isNull(adInfo)) {
+            return new AdInfo();
+        }
+        return adInfo;
+    }
+
+    public List<MiniProgram> getMiniProgram() {
+        if (Objects.isNull(miniProgram)) {
+            return new ArrayList<>();
+        }
+        return miniProgram;
+    }
+
+    @Getter
+    @Setter
+    public static class AdInfo {
+        @JSONField(name = "文中")
+        private Integer middle;
+        @JSONField(name = "文末")
+        private Integer tail;
+    }
+
+    @Getter
+    @Setter
+    public static class MiniProgram {
+        private String path;
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/remote/WxFetchRemoteService.java

@@ -2,6 +2,7 @@ package com.tzld.longarticle.recommend.server.remote;
 
 import com.alibaba.fastjson.JSONObject;
 import com.tzld.longarticle.recommend.server.common.HttpPoolFactory;
+import com.tzld.longarticle.recommend.server.model.vo.WxContentDetailResponse;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.http.HttpEntity;
 import org.apache.http.StatusLine;
@@ -13,7 +14,7 @@ import org.apache.http.util.EntityUtils;
 import org.springframework.stereotype.Service;
 
 import java.nio.charset.StandardCharsets;
-import java.util.*;
+import java.util.Objects;
 
 /**
  * @author dyp
@@ -25,8 +26,8 @@ public class WxFetchRemoteService {
     private static final String url = "http://8.217.190.241:8888/crawler/wei_xin/detail";
 
 
-    public Map<String, String> getContent(String contentUrl) {
-        Map<String, String> result = new HashMap<>();
+    public WxContentDetailResponse getContent(String contentUrl) {
+        WxContentDetailResponse result = new WxContentDetailResponse();
         JSONObject bodyParam = new JSONObject();
         bodyParam.put("content_link", contentUrl);
         bodyParam.put("is_count", true);
@@ -45,9 +46,7 @@ public class WxFetchRemoteService {
                     JSONObject jsonObject = JSONObject.parseObject(responseBody);
                     if (jsonObject.getInteger("code") == 0) {
                         JSONObject data = jsonObject.getJSONObject("data");
-                        data = data.getJSONObject("data");
-                        result.put("view_count", data.getString("view_count"));
-                        result.put("title", data.getString("title"));
+                        result = data.getObject("data", WxContentDetailResponse.class);
                     }
                 }
             }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/remote/aigc/AIGCProduceContentListService.java

@@ -4,7 +4,6 @@ import cn.hutool.core.io.resource.ResourceUtil;
 import com.alibaba.fastjson.JSONObject;
 import com.tzld.longarticle.recommend.server.common.HttpPoolFactory;
 import com.tzld.longarticle.recommend.server.model.vo.aigc.ProduceContentListItemVO;
-import com.tzld.longarticle.recommend.server.model.vo.aigc.ProducePlanDetailVO;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.http.HttpEntity;
 import org.apache.http.StatusLine;
@@ -27,10 +26,10 @@ public class AIGCProduceContentListService {
 
     public Map<String, Object> list(List<String> planIdList, int pageNum, int pageSize, List<Integer> produceStatus) {
         Map<String, Object> result = new HashMap<>();
+        result.put("contentList", new ArrayList<>());
+        result.put("totalCnt", 0);
         if (planIdList.isEmpty()) {
             System.out.println("getProduceContentListByPlanIdListRaw: planIdList empty");
-            result.put("contentList", new ArrayList<>());
-            result.put("totalCnt", 0);
             return result;
         }
 
@@ -47,13 +46,30 @@ public class AIGCProduceContentListService {
         data.getJSONObject("params").put("pageNum", pageNum);
         data.getJSONObject("params").put("pageSize", pageSize);
 
-        String response = HttpRequest.post(apiUrl, data.toString(), Headers.HEADERS);
-        JSONObject jsonResponse = JSONObject.parseObject(response);
-
-        int totalCnt = jsonResponse.getJSONObject("data").getInteger("totalCount");
-        List<ProduceContentListItemVO> contentList = jsonResponse.getJSONObject("data").parseArray("data", ProduceContentListItemVO.class);
-        result.put("contentList", contentList);
-        result.put("totalCnt", totalCnt);
+        try {
+            HttpPost httpPost = new HttpPost(apiUrl);
+            StringEntity stringEntity = new StringEntity(data.toJSONString(), StandardCharsets.UTF_8);
+            httpPost.setHeader("Content-Type", "application/json;charset=UTF-8");
+            httpPost.setEntity(stringEntity);
+            CloseableHttpResponse response = client.execute(httpPost);
+            StatusLine statusLine = response.getStatusLine();
+            if (statusLine.getStatusCode() == 200) {
+                HttpEntity responseEntity = response.getEntity();
+                if (Objects.nonNull(responseEntity)) {
+                    String responseBody = EntityUtils.toString(responseEntity, "UTF-8");
+                    JSONObject jsonObject = JSONObject.parseObject(responseBody);
+                    if (jsonObject.getInteger("code") == 0) {
+                        int totalCnt = jsonObject.getJSONObject("data").getInteger("totalCount");
+                        List<ProduceContentListItemVO> contentList = jsonObject.getJSONObject("data").parseArray("data", ProduceContentListItemVO.class);
+                        result.put("contentList", contentList);
+                        result.put("totalCnt", totalCnt);
+                        return result;
+                    }
+                }
+            }
+        } catch (IOException e) {
+            log.error("articleGetProducePlanDetail error", e);
+        }
         return result;
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/JobService.java

@@ -1,7 +1,8 @@
 package com.tzld.longarticle.recommend.server.service;
 
-import com.tzld.longarticle.recommend.server.remote.WxFetchRemoteService;
 import com.tzld.longarticle.recommend.server.mapper.crawler.ArticleContentLinkMapper;
+import com.tzld.longarticle.recommend.server.model.vo.WxContentDetailResponse;
+import com.tzld.longarticle.recommend.server.remote.WxFetchRemoteService;
 import com.tzld.longarticle.recommend.server.repository.model.ArticleContentLink;
 import com.tzld.longarticle.recommend.server.repository.model.ArticleContentLinkExample;
 import lombok.extern.slf4j.Slf4j;
@@ -13,7 +14,6 @@ import java.time.LocalDate;
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
 import java.util.List;
-import java.util.Map;
 
 /**
  * @author dyp
@@ -46,9 +46,9 @@ public class JobService {
         example.createCriteria().andCreateTimeGreaterThanOrEqualTo(time);
         List<ArticleContentLink> links = articleContentLinkMapper.selectByExample(example);
         for (ArticleContentLink link : links) {
-            Map<String, String> content = wxFetchRemoteService.getContent(link.getContentLink());
-            link.setTitle(content.getOrDefault("title", ""));
-            link.setViewCount(NumberUtils.toLong(content.getOrDefault("view_count", "0"), 0L));
+            WxContentDetailResponse content = wxFetchRemoteService.getContent(link.getContentLink());
+            link.setTitle(content.getTitle());
+            link.setViewCount(NumberUtils.toLong(String.valueOf(content.getViewCount()), 0L));
             link.setUpdateTime(System.currentTimeMillis());
 
             articleContentLinkMapper.updateByPrimaryKey(link);

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/ArticlePromotionService.java

@@ -1,17 +1,25 @@
 package com.tzld.longarticle.recommend.server.service.recommend;
 
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.longarticle.recommend.server.mapper.longArticle.LongArticleBaseMapper;
+import com.tzld.longarticle.recommend.server.model.entity.aigc.PublishAccount;
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.ArticlePoolPromotionSource;
 import com.tzld.longarticle.recommend.server.model.entity.longArticle.DatastatSortStrategy;
 import com.tzld.longarticle.recommend.server.model.vo.IdNameVO;
+import com.tzld.longarticle.recommend.server.model.vo.WxContentDetailResponse;
 import com.tzld.longarticle.recommend.server.model.vo.aigc.ProduceContentListItemVO;
 import com.tzld.longarticle.recommend.server.model.vo.aigc.ProducePlanDetailVO;
 import com.tzld.longarticle.recommend.server.model.vo.aigc.ProducePlanInputSourceParam;
+import com.tzld.longarticle.recommend.server.remote.WxFetchRemoteService;
 import com.tzld.longarticle.recommend.server.remote.aigc.AIGCCrawlerPlanSaveService;
 import com.tzld.longarticle.recommend.server.remote.aigc.AIGCProduceContentListService;
 import com.tzld.longarticle.recommend.server.remote.aigc.AIGCProducePlanDetailService;
 import com.tzld.longarticle.recommend.server.remote.aigc.AIGCProducePlanSaveService;
+import com.tzld.longarticle.recommend.server.repository.aigc.PublishAccountRepository;
+import com.tzld.longarticle.recommend.server.repository.longArticle.ArticlePoolPromotionSourceRepository;
 import com.tzld.longarticle.recommend.server.repository.longArticle.DatastatSortStrategyRepository;
 import com.tzld.longarticle.recommend.server.util.DateUtils;
+import com.tzld.longarticle.recommend.server.util.Md5Util;
 import com.tzld.longarticle.recommend.server.util.TitleSimilarCheckUtil;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -26,6 +34,8 @@ import java.util.stream.Collectors;
 @Slf4j
 public class ArticlePromotionService {
 
+    @Autowired
+    LongArticleBaseMapper longArticleBaseMapper;
     @Autowired
     DatastatSortStrategyRepository datastatSortStrategyRepository;
     @Autowired
@@ -36,6 +46,12 @@ public class ArticlePromotionService {
     AIGCProducePlanSaveService aigcProducePlanSaveService;
     @Autowired
     AIGCProduceContentListService aigcProduceContentListService;
+    @Autowired
+    ArticlePoolPromotionSourceRepository articlePoolPromotionSourceRepository;
+    @Autowired
+    WxFetchRemoteService wxFetchRemoteService;
+    @Autowired
+    PublishAccountRepository publishAccountRepository;
 
     @ApolloJsonValue("${articlePromotionProduceConfig:{}}")
     private Map<String, Map<String, Map<String, String>>> produceConfig;
@@ -45,9 +61,11 @@ public class ArticlePromotionService {
     public void articlePromotion(String pos, String way, String accountNickName, String tag,
                                  Integer viewCountFilter, Double viewCountRateFilter, List<Integer> positionFilter) {
         String today = DateUtils.getCurrentDateStr("yyyyMMdd");
+        String dateStrFilter = DateUtils.getBeforeDaysDateStr("yyyyMMdd", 10);
         // 获取内部表现
-        List<DatastatSortStrategy> list = datastatSortStrategyRepository.getByViewCountGreaterThanEqualAndReadRateGreaterThanEqualAndPositionIn(
-                viewCountFilter, viewCountRateFilter, positionFilter);
+        List<DatastatSortStrategy> list = longArticleBaseMapper.getArticlePromotion(viewCountFilter, viewCountRateFilter,
+                10000, dateStrFilter, positionFilter);
+        list = filterEarlyContent(list);
         log.info("优质{}文章数量: {}", accountNickName, list.size());
         List<DatastatSortStrategy> distinct = filterSameTitle(list);
         distinct.sort(Comparator.comparing(DatastatSortStrategy::getDateStr, Comparator.reverseOrder()));
@@ -55,6 +73,19 @@ public class ArticlePromotionService {
         addUrlListToAccount(accountNickName, distinct, pos, way, today, tag);
     }
 
+    private List<DatastatSortStrategy> filterEarlyContent(List<DatastatSortStrategy> list) {
+        List<String> ghIds = list.stream().map(DatastatSortStrategy::getGhId).distinct().collect(Collectors.toList());
+        List<PublishAccount> publishAccountList = publishAccountRepository.getAllByGhIdIn(ghIds);
+        Map<String, Long> publishAccountCreateTimeMap = publishAccountList.stream().collect(Collectors.toMap(
+                PublishAccount::getGhId, PublishAccount::getCreateTimestamp));
+        list = list.stream().filter(o -> {
+            long publishTime = DateUtils.dateStrToTimestamp(o.getDateStr(), "yyyyMMdd");
+            Long accountCreateTime = publishAccountCreateTimeMap.get(o.getGhId());
+            return publishTime * 1000 > accountCreateTime;
+        }).collect(Collectors.toList());
+        return list;
+    }
+
     private List<DatastatSortStrategy> filterSameTitle(List<DatastatSortStrategy> list) {
         List<DatastatSortStrategy> result = new ArrayList<>();
         List<String> titles = new ArrayList<>();
@@ -103,11 +134,11 @@ public class ArticlePromotionService {
                 continue;
             }
             filterUrlList.add(url);
-            // todo 调用爬虫 detail 接口并保存数据
-            Map<String, String> detail = getArticleDetail(url);
+            // 调用爬虫 detail 接口并保存数据
+            WxContentDetailResponse detail = getArticleDetail(url);
             if (detail != null) {
                 String level = pos.equals("【1】") ? contentPoolType.get(0) : contentPoolType.get(1);
-                saveArticlePoolPromotionSource(detail.get("channel_content_id"), wxSn, title, level);
+                saveArticlePoolPromotionSource(detail.getChannelContentId(), wxSn, title, level);
             }
         }
         if (filterUrlList.isEmpty()) {
@@ -228,14 +259,27 @@ public class ArticlePromotionService {
         return params;
     }
 
-    private Map<String, String> getArticleDetail(String url) {
-        // 返回文章详情的逻辑
-        return new HashMap<>();
+    public WxContentDetailResponse getArticleDetail(String url) {
+        if (url == null) {
+            return null;
+        }
+        try {
+            return wxFetchRemoteService.getContent(url);
+        } catch (Exception e) {
+            log.error("URL error: " + url);
+        }
+        return null;
     }
 
     private void saveArticlePoolPromotionSource(String channelContentId, String wxSn, String title, String level) {
-        // 保存文章数据的逻辑
-        log.info("Saved: " + channelContentId + ", " + wxSn + ", " + title + ", " + level);
+        ArticlePoolPromotionSource articlePromotion = new ArticlePoolPromotionSource();
+        articlePromotion.setChannelContentId(channelContentId);
+        articlePromotion.setWxSn(wxSn);
+        articlePromotion.setTitle(title);
+        articlePromotion.setTitleMd5(Md5Util.encoderByMd5(title));
+        articlePromotion.setLevel(level);
+        articlePromotion.setCreateTimestamp(System.currentTimeMillis());
+        articlePoolPromotionSourceRepository.save(articlePromotion);
     }
 
     private void articleAddDependPlan(String produceId, String planId, String planName) {

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/ArticleService.java

@@ -335,6 +335,7 @@ public class ArticleService {
                         item.setChannelContentId(midChannelContentId);
                         item.setCreateTimestamp(item.getUpdateTimestamp());
                     }
+                    item.setDeleted(0);
                     articlePoolPromotionSourceRepository.save(item);
                 }
                 longArticleBaseMapper.updateRootProduceContentLevel(task.getRootProduceContentId(), task.getLevel());

long-article-recommend-service/src/main/resources/mapper/longArticle/LongArticleBaseMapper.xml

@@ -88,4 +88,18 @@
         </foreach>
     </insert>
 
+    <select id="getArticlePromotion"
+            resultType="com.tzld.longarticle.recommend.server.model.entity.longArticle.DatastatSortStrategy">
+        select *
+        from datastat_sort_strategy
+        where view_count >= #{viewCount}
+        and avg_view_count >= #{viewCountRate}
+        and fans > #{fans}
+        and date_str > #{dateStr}
+        and position in
+        <foreach collection="positions" item="item" open="(" close=")" separator=",">
+            #{item}
+        </foreach>
+    </select>
+
 </mapper>