|
@@ -76,57 +76,57 @@ class ArticleSpider(object):
|
|
|
appMsgId = info.get("AppMsg", {}).get("BaseInfo", {}).get("AppMsgId", None)
|
|
|
createTime = info.get("AppMsg", {}).get("BaseInfo", {}).get("CreateTime", None)
|
|
|
updateTime = info.get("AppMsg", {}).get("BaseInfo", {}).get("UpdateTime", None)
|
|
|
- if int(time.time()) - int(updateTime) <= 20 * 60 * 60:
|
|
|
- continue
|
|
|
+ # if int(time.time()) - int(updateTime) <= 20 * 60 * 60:
|
|
|
+ # continue
|
|
|
Type = info.get("AppMsg", {}).get("BaseInfo", {}).get("Type", None)
|
|
|
detail_article_list = info.get("AppMsg", {}).get("DetailInfo", [])
|
|
|
if detail_article_list:
|
|
|
for article in detail_article_list:
|
|
|
+ title = article.get("Title", None)
|
|
|
+ Digest = article.get("Digest", None)
|
|
|
+ ItemIndex = article.get("ItemIndex", None)
|
|
|
+ ContentUrl = article.get("ContentUrl", None)
|
|
|
+ SourceUrl = article.get("SourceUrl", None)
|
|
|
+ CoverImgUrl = article.get("CoverImgUrl", None)
|
|
|
+ CoverImgUrl_1_1 = article.get("CoverImgUrl_1_1", None)
|
|
|
+ CoverImgUrl_235_1 = article.get("CoverImgUrl_235_1", None)
|
|
|
+ ItemShowType = article.get("ItemShowType", None)
|
|
|
+ IsOriginal = article.get("IsOriginal", None)
|
|
|
+ ShowDesc = article.get("ShowDesc", None)
|
|
|
+ show_stat = show_desc_to_sta(ShowDesc)
|
|
|
+ ori_content = article.get("ori_content", None)
|
|
|
+ show_view_count = show_stat.get("show_view_count", 0)
|
|
|
+ show_like_count = show_stat.get("show_like_count", 0)
|
|
|
+ show_zs_count = show_stat.get("show_zs_count", 0)
|
|
|
+ show_pay_count = show_stat.get("show_pay_count", 0)
|
|
|
+ wx_sn = ContentUrl.split("&sn=")[1].split("&")[0] if ContentUrl else None
|
|
|
+ info_tuple = (
|
|
|
+ gh_id,
|
|
|
+ account_name,
|
|
|
+ appMsgId,
|
|
|
+ title,
|
|
|
+ Type,
|
|
|
+ createTime,
|
|
|
+ updateTime,
|
|
|
+ Digest,
|
|
|
+ ItemIndex,
|
|
|
+ ContentUrl,
|
|
|
+ SourceUrl,
|
|
|
+ CoverImgUrl,
|
|
|
+ CoverImgUrl_1_1,
|
|
|
+ CoverImgUrl_235_1,
|
|
|
+ ItemShowType,
|
|
|
+ IsOriginal,
|
|
|
+ ShowDesc,
|
|
|
+ ori_content,
|
|
|
+ show_view_count,
|
|
|
+ show_like_count,
|
|
|
+ show_zs_count,
|
|
|
+ show_pay_count,
|
|
|
+ wx_sn,
|
|
|
+ json.dumps(baseInfo, ensure_ascii=False)
|
|
|
+ )
|
|
|
try:
|
|
|
- title = article.get("Title", None)
|
|
|
- Digest = article.get("Digest", None)
|
|
|
- ItemIndex = article.get("ItemIndex", None)
|
|
|
- ContentUrl = article.get("ContentUrl", None)
|
|
|
- SourceUrl = article.get("SourceUrl", None)
|
|
|
- CoverImgUrl = article.get("CoverImgUrl", None)
|
|
|
- CoverImgUrl_1_1 = article.get("CoverImgUrl_1_1", None)
|
|
|
- CoverImgUrl_235_1 = article.get("CoverImgUrl_235_1", None)
|
|
|
- ItemShowType = article.get("ItemShowType", None)
|
|
|
- IsOriginal = article.get("IsOriginal", None)
|
|
|
- ShowDesc = article.get("ShowDesc", None)
|
|
|
- show_stat = show_desc_to_sta(ShowDesc)
|
|
|
- ori_content = article.get("ori_content", None)
|
|
|
- show_view_count = show_stat.get("show_view_count", 0)
|
|
|
- show_like_count = show_stat.get("show_like_count", 0)
|
|
|
- show_zs_count = show_stat.get("show_zs_count", 0)
|
|
|
- show_pay_count = show_stat.get("show_pay_count", 0)
|
|
|
- wx_sn = ContentUrl.split("&sn=")[1].split("&")[0] if ContentUrl else None
|
|
|
- info_tuple = (
|
|
|
- gh_id,
|
|
|
- account_name,
|
|
|
- appMsgId,
|
|
|
- title,
|
|
|
- Type,
|
|
|
- createTime,
|
|
|
- updateTime,
|
|
|
- Digest,
|
|
|
- ItemIndex,
|
|
|
- ContentUrl,
|
|
|
- SourceUrl,
|
|
|
- CoverImgUrl,
|
|
|
- CoverImgUrl_1_1,
|
|
|
- CoverImgUrl_235_1,
|
|
|
- ItemShowType,
|
|
|
- IsOriginal,
|
|
|
- ShowDesc,
|
|
|
- ori_content,
|
|
|
- show_view_count,
|
|
|
- show_like_count,
|
|
|
- show_zs_count,
|
|
|
- show_pay_count,
|
|
|
- wx_sn,
|
|
|
- json.dumps(baseInfo, ensure_ascii=False)
|
|
|
- )
|
|
|
insert_sql = f"""
|
|
|
INSERT INTO official_articles
|
|
|
(ghId, accountName, appMsgId, title, Type, createTime, updateTime, Digest, ItemIndex, ContentUrl, SourceUrl, CoverImgUrl, CoverImgUrl_1_1, CoverImgUrl_255_1, ItemShowType, IsOriginal, ShowDesc, ori_content, show_view_count, show_like_count, show_zs_count, show_pay_count, wx_sn, baseInfo)
|
|
@@ -134,11 +134,19 @@ class ArticleSpider(object):
|
|
|
(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);
|
|
|
"""
|
|
|
await self.mysql_client.async_insert(sql=insert_sql, params=info_tuple)
|
|
|
- print("更新成功")
|
|
|
+ print("插入成功")
|
|
|
except Exception as e:
|
|
|
- print("error")
|
|
|
- print(e)
|
|
|
- continue
|
|
|
+ try:
|
|
|
+ update_sql = f"""
|
|
|
+ UPDATE official_articles
|
|
|
+ SET show_view_count = %s, show_like_count=%s
|
|
|
+ WHERE wx_sn = %s;
|
|
|
+ """
|
|
|
+ await self.mysql_client.async_insert(sql=update_sql, params=(show_view_count, show_like_count, wx_sn))
|
|
|
+ print("更新成功")
|
|
|
+ except Exception as e:
|
|
|
+ print("失败-{}".format(e))
|
|
|
+ continue
|
|
|
|
|
|
async def getAccountArticleList(self, gh_id, account_name, last_update_time, cursor=None):
|
|
|
"""
|