|
|
@@ -9,11 +9,13 @@ import sys
|
|
|
import time
|
|
|
import requests
|
|
|
import urllib3
|
|
|
+
|
|
|
sys.path.append(os.getcwd())
|
|
|
from common.common import Common
|
|
|
from common.scheduling_db import MysqlHelper
|
|
|
from common.publish import Publish
|
|
|
from common.feishu import Feishu
|
|
|
+from common.public import filter_word
|
|
|
proxies = {"http": None, "https": None}
|
|
|
|
|
|
|
|
|
@@ -21,29 +23,6 @@ class XiaoniangaoFollow:
|
|
|
platform = "小年糕"
|
|
|
# 小程序个人主页视频列表翻页参数
|
|
|
next_t = None
|
|
|
- # 配置微信
|
|
|
- # wechat_sheet = Feishu.get_values_batch("follow", "xiaoniangao", "dzcWHw")
|
|
|
- # follow_x_mid = wechat_sheet[2][3]
|
|
|
- # follow_x_token_id = wechat_sheet[3][3]
|
|
|
- # follow_referer = wechat_sheet[4][3]
|
|
|
- # follow_uid = wechat_sheet[5][3]
|
|
|
- # follow_token = wechat_sheet[6][3]
|
|
|
-
|
|
|
- # 过滤敏感词
|
|
|
- @classmethod
|
|
|
- def filter_words(cls, log_type, crawler):
|
|
|
- # 敏感词库列表
|
|
|
- word_list = []
|
|
|
- # 从云文档读取所有敏感词，添加到词库列表
|
|
|
- lists = Feishu.get_values_batch(log_type, crawler, "DRAnZh")
|
|
|
- for i in lists:
|
|
|
- for j in i:
|
|
|
- # 过滤空的单元格内容
|
|
|
- if j is None:
|
|
|
- pass
|
|
|
- else:
|
|
|
- word_list.append(j)
|
|
|
- return word_list
|
|
|
|
|
|
# 基础门槛规则
|
|
|
@staticmethod
|
|
|
@@ -73,30 +52,200 @@ class XiaoniangaoFollow:
|
|
|
|
|
|
# 从云文档获取关注用户列表
|
|
|
@classmethod
|
|
|
- def get_users(cls, log_type, crawler):
|
|
|
- try:
|
|
|
- while True:
|
|
|
- follow_sheet = Feishu.get_values_batch(log_type, "xiaoniangao", "oNpThi")
|
|
|
- if follow_sheet is None:
|
|
|
- time.sleep(1)
|
|
|
- continue
|
|
|
- if len(follow_sheet) == 1:
|
|
|
- Common.logger(log_type, crawler).info("暂无定向爬取账号")
|
|
|
- else:
|
|
|
- user_list = []
|
|
|
- for i in range(1, len(follow_sheet)):
|
|
|
- profile_id = follow_sheet[i][0]
|
|
|
- profile_mid = follow_sheet[i][1]
|
|
|
- user_name = follow_sheet[i][2]
|
|
|
- user_dict = {
|
|
|
- "profile_id": profile_id,
|
|
|
- "profile_mid": profile_mid,
|
|
|
- "user_name": user_name,
|
|
|
- }
|
|
|
- user_list.append(user_dict)
|
|
|
- return user_list
|
|
|
- except Exception as e:
|
|
|
- Common.logger(log_type, crawler).error("从云文档获取关注用户列表异常:{}", e)
|
|
|
+ def get_users(cls):
|
|
|
+ # try:
|
|
|
+ # while True:
|
|
|
+ # follow_sheet = Feishu.get_values_batch(log_type, "xiaoniangao", "oNpThi")
|
|
|
+ # if follow_sheet is None:
|
|
|
+ # time.sleep(1)
|
|
|
+ # continue
|
|
|
+ # if len(follow_sheet) == 1:
|
|
|
+ # Common.logger(log_type, crawler).info("暂无定向爬取账号")
|
|
|
+ # else:
|
|
|
+ # user_list = []
|
|
|
+ # for i in range(1, len(follow_sheet)):
|
|
|
+ # profile_id = follow_sheet[i][0]
|
|
|
+ # profile_mid = follow_sheet[i][1]
|
|
|
+ # user_name = follow_sheet[i][2]
|
|
|
+ # user_dict = {
|
|
|
+ # "profile_id": str(profile_id),
|
|
|
+ # "profile_mid": str(profile_mid),
|
|
|
+ # "user_name": user_name,
|
|
|
+ # }
|
|
|
+ # user_list.append(user_dict)
|
|
|
+ # return user_list
|
|
|
+ # except Exception as e:
|
|
|
+ # Common.logger(log_type, crawler).error("从云文档获取关注用户列表异常:{}", e)
|
|
|
+ user_list = [{'profile_id': '43549964', 'profile_mid': '116311065', 'user_name': '青峰明月老人'},
|
|
|
+ {'profile_id': '43486474', 'profile_mid': '1113556014', 'user_name': '好人一生平安'},
|
|
|
+ {'profile_id': '43590330', 'profile_mid': '192501730', 'user_name': '风轻云淡'},
|
|
|
+ {'profile_id': '43559741', 'profile_mid': '90261331', 'user_name': '歌一曲'},
|
|
|
+ {'profile_id': '43785654', 'profile_mid': '104703232', 'user_name': '涧欣'},
|
|
|
+ {'profile_id': '44211463', 'profile_mid': '1149063512', 'user_name': '美食分享'},
|
|
|
+ {'profile_id': '46661040', 'profile_mid': '293457861', 'user_name': '阿妞'},
|
|
|
+ {'profile_id': '46731019', 'profile_mid': '1059483000', 'user_name': '早睡早起锻炼身体'},
|
|
|
+ {'profile_id': '46644244', 'profile_mid': '609255292', 'user_name': '雨太漫长'},
|
|
|
+ {'profile_id': '46553912', 'profile_mid': '1160417287', 'user_name': '葛锡琴'},
|
|
|
+ {'profile_id': '46802824', 'profile_mid': '1157645694', 'user_name': '留白'},
|
|
|
+ {'profile_id': '46795509', 'profile_mid': '1161593368', 'user_name': '绒花漫舞'},
|
|
|
+ {'profile_id': '46837469', 'profile_mid': '1160417275', 'user_name': '苏超灵'},
|
|
|
+ {'profile_id': '46830553', 'profile_mid': '1160417336', 'user_name': '山里红'},
|
|
|
+ {'profile_id': '50541284', 'profile_mid': '1210725734', 'user_name': '语琴'},
|
|
|
+ {'profile_id': '50541428', 'profile_mid': '1222976096', 'user_name': '血性娘子'},
|
|
|
+ {'profile_id': '50364411', 'profile_mid': '1192924729', 'user_name': '好运莲莲'},
|
|
|
+ {'profile_id': '50499182', 'profile_mid': '329676836', 'user_name': '花好月圆'},
|
|
|
+ {'profile_id': '50449938', 'profile_mid': '341183833', 'user_name': '花开富贵'},
|
|
|
+ {'profile_id': '50199770', 'profile_mid': '1166029432', 'user_name': '兰花草1'},
|
|
|
+ {'profile_id': '50299362', 'profile_mid': '1013092427', 'user_name': '感悟人生'},
|
|
|
+ {'profile_id': '50539176', 'profile_mid': '1203033756', 'user_name': '风之别鹤'},
|
|
|
+ {'profile_id': '50541092', 'profile_mid': '1237693278', 'user_name': '莲心'},
|
|
|
+ {'profile_id': '50523015', 'profile_mid': '260159327', 'user_name': '微笑😊123'},
|
|
|
+ {'profile_id': '50216970', 'profile_mid': '1219153129', 'user_name': '秋夕'},
|
|
|
+ {'profile_id': '50286181', 'profile_mid': '1113733998', 'user_name': '上善若水'},
|
|
|
+ {'profile_id': '50505135', 'profile_mid': '185546', 'user_name': '秋水揽星河'},
|
|
|
+ {'profile_id': '50555007', 'profile_mid': '1113739184', 'user_name': '淡墨青衫'},
|
|
|
+ {'profile_id': '50500343', 'profile_mid': '1072539904', 'user_name': '微笑向明天'},
|
|
|
+ {'profile_id': '50628735', 'profile_mid': '1142824266', 'user_name': '清爽'},
|
|
|
+ {'profile_id': '50580367', 'profile_mid': '26055443', 'user_name': '玉儿Syy'},
|
|
|
+ {'profile_id': '50585024', 'profile_mid': '1105334217', 'user_name': '老树开花🌸'},
|
|
|
+ {'profile_id': '50380683', 'profile_mid': '63682047', 'user_name': '海王岛浩哥'},
|
|
|
+ {'profile_id': '46386724', 'profile_mid': '271535337', 'user_name': '雪梅'},
|
|
|
+ {'profile_id': '50636398', 'profile_mid': '237071458', 'user_name': '看花人'},
|
|
|
+ {'profile_id': '50624991', 'profile_mid': '1202244309', 'user_name': '无关风月'},
|
|
|
+ {'profile_id': '50561528', 'profile_mid': '1134790664', 'user_name': '老有所乐'},
|
|
|
+ {'profile_id': '50612235', 'profile_mid': '1162974507', 'user_name': '花素千昔'},
|
|
|
+ {'profile_id': '50933310', 'profile_mid': '1237661365', 'user_name': '淡淡浅香'},
|
|
|
+ {'profile_id': '50492961', 'profile_mid': '1160417339', 'user_name': '胡淑华'},
|
|
|
+ {'profile_id': '49965729', 'profile_mid': '1160417117', 'user_name': '百姓生活'},
|
|
|
+ {'profile_id': '50928693', 'profile_mid': '1163097085', 'user_name': '多福'},
|
|
|
+ {'profile_id': '50819153', 'profile_mid': '224365299', 'user_name': '欢天喜地'},
|
|
|
+ {'profile_id': '51510752', 'profile_mid': '1160417314', 'user_name': '郝建兵'},
|
|
|
+ {'profile_id': '51470002', 'profile_mid': '631317964', 'user_name': '九牛二虎'},
|
|
|
+ {'profile_id': '51466430', 'profile_mid': '177806025', 'user_name': '越来越好'},
|
|
|
+ {'profile_id': '51480157', 'profile_mid': '1000526700', 'user_name': '光伯伯'},
|
|
|
+ {'profile_id': '51396620', 'profile_mid': '1160417243', 'user_name': '亚萍'},
|
|
|
+ {'profile_id': '51538137', 'profile_mid': '154002316', 'user_name': 'hua jean-pierre 許遵訓'},
|
|
|
+ {'profile_id': '51533676', 'profile_mid': '197908577', 'user_name': '萍水相逢'},
|
|
|
+ {'profile_id': '51488254', 'profile_mid': '153685063', 'user_name': '英勇'},
|
|
|
+ {'profile_id': '51425929', 'profile_mid': '1161593426', 'user_name': '百年字号'},
|
|
|
+ {'profile_id': '51514370', 'profile_mid': '1198361106', 'user_name': '品味人生'},
|
|
|
+ {'profile_id': '51405010', 'profile_mid': '1161593390', 'user_name': '低调生活'},
|
|
|
+ {'profile_id': '51535995', 'profile_mid': '690938173', 'user_name': '世界风'},
|
|
|
+ {'profile_id': '51468900', 'profile_mid': '1242534599', 'user_name': '人生如茶'},
|
|
|
+ {'profile_id': '51489772', 'profile_mid': '269038284', 'user_name': '韵律'},
|
|
|
+ {'profile_id': '51377241', 'profile_mid': '194287386', 'user_name': '霹雳雷天发'},
|
|
|
+ {'profile_id': '51538527', 'profile_mid': '1064479786', 'user_name': '鑫'},
|
|
|
+ {'profile_id': '51365161', 'profile_mid': '1161593351', 'user_name': '黄金三十分'},
|
|
|
+ {'profile_id': '51527297', 'profile_mid': '1163011474', 'user_name': '指日东升'},
|
|
|
+ {'profile_id': '51356555', 'profile_mid': '76036942', 'user_name': '利剑'},
|
|
|
+ {'profile_id': '51382702', 'profile_mid': '1160417349', 'user_name': '丽丽'},
|
|
|
+ {'profile_id': '51536925', 'profile_mid': '1160417133', 'user_name': '福海'},
|
|
|
+ {'profile_id': '51525148', 'profile_mid': '1160417333', 'user_name': '君子兰'},
|
|
|
+ {'profile_id': '51530896', 'profile_mid': '1162996019', 'user_name': '你好，朋友'},
|
|
|
+ {'profile_id': '51532418', 'profile_mid': '1162998153', 'user_name': '旭日东升'},
|
|
|
+ {'profile_id': '51329950', 'profile_mid': '285062559', 'user_name': '阿静'},
|
|
|
+ {'profile_id': '51512535', 'profile_mid': '114981390', 'user_name': '泽南'},
|
|
|
+ {'profile_id': '51480066', 'profile_mid': '1160417334', 'user_name': '克平'},
|
|
|
+ {'profile_id': '51520908', 'profile_mid': '1161593408', 'user_name': '永生'},
|
|
|
+ {'profile_id': '51496944', 'profile_mid': '1160417145', 'user_name': '一生快乐'},
|
|
|
+ {'profile_id': '51511833', 'profile_mid': '1105154277', 'user_name': '老周.物流全国.'},
|
|
|
+ {'profile_id': '51515937', 'profile_mid': '1178673137', 'user_name': '风起风落'},
|
|
|
+ {'profile_id': '51539903', 'profile_mid': '1175023459', 'user_name': '花开福贵 &吉祥'},
|
|
|
+ {'profile_id': '51394812', 'profile_mid': '1160417173', 'user_name': '尔东'},
|
|
|
+ {'profile_id': '51512298', 'profile_mid': '22171304', 'user_name': '张杰真'},
|
|
|
+ {'profile_id': '51472642', 'profile_mid': '175855074', 'user_name': '张顺淑'},
|
|
|
+ {'profile_id': '51548416', 'profile_mid': '1116847111', 'user_name': '沁水百合'},
|
|
|
+ {'profile_id': '51456595', 'profile_mid': '1075862992', 'user_name': '永保青春'},
|
|
|
+ {'profile_id': '51502433', 'profile_mid': '1162980250', 'user_name': '午夜想念'},
|
|
|
+ {'profile_id': '51495781', 'profile_mid': '1162961480', 'user_name': '快乐每一天'},
|
|
|
+ {'profile_id': '51527180', 'profile_mid': '1161593427', 'user_name': '彩云追月'},
|
|
|
+ {'profile_id': '51537082', 'profile_mid': '1163005063', 'user_name': '峥嵘岁月'},
|
|
|
+ {'profile_id': '51510904', 'profile_mid': '64295407', 'user_name': '文文'},
|
|
|
+ {'profile_id': '51497047', 'profile_mid': '59349938', 'user_name': '老马识途'},
|
|
|
+ {'profile_id': '51544604', 'profile_mid': '1160417260', 'user_name': '星'},
|
|
|
+ {'profile_id': '51411909', 'profile_mid': '801020924', 'user_name': '相思鸟'},
|
|
|
+ {'profile_id': '51525860', 'profile_mid': '1160417222', 'user_name': '老来得福'},
|
|
|
+ {'profile_id': '51540902', 'profile_mid': '1163003217', 'user_name': '银发中年'},
|
|
|
+ {'profile_id': '51526760', 'profile_mid': '295640510', 'user_name': '阿中'},
|
|
|
+ {'profile_id': '51529247', 'profile_mid': '674650199', 'user_name': '张恩培'},
|
|
|
+ {'profile_id': '51540032', 'profile_mid': '18103420', 'user_name': '吴希孟'},
|
|
|
+ {'profile_id': '51491818', 'profile_mid': '1162991035', 'user_name': '幸福每一天'},
|
|
|
+ {'profile_id': '51539843', 'profile_mid': '210721352', 'user_name': '掉地人参果'},
|
|
|
+ {'profile_id': '51521237', 'profile_mid': '1160417237', 'user_name': '山野村夫'},
|
|
|
+ {'profile_id': '51531438', 'profile_mid': '103235841', 'user_name': '开心'},
|
|
|
+ {'profile_id': '51528239', 'profile_mid': '1160417288', 'user_name': '戴玉亭'},
|
|
|
+ {'profile_id': '51542817', 'profile_mid': '126121769', 'user_name': '芳芳'},
|
|
|
+ {'profile_id': '51541108', 'profile_mid': '287637208', 'user_name': '心无旁骛'},
|
|
|
+ {'profile_id': '51462517', 'profile_mid': '247008734', 'user_name': '华'},
|
|
|
+ {'profile_id': '51543209', 'profile_mid': '1214482340', 'user_name': '丽'},
|
|
|
+ {'profile_id': '51509548', 'profile_mid': '1162977406', 'user_name': '绚丽多彩'},
|
|
|
+ {'profile_id': '51536733', 'profile_mid': '1161593366', 'user_name': '夕阳红'},
|
|
|
+ {'profile_id': '51519407', 'profile_mid': '1122750580', 'user_name': '幸福满堂'},
|
|
|
+ {'profile_id': '51459413', 'profile_mid': '348958961', 'user_name': '段平清'},
|
|
|
+ {'profile_id': '51487820', 'profile_mid': '605290310', 'user_name': '青青'},
|
|
|
+ {'profile_id': '51469080', 'profile_mid': '131475097', 'user_name': '手宝'},
|
|
|
+ {'profile_id': '51426171', 'profile_mid': '196597807', 'user_name': '妙'},
|
|
|
+ {'profile_id': '51470293', 'profile_mid': '78386273', 'user_name': '芳草'},
|
|
|
+ {'profile_id': '51505305', 'profile_mid': '172977522', 'user_name': '伟星'},
|
|
|
+ {'profile_id': '51516061', 'profile_mid': '913804992', 'user_name': '荷塘月色'},
|
|
|
+ {'profile_id': '51541167', 'profile_mid': '1100776170', 'user_name': '闪电⚡️'},
|
|
|
+ {'profile_id': '51391450', 'profile_mid': '982518039', 'user_name': '吕梁'},
|
|
|
+ {'profile_id': '51529786', 'profile_mid': '168833501', 'user_name': '：shen. ser'},
|
|
|
+ {'profile_id': '51416422', 'profile_mid': '376557509', 'user_name': '于孝震'},
|
|
|
+ {'profile_id': '51342944', 'profile_mid': '556796156', 'user_name': '☀吉祥☀'},
|
|
|
+ {'profile_id': '51536144', 'profile_mid': '1162956349', 'user_name': '凤舞九天'},
|
|
|
+ {'profile_id': '51533164', 'profile_mid': '153787914', 'user_name': '红梅'},
|
|
|
+ {'profile_id': '51450974', 'profile_mid': '20396528', 'user_name': '学友'},
|
|
|
+ {'profile_id': '51512622', 'profile_mid': '35883962', 'user_name': '珎珎'},
|
|
|
+ {'profile_id': '51525737', 'profile_mid': '69546032', 'user_name': '欣然'},
|
|
|
+ {'profile_id': '51414093', 'profile_mid': '1154645646', 'user_name': '老中医'},
|
|
|
+ {'profile_id': '51372351', 'profile_mid': '154481426', 'user_name': '寻花'},
|
|
|
+ {'profile_id': '51511869', 'profile_mid': '1162958849', 'user_name': '保持微笑'},
|
|
|
+ {'profile_id': '51367397', 'profile_mid': '173682337', 'user_name': '莲'},
|
|
|
+ {'profile_id': '51494361', 'profile_mid': '1161593403', 'user_name': '红叶'},
|
|
|
+ {'profile_id': '51352260', 'profile_mid': '31040287', 'user_name': '52部队'},
|
|
|
+ {'profile_id': '51430672', 'profile_mid': '69886497', 'user_name': '张俊宽'},
|
|
|
+ {'profile_id': '51440076', 'profile_mid': '1186145367', 'user_name': '金光'},
|
|
|
+ {'profile_id': '51431733', 'profile_mid': '112830331', 'user_name': '幸福安康'},
|
|
|
+ {'profile_id': '51511957', 'profile_mid': '1214482340', 'user_name': '丽'},
|
|
|
+ {'profile_id': '51466969', 'profile_mid': '131962346', 'user_name': '丹桂'},
|
|
|
+ {'profile_id': '51515739', 'profile_mid': '1162950925', 'user_name': '烈火'},
|
|
|
+ {'profile_id': '51425141', 'profile_mid': '207104529', 'user_name': '雪梅'},
|
|
|
+ {'profile_id': '51465625', 'profile_mid': '913804992', 'user_name': '荷塘月色'},
|
|
|
+ {'profile_id': '51470424', 'profile_mid': '690938173', 'user_name': '世界风'},
|
|
|
+ {'profile_id': '51502065', 'profile_mid': '1105154277', 'user_name': '老周.物流全国.'},
|
|
|
+ {'profile_id': '51505082', 'profile_mid': '674650199', 'user_name': '张恩培'},
|
|
|
+ {'profile_id': '51508736', 'profile_mid': '30955754', 'user_name': '硕'},
|
|
|
+ {'profile_id': '51363432', 'profile_mid': '1160417336', 'user_name': '山里红'},
|
|
|
+ {'profile_id': '51472849', 'profile_mid': '1160417336', 'user_name': '山里红'},
|
|
|
+ {'profile_id': '51531002', 'profile_mid': '1149063512', 'user_name': '美食分享'},
|
|
|
+ {'profile_id': '51369043', 'profile_mid': '1162996019', 'user_name': '你好，朋友'},
|
|
|
+ {'profile_id': '51488704', 'profile_mid': '1160417201', 'user_name': '伴迪伴你'},
|
|
|
+ {'profile_id': '51514738', 'profile_mid': '1160417222', 'user_name': '老来得福'},
|
|
|
+ {'profile_id': '51526888', 'profile_mid': '1160417349', 'user_name': '丽丽'},
|
|
|
+ {'profile_id': '51513568', 'profile_mid': '1163005063', 'user_name': '峥嵘岁月'},
|
|
|
+ {'profile_id': '51514871', 'profile_mid': '1161593379', 'user_name': '心无界'},
|
|
|
+ {'profile_id': '51518737', 'profile_mid': '1160417333', 'user_name': '君子兰'},
|
|
|
+ {'profile_id': '51525297', 'profile_mid': '1160417115', 'user_name': '绚丽多彩'},
|
|
|
+ {'profile_id': '51383706', 'profile_mid': '1162953017', 'user_name': '荷塘月色'},
|
|
|
+ {'profile_id': '51528569', 'profile_mid': '1162977406', 'user_name': '绚丽多彩'},
|
|
|
+ {'profile_id': '51437102', 'profile_mid': '1162958849', 'user_name': '保持微笑'},
|
|
|
+ {'profile_id': '51536771', 'profile_mid': '1160417243', 'user_name': '亚萍'},
|
|
|
+ {'profile_id': '51516969', 'profile_mid': '1161593390', 'user_name': '低调生活'},
|
|
|
+ {'profile_id': '51535885', 'profile_mid': '1160417243', 'user_name': '亚萍'},
|
|
|
+ {'profile_id': '51517906', 'profile_mid': '1160417268', 'user_name': '宏蕴'},
|
|
|
+ {'profile_id': '51530809', 'profile_mid': '1163003217', 'user_name': '银发中年'},
|
|
|
+ {'profile_id': '51458395', 'profile_mid': '1160417288', 'user_name': '戴玉亭'},
|
|
|
+ {'profile_id': '51517169', 'profile_mid': '1163601308', 'user_name': '今日祝福'},
|
|
|
+ {'profile_id': '51515458', 'profile_mid': '1162953017', 'user_name': '荷塘月色'},
|
|
|
+ {'profile_id': '51498134', 'profile_mid': '1160417262', 'user_name': '快乐每一天'},
|
|
|
+ {'profile_id': '51436822', 'profile_mid': '1162996019', 'user_name': '你好，朋友'},
|
|
|
+ {'profile_id': '51466833', 'profile_mid': '1160417349', 'user_name': '丽丽'},
|
|
|
+ {'profile_id': '51514420', 'profile_mid': '1242534599', 'user_name': '人生如茶'},
|
|
|
+ {'profile_id': '51329643', 'profile_mid': '1160417133', 'user_name': '福海'},
|
|
|
+ {'profile_id': '51384888', 'profile_mid': '1160417349', 'user_name': '丽丽'},
|
|
|
+ {'profile_id': '51382334', 'profile_mid': '1177277385', 'user_name': '丝花'}]
|
|
|
+ return user_list
|
|
|
|
|
|
# 获取个人主页视频
|
|
|
@classmethod
|
|
|
@@ -120,7 +269,7 @@ class XiaoniangaoFollow:
|
|
|
"Referer": 'https://servicewechat.com/wxd7911e4c177690e4/654/page-frame.html'
|
|
|
}
|
|
|
json_text = {
|
|
|
- "visited_mid": p_mid,
|
|
|
+ "visited_mid": str(p_mid),
|
|
|
"start_t": cls.next_t,
|
|
|
"qs": "imageMogr2/gravity/center/rotate/$/thumbnail/!690x385r/crop/690x385/interlace/1/format/jpg",
|
|
|
"h_qs": "imageMogr2/gravity/center/rotate/$/thumbnail/!120x120r/crop/120x120/interlace/1/format/jpg",
|
|
|
@@ -291,31 +440,31 @@ class XiaoniangaoFollow:
|
|
|
or video_duration == 0 \
|
|
|
or video_url == 0:
|
|
|
Common.logger(log_type, crawler).info("无效视频\n")
|
|
|
- elif int(time.time()) - publish_time_stamp > 3600*24*3:
|
|
|
+ elif int(time.time()) - publish_time_stamp > 3600 * 24 * 3:
|
|
|
Common.logger(log_type, crawler).info(f"发布时间超过3天:{publish_time_str}\n")
|
|
|
cls.next_t = None
|
|
|
return
|
|
|
else:
|
|
|
video_dict = {
|
|
|
- "video_id": video_id,
|
|
|
- "video_title": video_title,
|
|
|
- "duration": video_duration,
|
|
|
- "play_cnt": video_play_cnt,
|
|
|
- "like_cnt": video_like_cnt,
|
|
|
- "comment_cnt": video_comment_cnt,
|
|
|
- "share_cnt": video_share_cnt,
|
|
|
- "user_name": user_name,
|
|
|
- "publish_time_stamp": publish_time_stamp,
|
|
|
- "publish_time_str": publish_time_str,
|
|
|
- "video_width": video_width,
|
|
|
- "video_height": video_height,
|
|
|
- "avatar_url": head_url,
|
|
|
- "profile_id": profile_id,
|
|
|
- "profile_mid": profile_mid,
|
|
|
- "cover_url": cover_url,
|
|
|
- "video_url": video_url,
|
|
|
- "session": f"xiaoniangao-follow-{int(time.time())}"
|
|
|
- }
|
|
|
+ "video_id": video_id,
|
|
|
+ "video_title": video_title,
|
|
|
+ "duration": video_duration,
|
|
|
+ "play_cnt": video_play_cnt,
|
|
|
+ "like_cnt": video_like_cnt,
|
|
|
+ "comment_cnt": video_comment_cnt,
|
|
|
+ "share_cnt": video_share_cnt,
|
|
|
+ "user_name": user_name,
|
|
|
+ "publish_time_stamp": publish_time_stamp,
|
|
|
+ "publish_time_str": publish_time_str,
|
|
|
+ "video_width": video_width,
|
|
|
+ "video_height": video_height,
|
|
|
+ "avatar_url": head_url,
|
|
|
+ "profile_id": profile_id,
|
|
|
+ "profile_mid": profile_mid,
|
|
|
+ "cover_url": cover_url,
|
|
|
+ "video_url": video_url,
|
|
|
+ "session": f"xiaoniangao-follow-{int(time.time())}"
|
|
|
+ }
|
|
|
for k, v in video_dict.items():
|
|
|
Common.logger(log_type, crawler).info(f"{k}:{v}")
|
|
|
cls.download_publish(log_type=log_type,
|
|
|
@@ -341,13 +490,15 @@ class XiaoniangaoFollow:
|
|
|
Common.logger(log_type, crawler).info("不满足基础门槛\n")
|
|
|
elif cls.repeat_video(log_type, crawler, video_dict['video_id'], env) != 0:
|
|
|
Common.logger(log_type, crawler).info('视频已下载\n')
|
|
|
- elif any(str(word) if str(word) in video_dict['video_title'] else False for word in cls.filter_words(log_type, crawler)) is True:
|
|
|
+ elif any(str(word) if str(word) in video_dict['video_title'] else False for word in filter_word(log_type, crawler, "小年糕", env)) is True:
|
|
|
Common.logger(log_type, crawler).info("视频已中过滤词\n")
|
|
|
else:
|
|
|
# 下载封面
|
|
|
- Common.download_method(log_type=log_type, crawler=crawler, text="cover", title=video_dict["video_title"], url=video_dict["cover_url"])
|
|
|
+ Common.download_method(log_type=log_type, crawler=crawler, text="cover",
|
|
|
+ title=video_dict["video_title"], url=video_dict["cover_url"])
|
|
|
# 下载视频
|
|
|
- Common.download_method(log_type=log_type, crawler=crawler, text="video", title=video_dict["video_title"], url=video_dict["video_url"])
|
|
|
+ Common.download_method(log_type=log_type, crawler=crawler, text="video",
|
|
|
+ title=video_dict["video_title"], url=video_dict["video_url"])
|
|
|
# 保存视频信息至 "./videos/{download_video_title}/info.txt"
|
|
|
Common.save_video_info(log_type=log_type, crawler=crawler, video_dict=video_dict)
|
|
|
|
|
|
@@ -442,7 +593,7 @@ class XiaoniangaoFollow:
|
|
|
def get_follow_videos(cls, log_type, crawler, strategy, oss_endpoint, env):
|
|
|
try:
|
|
|
# 已关注的用户列表 mids
|
|
|
- user_list = cls.get_users(log_type, crawler)
|
|
|
+ user_list = cls.get_users()
|
|
|
for user in user_list:
|
|
|
user_name = user['user_name']
|
|
|
profile_mid = user['profile_mid']
|
|
|
@@ -461,5 +612,6 @@ class XiaoniangaoFollow:
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
# print(XiaoniangaoFollow.repeat_video("follow", "xiaoniangao", "4919087666", "prod", "aliyun"))
|
|
|
- print(XiaoniangaoFollow.repeat_video("follow", "xiaoniangao", "4919087666", "dev"))
|
|
|
- pass
|
|
|
+ # print(XiaoniangaoFollow.repeat_video("follow", "xiaoniangao", "4919087666", "dev"))
|
|
|
+ print(XiaoniangaoFollow.get_users())
|
|
|
+ pass
|
