| 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061 |
- import json
- from common.mq import MQ
- from tqdm import tqdm
- if __name__ == "__main__":
- # path = 'message.txt'
- # with open(path, "r", encoding="utf-8") as f:
- # datas = f.readlines()
- # for line in tqdm(datas):
- # # for line in datas:
- # video_dict = {}
- # msg = json.loads(line)['msg']
- # # print(msg)
- # platform = json.loads(line)['platform']
- # strategy = json.loads(line)['strategy']
- # strs = msg.replace("CrawlerEtlParam", "")[1: -1]
- # str_list = strs.split(", ")
- # str_list = [i.replace("'", "") for i in str_list]
- # key_dict = {
- # "crawlerRule": "crawler_rule",
- # "userId": "user_id",
- # "outUserId": "out_user_id",
- # "userName": "user_name",
- # "avatarUrl": "avatar_url",
- # "outVideoId": "out_video_id",
- # "videoTitle": "video_title",
- # "coverUrl": "cover_url",
- # "videoUrl": "video_url",
- # "publishTime": "publish_time",
- # "playCnt": "play_cnt",
- # "likeCnt": "like_cnt",
- # "shareCnt": "share_cnt",
- # "collectionCnt": "collection_cnt",
- # "commentCnt": "comment_cnt",
- # "strategyType": "strategy"
- # }
- # for index, i in enumerate(str_list[:-4]):
- # if "=" not in i:
- # continue
- # else:
- # key = i.split("=")[0]
- # value = i[len(key) + 1:]
- # new_key = key_dict.get(key, key)
- # video_dict[new_key] = value.replace("null", "")
- #
- # video_dict['strategy'] = strategy
- # video_dict['platform'] = platform
- # video_dict['crawler_rule'] = json.dumps({})
- # print(json.dumps(video_dict, ensure_ascii=False, indent=4))
- # mq = MQ(topic_name="topic_crawler_etl_" + "prod")
- # mq.send_msg(video_dict)
- video_dict = {'video_title': '吴尊友因病去世！吴老师，您真的不容易，千言万语，汇成一句话您走好❗️', 'video_id': '5262651713', 'duration': 49, 'play_cnt': 71, 'like_cnt': 0, 'comment_cnt': 0, 'share_cnt': 1, 'user_name': '夏日❤️莲莲', 'publish_time_stamp': 1698398572, 'publish_time_str': '2023-10-27 17:22:52', 'video_width': 537, 'video_height': 954, 'avatar_url': 'https://cdn-xphoto2.xiaoniangao.cn/4987933869?Expires=1704038400&OSSAccessKeyId=LTAI4G2W1FsgwzAWYpPoB3v6&Signature=wopOmtlcp9tGyWHYW9uy7DIXO%2Bg%3D&x-oss-process=image%2Fresize%2Cw_200%2Ch_200%2Climit_0%2Finterlace%2C1%2Fquality%2Cq_50%2Fcrop%2Cw_200%2Ch_200%2Cg_center%2Fformat%2Cjpg%2Fauto-orient%2C0', 'profile_id': 55888345, 'profile_mid': 185546, 'cover_url': 'https://cdn-xphoto2.xiaoniangao.cn/5262652619?Expires=1704038400&OSSAccessKeyId=LTAI4G2W1FsgwzAWYpPoB3v6&Signature=qIIRzRICgyv40n3uMFeMwHCY8JY%3D&x-oss-process=image%2Fresize%2Cw_690%2Ch_385%2Climit_0%2Finterlace%2C1%2Fformat%2Cjpg%2Fauto-orient%2C0', 'video_url': 'https://cdn-xalbum2.xiaoniangao.cn/5262651713?Expires=1704038400&OSSAccessKeyId=LTAI5tB7cRkYiqHcTdkVprwb&Signature=hFGFAB49mmgUYwYcF4679bE%2BgLg%3D', 'session': 'xiaoniangao-author-1698402882'}
- video_dict['strategy'] = "author"
- video_dict['platform'] = "xiaoniangao"
- video_dict['user_id'] = 58528269
- video_dict['out_video_id'] = video_dict['video_id']
- print(json.dumps(video_dict, ensure_ascii=False, indent=4))
- mq = MQ(topic_name="topic_crawler_etl_" + "prod")
- mq.send_msg(video_dict)
|
