# -*- coding: utf-8 -*- import json import os import random import subprocess import sys import time import uuid from datetime import datetime import requests from appium import webdriver from appium.webdriver.extensions.android.nativekey import AndroidKey from appium.webdriver.common.touch_action import TouchAction from bs4 import BeautifulSoup from selenium.common.exceptions import NoSuchElementException from selenium.webdriver.common.by import By sys.path.append(os.getcwd()) from application.common import MysqlHelper, Feishu class XiaoNianGaoZH(object): """ 小年糕+线下爬虫 """ def __init__(self): self.count = 0 self.swipe_count = 0 chromedriverExecutable = "/Users/a123456/Downloads/chromedriver-mac-x64/chromedriver" print("启动微信") # 微信的配置文件 caps = { "platformName": "Android", "devicesName": "Android", "appPackage": "com.tencent.mm", "appActivity": ".ui.LauncherUI", "autoGrantPermissions": True, "noReset": True, "resetkeyboard": True, "unicodekeyboard": True, "showChromedriverLog": True, "printPageSourceOnFailure": True, "recreateChromeDriverSessions": True, "enableWebviewDetailsCollection": True, "setWebContentsDebuggingEnabled": True, "newCommandTimeout": 6000, "automationName": "UiAutomator2", "chromedriverExecutable": chromedriverExecutable, "chromeOptions": {"androidProcess": "com.tencent.mm:appbrand0"}, } try: self.driver = webdriver.Remote("http://localhost:4723/wd/hub", caps) except Exception as e: print(e) return self.driver.implicitly_wait(30) for i in range(10): try: if self.driver.find_elements(By.ID, "com.tencent.mm:id/f2s"): print("启动微信成功") break elif self.driver.find_element( By.ID, "com.android.systemui:id/dismiss_view" ): print("发现并关闭系统下拉菜单") size = self.driver.get_window_size() self.driver.swipe( int(size["width"] * 0.5), int(size["height"] * 0.8), int(size["width"] * 0.5), int(size["height"] * 0.2), 200, ) else: pass except Exception as e: print(f"打开微信异常：{e}") time.sleep(1) size = self.driver.get_window_size() self.driver.swipe( int(size["width"] * 0.5), int(size["height"] * 0.2), int(size["width"] * 0.5), int(size["height"] * 0.8), 200, ) time.sleep(1) command = 'adb shell service call statusbar 2' process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE) process.communicate() self.driver.find_elements(By.XPATH, '//*[@text="小年糕+"]')[-1].click() print("打开小程序小年糕+成功") time.sleep(5) self.get_videoList() time.sleep(1) self.driver.quit() def save_pq_uid(self, uid): url = "https://admin.piaoquantv.com/manager/crawler/v3/task/save" payload = { "taskName": "小年糕账号", "source": "xiaoniangao", "mode": "author", "modeValue": "0", "modeBoard": "0", "spiderName": "run_xng_author", "startTime": 1720589960000, "interval": 4800, "uid": "58527261,58527262,58527263,58527264,58527265,58527266,58527267,58527268,58527269,58527270,58527271,58527272,58527274,58527275,58527277,58527278,58527279,58527280,58527281,58527282,58527284,58527285,58527287,58527289,58527291,58527292,58527293,58527294,58527297,58527302,58527303,58527304,58527307,58527310,58527313,58527314,58527316,58527317,58527318,58527319,58527320,58527323,58527324,58527326,58527328,58527329,58527331,58527332,58527333,58527334,58527335,58527336,58527337,58527338,58527339,58527341,58527342,58527343,58527344,58527345,58527347,58527348,58527349,58527350,58527351,58527352,58527354,58527355,58527356,58527357,58527358,58527359,58527360,58527361,58527362,58527363,58527364,58527365,58527366,58527367,58527368,58527369,58527370,58527372,58527373,58527374,58527375,58527376,58527377,58527378,58527380,58527381,58527382,58527384,58527386,58527388,58527389,58527390,58527391,58527392,58527393,58527394,58527395,58527396,58527397,58527398,58527399,58527401,58527402,58527403,58527404,58527405,58527406,58527407,58527408,58527555,58527556,58527557,58527558,58527559,58527560,58527561,58527562,58527563,58527564,58527565,58527566,58527567,58527569,58527570,58527572,58527573,58527574,58527575,58527577,58527578,58527579,58527580,58527582,58527583,58527584,58527585,58527586,58527587,58527588,58527590,58527591,58527592,58527594,58527595,58527596,58527597,58527598,58527599,58527600,58527601,58527602,58527603,58527604,58527605,58527606,58527607,58527608,58527610,58527611,58527612,58527613,58527614,58527615,58527616,58527617,58527618,58527619,58527620,58527622,58527623,58527624,58527625,58527626,58527627,58527628,58527630,58527631,58527632,58527633,58527634,58527635,58527636,58527637,58527638,58527639,58527640,58527641,58527642,58527643,58527644,58527646,58527647,58527648,58527649,58527650,58527651,58527653,58527654,58527655,58527656,58527657,58527658,58527659,58527661,58527662,58527663,58527665,58527666,58527668,58527669,58527670,58527671,58527672,58527674,58527675,58527676,58527677,58527678,58527679,58527680,58527681,58527682,58527683,58527684,58527686,58527687,58527689,58527691,58527693,58527694,58527695,58527696,58527699,58527700,58527701,58527703,58527774,58527775,58527776,58527777,58527778,58527779,58527780,58527781,58527782,58527783,58527784,58527785,58527786,58527787,58527788,58527789,58527790,58527791,58528067,58528069,58528070,58528071,58528072,58528075,58528076,58528077,58528083,58528084,58528085,58528087,58528089,58528090,58528095,58528101,58528102,58528104,58528108,58528110,58528111,58528112,58528113,58528114,58528116,58528117,58528118,58528119,58528120,58528121,58528122,58528126,58528127,58528128,58528129,58528130,58528131,58528133,58528134,58528136,58528137,58528138,58528139,58528140,58528141,58528142,58528143,58528144,58528145,58528146,58528147,58528148,58528149,58528150,58528151,58528153,58528154,58528155,58528157,58528158,58528159,58528160,58528161,58528162,58528164,58528165,58528166,58528167,58528168,58528169,58528170,58528172,58528173,58528174,58528175,58528176,58528178,58528180,58528182,58528183,58528184,58528185,58528186,58528187,58528191,58528192,58528193,58528194,58528195,58528196,58528243,58528244,58528245,58528246,58528248,58528249,58528251,58528252,58528254,58528256,58528257,58528258,58528259,58528261,58528262,58528263,58528264,58528265,58528266,58528267,58528268,58528269,58528270,58528271,58528273,58528275,58528276,58528277,58528278,58528279,58528281,58528282,58528285,58528286,58528287,58528288,58528289,58528290,58528291,58528292,58528293,58528294,58528295,58528296,58528297,58528298,58528300,58528301,58528302,58528304,58528305,58528306,58528307,58528308,58528309,58528311,58528313,58528314,58528315,58528316,58528317,58528318,58528319,58528322,58528323,58528324,58528325,58528326,58528327,58528328,58528329,58528330,58528331,58528332,58528333,58528334,58528335,58528336,58528337,58528338,58528339,58528340,58528341,58528342,58528343,58528344,58528345,58528346,58528347,58528349,58528350,58528351,58528352,58528353,58528354,58528355,58528356,58528357,58528358,58528359,58528360,58528361,58528362,58528363,58528364,58528365,58528366,58528368,58528369,58528370,58528371,58528372,58528373,58528374,58528375,58528376,58528377,58528378,58528379,58528380,58528381,58528382,58528383,58528385,58528386,58528387,58528389,59441154,59441155,59441156,59441157,59441159,59441161,59441162,59441163,59441164,59441165,59441167,59441168,59441169,59441171,59441172,59441173,59441174,59441175,59441176,59441177,59441178,59441179,59441180,59441181,59441184,59441185,59441187,59441189,59441190,59441191,59441192,59441193,59441195,59441196,59441198,59441199,59441200,59441201,59441203,59441204,59441205,59441206,59441207,59441208,59441209,59441542,59441543,59441544,59441545,59441546,59441548,59441549,59441550,59441551,59441552,59441553,59441554,59441555,59441556,59441557,59441558,59441559,59441561,59441562,59441563,59441564,59441567,59441568,59441570,59441571,59441572,59441573,59441574,59441575,59441576,59441578,59441579,59441580,59441581,59441582,59441583,59441585,59441586,59441587,59441588,59441589,59441590,59441591,59441592,59441593,59758578,60450738,60450739,60450741,60450742,60450743,60450745,60450746,60450804,60450805,60450806,60450807,60450808,60450809,60450810,60450864,60450865,60450867,60450868,60450869,60450871,61423955,61423957,61423959,61423960,61423961,61423962,61423963,61424502,63618094,63618095,63618096,63618097,63618098,63618100,63618101,63618102,63618103,63618104,63618105,63642197,63642198,63642199,63642200,63642201,63642202,63642203,63642204,63642205,63642206,63642207,63642208,63642209,63642210,63642211,63642212,63642213,64174802,64174948,64552369,64552371,64552372,64552373,64552375,64553460,64989051,64989053,64989054,64989055,64989056,64989057,64989058,64989059,64989060,64989062,64989063,64989065,64989066,64989067,64989069,64989071,64989073,64989074,64989075,64989076,64989077,65084914,65084915,65084916,65084917,65452079,65452080,65452082,65452083,65487732,65487734,65487735,65487736,65533417,65533418,65676858,65676859,65676860,65676862,65886766,65886769,65886770,65886771,65886772,65926643,66145108,66145109,66145110,66145111,66145112,66145114,66145117,66807288,66807289,66807290,66807291,66807292,66807293,66807294,66807296,66807297,66807298,66807300,66807301,66807302,66807303,66807304,66807305,66807306,66807307,66807308,66807310,68346900,68346902,68346904,68346905,68346906,68346907,68346908,68346909,68346910,68346911,68346912,68346914,68346915,68346916,68346917,68346920,68346921,68346923,68892013,68892178,68892325,68892424,69017089,69017581,69017582,69084282,69084363,69091905,69091954,69091983,69092023,69161560,69161602,69161850,69163633,69163634,69179667,69180147,69180493,69180556,69180599,69180683,69180710,69180734,69180776,69181050,69297987,69298154,69298238,69304074,69304179,69304253,69304319,69304462,69304493,69304813,69304841,69324493,69324596,69324714,69325233,69325509,69325784,69328364,69343378,69344922,69345210,69345361,69347833,69347873,69386357,69480915,69480934,69480960,69481068,69481088,69483032,69483054,69483187,69493233,69493272,69493437,69493438,69493538,69493779,69493801,69493817,69493984,69493986,69493987,69493988,69514013,69580705,69580740,69580783,69580858,69580977,69591985,69592122,69592123,69610885,69610951,70915819,70915891,70915922,70915979,70916046,70916056,70916058,70916059,70916078,70916092,70916137,70916250,70921732,70921733,70921735,70921736,70921737,70921738,70921739,70921740,70921741,70921742,70921743,70921744,70921746,70921747,70921749,70921750,70921752,70921754,70937312,70937313,70937365,70943283,70943312,70943503,71075813,71080124,71105342,71105754,71105836,71106177,71106292,71106719,71106782,71106857,71107010,71107213,71107656,71107962,71108256,71108389,71108562,71108583,71108983,71109030,71109204,71109271,71110512,71110659,71110728,71110909,71111012,71111286,71111691,71112132,71112174,71112329,71112791,71132231,71132393,71132506,71132673,71132730,71132886,71133848,71134279,71134404,71134791,71134830,71134988,71135278,71136315,71136398,71136587,71136909,71137395,71137425,71137703,71138142,71138353,71138689,71139404,71159271,71159510,71159644,71159884,71160166,71160210,71160616,71161162,71161764,71161857,71162146,71162563,71162939,71163345,71164417,71164818,71165300,71165365,71165824,71166074,71166378,71166531,71191366,71191499,71191936,71192522,71192553,71193793,71194093,71194296,71194818,71195076,71195822,71196276,71196724,71196804,71197109,71197175,71198217,71198711,71199541,71199580,71200008,71201487,71201612,71201706,71241510,71242012,71242327,71242695,71243018,71243887,71244006,71244951,71245411,71245457,71249762,71250870,71251336,71251833,71280753,71281457,71281917,71282352,71282742,71283423,71283505,71284478,71285890,71286177,71286894,71288832,71312885,71313427,71313854,71314180,71316095,71316151,71316526,71317777,71318449,71318504,71318539,71318698,71320268,71345584,71346008,71346178,71347856,71348630,71349539,71349621,71350830,71350981,71351770,71375796,71375860,71376356,71376472,71377289,71379151,71379639,71379699,71379845,71379907,71380924,71381502,71382022,71383383,71406670,71406744,71406767", "machine": "aliyun", "rule": [{"period": {"min": 15, "max": 3}}, {"duration": {"min": 50, "max": 0}}, {"share_cnt": {"min": 2, "max": 0}}, {"videos_cnt": {"min": 300, "max": 0}}], "id": 21 } payload['uid'] += f",{str(uid)}" headers = { 'accept': 'application/json', 'content-type': 'application/json;', 'cookie': 'SESSION=ZmYwMzBmOWItM2M5YS00ZGMyLTk3MjctMzE0YzE4MmUxNThh', 'origin': 'https://admin.piaoquantv.com', 'pragma': 'no-cache', 'priority': 'u=1, i', 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36' } requests.request("POST", url, headers=headers, json=payload) def search_elements(self, xpath): time.sleep(1) windowHandles = self.driver.window_handles for handle in windowHandles: self.driver.switch_to.window(handle) time.sleep(1) try: elements = self.driver.find_elements(By.XPATH, xpath) if elements: return elements except NoSuchElementException: pass def check_to_applet(self, xpath): time.sleep(1) webViews = self.driver.contexts self.driver.switch_to.context(webViews[-1]) windowHandles = self.driver.window_handles for handle in windowHandles: self.driver.switch_to.window(handle) time.sleep(1) try: self.driver.find_element(By.XPATH, xpath) print("切换到WebView成功\n") return except NoSuchElementException: time.sleep(1) def swipe_up(self): self.search_elements('//*[@class="list-list--list"]') size = self.driver.get_window_size() action = TouchAction(self.driver) action.press(x=int(size["width"] * 0.5), y=int(size["height"] * 0.85)) action.wait(ms=1300) # 可以调整等待时间 action.move_to(x=int(size["width"] * 0.5), y=int(size["height"] * 0.2)) action.release() action.perform() self.swipe_count += 1 def get_video_url(self, video_title_element): for i in range(3): self.search_elements('//*[@class="list-list--list"]') time.sleep(1) self.driver.execute_script( "arguments[0].scrollIntoView({block:'center',inline:'center'});", video_title_element[0], ) time.sleep(3) video_title_element[0].click() self.check_to_applet( xpath=r'//wx-video[@class="dynamic-index--video-item dynamic-index--video"]' ) time.sleep(10) video_url_elements = self.search_elements( '//wx-video[@class="dynamic-index--video-item dynamic-index--video"]' ) return video_url_elements[0].get_attribute("src") def parse_detail(self, index): page_source = self.driver.page_source soup = BeautifulSoup(page_source, "html.parser") soup.prettify() video_list = soup.findAll( name="wx-view", attrs={"class": "expose--adapt-parent"} ) index = index + 1 element_list = [i for i in video_list][index:] return element_list[0] def get_video_info_2(self, video_element): self.count += 1 video_title = video_element.find("wx-view", class_="dynamic--title").text # 头像 URL avatar_url = video_element.find("wx-image", class_="avatar--avatar")["src"] # 用户名称 user_name = video_element.find("wx-view", class_="dynamic--nick-top").text name_url = self.select_name_url(avatar_url, user_name) if name_url: video_title_element = self.search_elements(f'//*[contains(text(), "{video_title}")]') if video_title_element is None: return self.get_video_url(video_title_element) video_mid_elements = self.search_elements("//wx-view[@class='bar--navBar-content-capsule-wrap']") mid = int(video_mid_elements[0].get_attribute("data-mid")) self.driver.press_keycode(AndroidKey.BACK) time.sleep(5) uid = self.select_id(mid) if uid: self.update_name_url(mid, avatar_url, user_name) else: time.sleep(1) link = self.select_id_status(mid) if link: current_time = datetime.now() formatted_time = current_time.strftime("%Y%m%d") date_int = int(formatted_time) # 获取时间标签 tag_id = self.get_tag_id(date_int) time.sleep(5) print(tag_id) # 新增账号 pq_uid = self.insert_number(mid, tag_id) time.sleep(5) if pq_uid: self.insert_name_url(mid, avatar_url, user_name) time.sleep(2) self.save_pq_uid(pq_uid) # 获取当前时间 current_time = datetime.now() formatted_time = current_time.strftime("%Y-%m-%d %H:%M:%S") values = [[ str(mid), user_name, avatar_url, str(pq_uid), formatted_time, ]] Feishu.insert_columns('xiaoniangao', 'xiaoniangao', "8zlceR", "ROWS", 1, 2) time.sleep(0.5) Feishu.update_values('xiaoniangao', 'xiaoniangao', "8zlceR", "A2:Z2", values) print("写入飞书表格成功") def get_video_info(self, video_element): try: self.get_video_info_2(video_element) except Exception as e: print(f"抓取单条视频异常:{e}\n") def get_videoList(self): """ 获取视频列表 :return: """ # while True: self.driver.implicitly_wait(20) # 切换到 web_view self.check_to_applet(xpath='//*[@class="tab-bar--tab tab-bar--tab-selected"]') print("切换到 webview 成功") time.sleep(1) if self.search_elements('//*[@class="list-list--list"]') is None: print("窗口已销毁") self.count = 0 self.download_cnt = 0 self.element_list = [] return print("开始获取视频信息") for i in range(50): print("下滑{}次".format(i)) element = self.parse_detail(i) self.get_video_info(element) self.swipe_up() time.sleep(random.randint(1, 5)) def insert_number(self, mid, tag_id): for i in range(3): url = "https://admin.piaoquantv.com/manager/crawler/v3/user/save" payload = { "source": "xiaoniangao", "mode": "author", "modeValue": "", "modeBoard": "", "recomStatus": -6, "appRecomStatus": -6, "autoAuditStatus": 0, "tag": f"459,454,106,8240,{int(tag_id)}", "contentCategory": 0, "link": str(mid) } headers = { 'content-length': '0', 'cookie': 'SESSION=MWM4YzVlMTctNzdkNC00NjE3LWIxZTctOGQwYzgzYmVmN2Qw', 'origin': 'https://admin.piaoquantv.com', 'priority': 'u=1, i', 'sec-ch-ua': '"Not/A)Brand";v="8", "Chromium";v="126", "Google Chrome";v="126"', 'sec-ch-ua-mobile': '?0', 'sec-ch-ua-platform': '"macOS"' } response = requests.request("POST", url, headers=headers, json=payload) response = response.json() code = response["code"] if code == 0: print("添加账号成功") time.sleep(1) url = "https://admin.piaoquantv.com/manager/crawler/v3/user/list" payload = { "pageNum": 1, "pageSize": 20 } response = requests.request("POST", url, headers=headers, json=payload) response = response.json() list = response["content"]['list'] link = list[0]["link"] if link == str(mid): print("获取站内账号ID成功") return list[0]["uid"] """ 查询用户名+头像是否存在 """ def select_name_url(self, avatar_url, user_name): sql = f""" select uid from xng_uid where avatar_url = "{avatar_url}" and user_name="{user_name}"; """ db = MysqlHelper() repeat_video = db.select(sql=sql) if repeat_video: return False return True def get_tag_id(self, date_int): for i in range(3): url = f"https://admin.piaoquantv.com/manager/user/up/searchUserTypeTag?keyword={date_int}&muid=7" payload = {} headers = { 'content-length': '0', 'cookie': 'SESSION=MWM4YzVlMTctNzdkNC00NjE3LWIxZTctOGQwYzgzYmVmN2Qw', 'origin': 'https://admin.piaoquantv.com', 'priority': 'u=1, i', 'sec-ch-ua': '"Not/A)Brand";v="8", "Chromium";v="126", "Google Chrome";v="126"', 'sec-ch-ua-mobile': '?0', 'sec-ch-ua-platform': '"macOS"' } response = requests.request("POST", url, headers=headers, data=payload) response = response.json() content = response["content"] if content: tagId = content[0]['tagId'] return tagId else: url = f"https://admin.piaoquantv.com/manager/user/up/createUserTypeTag?tagName={date_int}&muid=7" response = requests.request("POST", url, headers=headers, data=payload) response = response.json() content = response["content"] if content: tagId = content['tagId'] return tagId """ 修改用户名+头像 """ def update_name_url(self, mid, avatar_url, user_name): sql = f""" update xng_uid set avatar_url = "{avatar_url}", user_name="{user_name}" where uid = "{mid}"; """ db = MysqlHelper() repeat_video = db.update(sql=sql) if repeat_video: return True return False """ 插入 用户名 头像 用户id """ def insert_name_url(self, uid, avatar_url, user_name): current_time = datetime.now() formatted_time = current_time.strftime("%Y-%m-%d %H:%M:%S") insert_sql = f"""INSERT INTO xng_uid (uid, avatar_url, user_name, data_time) values ('{uid}' ,'{avatar_url}','{user_name}', '{formatted_time}')""" db = MysqlHelper() repeat_video = db.update(sql=insert_sql) if repeat_video: return True return False """ 查询用户id是否存在 """ def select_id(self, uid): sql = f""" select uid from xng_uid where uid = "{uid}"; """ db = MysqlHelper() repeat_video = db.select(sql=sql) if repeat_video: return True return False """ 查询用户id是否之前已添加过 """ def select_id_status(self, uid): sql = f""" select uid from crawler_user_v3 where link = "{uid}"; """ db = MysqlHelper() repeat_video = db.select(sql=sql) if repeat_video: return False return True if __name__ == "__main__": XiaoNianGaoZH()