piaoquan_crawler/xigua/xigua_search/xigua_search_dev.py

# -*- coding: utf-8 -*-
# @Author: wangkun
# @Time: 2023/6/25
import json
import time
import requests
import urllib.parse
from selenium.webdriver import DesiredCapabilities
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from seleniumwire import webdriver


class SearchDev:
    @classmethod
    def get_videoList_requests(cls):
        url = "https://www.ixigua.com/api/searchv2/complex/猪八戒/0?" \
              "fss=default_search&" \
              "order_type=publish_time&" \
              "click_position=new&" \
              "aid=1768&" \
              "msToken=EV6DlzmvSZH6yBIIm7tCdxb6EY7xuV7p0EZw4nZUyznGvXk9Wkyx0GiT39zCO2HRROdUYZc0XYpAztUSzg14q3a1Fkoj01Avy_BGjKFFn5wRQDP8nVWECA==&" \
              "X-Bogus=DFSzswVuSIsANrq4tnr0UFm4pID1&" \
              "_signature=_02B4Z6wo00001jeNZ4AAAIDCr-bw8w.DSLY3jWMAAOmJTnwirif4XNCUKjt3Ms0gS9-upb8jMBZJL5RSZ5dHBQm6GRMtSyn8h6D5rc1Y7tmwZL7a2nP390R3ARXFwF6tVQi97vqO5viH53M0c3"

        payload = {}
        headers = {
            # 'authority': 'www.ixigua.com',
            'accept': 'application/json, text/plain, */*',
            'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
            'cache-control': 'no-cache',
            # 'cookie': 'MONITOR_WEB_ID=67cb5099-a022-4ec3-bb8e-c4de6ba51dd0; sid_guard=c7472b508ea631823ba765a60cf8757f%7C1680867422%7C3024002%7CFri%2C+12-May-2023+11%3A37%3A04+GMT; odin_tt=b893608d4dde2e1e8df8cd5d97a0e2fbeafc4ca762ac72ebef6e6c97e2ed19859bb01d46b4190ddd6dd17d7f9678e1de; s_v_web_id=verify_lhoket5d_0qlKZtzS_YZkf_4Uaj_82mX_j6lRT4PcYJ7A; __ac_signature=_02B4Z6wo00f01yB6eXwAAIDCWLSSerYAxYsgWn3AAKx5S2D2PsJJ92YblwdDE-9rnwnzZ87S0CUowZ3Xi8XmxMU3JHd0xfP-9VucrE9D.l9E7Vgn6y95sGbL2H6mgsddoCZX0cCgfcfKAzWgcd; ixigua-a-s=1; support_webp=true; support_avif=false; csrf_session_id=a5355d954d3c63ed1ba35faada452b4d; SEARCH_CARD_MODE=7168304743566296612_1; msToken=EV6DlzmvSZH6yBIIm7tCdxb6EY7xuV7p0EZw4nZUyznGvXk9Wkyx0GiT39zCO2HRROdUYZc0XYpAztUSzg14q3a1Fkoj01Avy_BGjKFFn5wRQDP8nVWECA==; tt_scid=rP8nVwFTm4wPZyREet0crbp-ZRgJsK.x5TE0lqU2uibGbUDAhlM.oA14pKRcGzXW0955; ttwid=1%7CHHtv2QqpSGuSu8r-zXF1QoWsvjmNi1SJrqOrZzg-UCY%7C1687685218%7Ca985a413a36bb156ba577dac11fbc14593e5a2a4000001f9cfc7fd72781c4cc5; ixigua-a-s=1',
            'pragma': 'no-cache',
            'referer': f'https://www.ixigua.com/search/{urllib.parse.quote("猪八戒")}/?logTag=e0b95015015c05e60b1b&tab_name=home&fss=default_search',
            'sec-ch-ua': '"Microsoft Edge";v="113", "Chromium";v="113", "Not-A.Brand";v="24"',
            'sec-ch-ua-mobile': '?0',
            'sec-ch-ua-platform': '"macOS"',
            'sec-fetch-dest': 'empty',
            'sec-fetch-mode': 'cors',
            'sec-fetch-site': 'same-origin',
            'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.57',
            # 'x-secsdk-csrf-token': '0001000000011fd0adbaee655439e86800862b81e3e34974cab6a8656af77695b76ff5c76c96176bdcbf2631eeb7'
        }

        response = requests.request("GET", url, headers=headers, data=payload)

        print(response.text)


    @classmethod
    def get_videoList_selenium(cls):
        # 打印请求配置
        ca = DesiredCapabilities.CHROME
        ca["goog:loggingPrefs"] = {"performance": "ALL"}
        # # 不打开浏览器运行
        chrome_options = webdriver.ChromeOptions()
        chrome_options.add_argument(
            f'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36')
        # chrome_options.add_argument("--headless")
        chrome_options.add_argument("--window-size=1920,1080")
        # chrome_options.add_argument("--no-sandbox")
        chromedriver = "/Users/wangkun/Downloads/chromedriver/chromedriver_v114/chromedriver"
        # driver初始化
        driver = webdriver.Chrome(desired_capabilities=ca, options=chrome_options, service=Service(chromedriver))
        driver.implicitly_wait(10)
        print("打开搜索页：健康")
        driver.get(f"https://www.ixigua.com/search/健康/")
        time.sleep(3)
        # logs = driver.get_log("performance")
        print("关闭登录弹框")
        if driver.find_elements(By.XPATH, '//*[@class="xg-notification-close"]') != 0:
            driver.find_element(By.XPATH, '//*[@class="xg-notification-close"]').click()
        driver.get_screenshot_as_file("./关闭弹框.png")
        print("点击筛选按钮")
        driver.find_element(By.XPATH, '//*[@class="searchPageV2__header-icons-categories"]').click()
        print("点击最新排序")
        driver.find_element(By.XPATH, '//*[@class="searchPageV2__header-categories-wrapper"]/*[1]/*[2]/*[1]').click()
        time.sleep(3)
        driver.get_screenshot_as_file("./最新排序.png")


        driver.quit()


if __name__ == "__main__":
    # SearchDev.get_videoList_requests()
    SearchDev.get_videoList_selenium()