toutiao.py 4.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596
  1. """
  2. @author: luojunhui
  3. """
  4. import urllib.parse
  5. import requests
  6. from fake_useragent import FakeUserAgent
  7. from lxml import etree
  8. def tunnel_proxies():
  9. """
  10. 快代理方法
  11. :return:
  12. """
  13. tunnel = "q796.kdltps.com:15818"
  14. username = "t17772369458618"
  15. password = "5zqcjkmy"
  16. proxies = {
  17. "http": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": username, "pwd": password, "proxy": tunnel},
  18. "https": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": username, "pwd": password, "proxy": tunnel}
  19. }
  20. return proxies
  21. def search_article(title):
  22. """
  23. 通过标题搜索文章
  24. :param title:
  25. :return:
  26. """
  27. url = "https://so.toutiao.com/search"
  28. params = {
  29. "dvpf": "pc",
  30. "source": "search_subtab_switch",
  31. "keyword": title,
  32. "page_num": 0,
  33. "pd": "information",
  34. "action_type": "search_subtab_switch",
  35. "search_id": "",
  36. "from": "news",
  37. "cur_tab_title": "news"
  38. }
  39. headers = {
  40. 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
  41. 'Accept-Language': 'en,zh;q=0.9,zh-CN;q=0.8',
  42. 'Cache-Control': 'max-age=0',
  43. 'Connection': 'keep-alive',
  44. 'Cookie': '__ac_nonce=06656db2c0044df592020; __ac_signature=_02B4Z6wo00f01f7SgiAAAIDA3giITC7VZDn-8oaAABnk69; msToken=2BrMcxLg3_PsS1iKcjuLpU1GS9iZsZ-51KSQTgUSRRLhGQqsQV3zKuJR49smQ7f8hQ8fahWtYCKC6TKJO3kR8022S-NsNfdHXu7X7mPM; ttwid=1%7C9b5sTIuwZxZKt0wFsvE-2t5OoFxH_Q5VIpVNWEREbAo%7C1716968863%7C3459387c0d736a3410ba2c5dbdaeb61a7f85f161bbf6cb1a4f646c2a2edc9aa5;',
  45. 'Sec-Fetch-Dest': 'document',
  46. 'Sec-Fetch-Mode': 'navigate',
  47. 'Sec-Fetch-Site': 'none',
  48. 'Sec-Fetch-User': '?1',
  49. 'Upgrade-Insecure-Requests': '1',
  50. 'User-Agent': FakeUserAgent().random,
  51. }
  52. response = requests.get(url=url, headers=headers, params=params)
  53. html = etree.HTML(response.text)
  54. xpath = '//div[@class="cs-view cs-view-block cs-card-content"]//a/@href'
  55. result_list = html.xpath(xpath)
  56. urls = [urllib.parse.unquote(i.replace("/search/jump?url=", "")) for i in result_list]
  57. article_urls = [url.replace("http://www.toutiao.com/a", "https://www.toutiao.com/article/") for url in urls]
  58. return article_urls
  59. def parse_detail(url):
  60. """
  61. :param url:
  62. :return:
  63. """
  64. payload = {}
  65. headers = {
  66. 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
  67. 'accept-language': 'en',
  68. 'cookie': '__ac_signature=_02B4Z6wo00f01uL1Y5QAAIDDwi9p-RyULmbi1WcAAN8B5b; tt_webid=7371293454351697471; ttcid=a7499fc4f17243e1a6f1d47fc054799e16; _ga=GA1.1.1771235425.1716434457; s_v_web_id=verify_lwiorbpi_iEdehlbX_70wd_4ldg_Bctq_iI8TdJIzXVnI; csrftoken=ee756af695a449eeb73b5a3fc78978b2; _S_DPR=2.200000047683716; _S_IPAD=0; notRedShot=1; local_city_cache=%E5%8C%97%E4%BA%AC; gfkadpd=24,6457; _S_WIN_WH=1336_726; ttwid=1%7C9b5sTIuwZxZKt0wFsvE-2t5OoFxH_Q5VIpVNWEREbAo%7C1717060308%7C4203558278ecaad696c5c5d3ff2bbab682c3ae0f89b876ab618b4646f04c81ef; tt_scid=GNVu.6vOxpT8JE14FSVyGcUl7IItTWcIP.qGQ.gRujo02xjV1BqV1tKznH7NlULMc251; msToken=dG4Y0x3XtlJGBx5WEqYilfjCRWJq629eYRg-hLWdfiTavnI5szx-h9KwUUAggGcG2i03AksGmCfQ034JCbnfWnTnhuguXU1yISZMr6YE; _ga_QEHZPBE5HH=GS1.1.1717057829.7.1.1717060394.0.0.0',
  69. 'priority': 'u=0, i',
  70. 'referer': url,
  71. 'upgrade-insecure-requests': '1',
  72. 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36'
  73. }
  74. response = requests.request("GET", url, headers=headers, data=payload, proxies=tunnel_proxies())
  75. html = etree.HTML(response.text)
  76. text_xpath = '//div[@class="article-content"]//p/text()'
  77. img_xpath = '//div[@class="pgc-img"]/img/@src'
  78. title_xpath = '//div[@class="article-content"]/h1/text()'
  79. result_list = html.xpath(text_xpath)
  80. img_list = html.xpath(img_xpath)
  81. title = html.xpath(title_xpath)
  82. obj = {
  83. "article_url": url,
  84. "text": "\n".join(result_list),
  85. "img_urls": img_list,
  86. "title": title
  87. }
  88. return obj