update

main/demo.py

@@ -3,94 +3,213 @@
 import datetime
 import logging
 import os
+import sys
 import time
-
+sys.path.append(os.getcwd())
 from main.feishu_lib import Feishu
-
+from main.publish import Publish
+from main.common import Common
 path = "../videoinfo/"
 
 
-def read_videoname():
-    with open(path + "videoname.txt", "r", encoding="utf8") as f:
-        content = f.read()
-        name = content
-        return name
-
-
-def write_videoname():
-    n = int(float(read_videoname()))
-    with open(path + "videoname.txt", "w", encoding="utf8") as f:
-        f.write(str(n + 1))
-
-
-def split_text():
-    path1 = "./chlsfiles/"
-    files = os.listdir(path1)
-    for file in files:
-        print(file)
-        print("\n==========\n")
-        a = os.path.splitext(file)
-        print(a[-1])
-
-        # print(file[0])
-        # print(file[-1])
-
-
-def split_query():
-    url = "encfilekey=Cvvj5Ix3eez3Y79SxtvVL0L7CkPM6dFibFeI6caGYwFHPibicjZB7UrwpKsG9wQrl01" \
-          "IlTMtIjicibib9iaGoaL2sxR7vGt4sofODkBGnic9EOCZjgCQTlO7ZTNjribxkDmPq40E2RMdXxzGlWDwYfmkWz4Q15g" \
-          "&token=AxricY7RBHdVbVKZOkvCOWwYWwj9zBqet1eia6GZib8GhmdrytdBu5HcPARtxW7l0AAzppsekDUXXQ&idx=1&adap" \
-          "tivelytrans=943&bizid=1023&dotrans=2991&hy=SH&m=d04052b0ab0e3dd64567fa60b16d0898&scene=0&t=1&taskid=0"
-    a = url.split("encfilekey=")[-1].split("&token=")[0]
-    print(a)
-
-
-def distent():
-    filekey = "Cvvj5Ix3eez3Y79SxtvVL0L7CkPM6dFibFeI6caGYwFEiaKINib2TA0ibV5JuXfIA5JysRWeytU3kTufCr7KFPLgwMiauCic" \
-              "UypoEbA31w7w3sibnraPEwcMcG6LqxSmdFN5S8AfhuibFicv86oBfSuSXcibrCqw"
-    filekey_txt_path = "../videoinfo/filekey.txt"
-
-    with open(filekey_txt_path, 'r', encoding="utf8") as f:
-        content = f.read()
-        print(content)
-        for line in content:
-            if filekey in line.strip():
-                print("yes")
-            elif line.strip() == "":
-                print("null")
+class Demo:
+
+    @classmethod
+    def read_videoname(cls):
+        with open(path + "videoname.txt", "r", encoding="utf8") as f:
+            content = f.read()
+            name = content
+            return name
+
+    @classmethod
+    def write_videoname(cls):
+        n = int(float(cls.read_videoname()))
+        with open(path + "videoname.txt", "w", encoding="utf8") as f:
+            f.write(str(n + 1))
+
+    @classmethod
+    def split_text(cls):
+        path1 = "./chlsfiles/"
+        files = os.listdir(path1)
+        for file in files:
+            print(file)
+            print("\n==========\n")
+            a = os.path.splitext(file)
+            print(a[-1])
+
+            # print(file[0])
+            # print(file[-1])
+
+    @classmethod
+    def split_query(cls):
+        url = "encfilekey=Cvvj5Ix3eez3Y79SxtvVL0L7CkPM6dFibFeI6caGYwFHPibicjZB7UrwpKsG9wQrl01" \
+              "IlTMtIjicibib9iaGoaL2sxR7vGt4sofODkBGnic9EOCZjgCQTlO7ZTNjribxkDmPq40E2RMdXxzGlWDwYfmkWz4Q15g" \
+              "&token=AxricY7RBHdVbVKZOkvCOWwYWwj9zBqet1eia6GZib8GhmdrytdBu5HcPARtxW7l0AAzppsekDUXXQ&idx=1&adap" \
+              "tivelytrans=943&bizid=1023&dotrans=2991&hy=SH&m=d04052b0ab0e3dd64567fa60b16d0898&scene=0&t=1&taskid=0"
+        a = url.split("encfilekey=")[-1].split("&token=")[0]
+        print(a)
+
+    @classmethod
+    def distent(cls):
+        filekey = "Cvvj5Ix3eez3Y79SxtvVL0L7CkPM6dFibFeI6caGYwFEiaKINib2TA0ibV5JuXfIA5JysRWeytU3kTufCr7KFPLgwMiauCic" \
+                  "UypoEbA31w7w3sibnraPEwcMcG6LqxSmdFN5S8AfhuibFicv86oBfSuSXcibrCqw"
+        filekey_txt_path = "../videoinfo/filekey.txt"
+
+        with open(filekey_txt_path, 'r', encoding="utf8") as f:
+            content = f.read()
+            print(content)
+            for line in content:
+                if filekey in line.strip():
+                    print("yes")
+                elif line.strip() == "":
+                    print("null")
+                else:
+                    print("no")
+
+    # 创建日志
+    @classmethod
+    def create_log(cls):
+        # 创建日志文件夹。如果不存在，则创建
+        log_dir = "../logs/"  # 日志文件夹名称
+        log_path = os.getcwd() + os.sep + log_dir  # 日志路径
+        if not os.path.isdir(log_path):
+            os.makedirs(log_path)
+
+        log_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+        date_format = "%Y-%m-%d %p %H:%M:%S"
+        # 初始化
+        logging.basicConfig(filename=log_path + 'logs.logs', level=logging.INFO, format=log_format, datefmt=date_format)
+
+        logger = logging.getLogger('shipinhao')
+
+        logger.info("这是第一条日志")
+
+    @classmethod
+    def split_title(cls):
+        title = Feishu.get_values_batch('demo', 'shipinhao', 'FSDlBy')[3][2]
+        print(title)
+        print('\n')
+        # print(title.split('\n')[0].split('#')[0])
+        print(title.split('\n')[0].replace('#', ''))
+
+    # 下载 、上传
+    @classmethod
+    def download_publish(cls, log_type, env):
+        try:
+            recommend_feeds_sheet = Feishu.get_values_batch(log_type, 'shipinhao', 'FSDlBy')
+            for i in range(1, len(recommend_feeds_sheet)):
+                download_title = recommend_feeds_sheet[i][2].strip().replace('"', '') \
+                    .replace('“', '').replace('“', '…').replace("\n", "") \
+                    .replace("/", "").replace("\r", "").replace("#", "") \
+                    .replace(".", "。").replace("\\", "").replace("&NBSP", "") \
+                    .replace(":", "").replace("*", "").replace("？", "") \
+                    .replace("?", "").replace('"', "").replace("<", "") \
+                    .replace(">", "").replace("|", "").replace(" ", "")
+                download_duration = recommend_feeds_sheet[i][3]
+                download_like_cnt = recommend_feeds_sheet[i][4]
+                download_share_cnt = recommend_feeds_sheet[i][5]
+                download_favorite_cnt = recommend_feeds_sheet[i][6]
+                download_comment_cnt = recommend_feeds_sheet[i][7]
+                download_username = recommend_feeds_sheet[i][8]
+                download_head_url = recommend_feeds_sheet[i][9]
+                download_cover_url = recommend_feeds_sheet[i][10]
+                download_video_url = recommend_feeds_sheet[i][11]
+
+                Common.logger(log_type).info("download_title:{}", download_title)
+                Common.logger(log_type).info("download_username:{}", download_username)
+                Common.logger(log_type).info("download_video_url:{}", download_video_url)
+
+                if download_title is None or download_duration is None or download_video_url is None:
+                    Feishu.dimension_range(log_type, 'shipinhao', 'FSDlBy', 'ROWS', i + 1, i + 1)
+                    Common.logger(log_type).info('空行，删除成功\n')
+                    return
+                elif str(download_title) in [x for y in Feishu.get_values_batch(log_type, 'shipinhao', 'c77cf9') for x
+                                             in y]:
+                    Feishu.dimension_range(log_type, 'shipinhao', 'FSDlBy', 'ROWS', i + 1, i + 1)
+                    Common.logger(log_type).info('视频已下载，删除成功\n')
+                    return
+                else:
+                    # 下载封面
+                    Common.download_method(log_type=log_type, text="cover",
+                                           d_name=str(download_title), d_url=str(download_cover_url))
+                    # 下载视频
+                    Common.download_method(log_type=log_type, text="video",
+                                           d_name=str(download_title), d_url=str(download_video_url))
+                    # 保存视频信息至 "./videos/{download_video_title}/info.txt"
+                    with open("./videos/" + download_title
+                              + "/" + "info.txt", "a", encoding="UTF-8") as f_a:
+                        f_a.write('shipinhao' + str(int(time.time())) + "\n" +
+                                  str(download_title) + "\n" +
+                                  str(download_duration) + "\n" +
+                                  str(download_favorite_cnt) + "\n" +
+                                  str(download_comment_cnt) + "\n" +
+                                  str(download_like_cnt) + "\n" +
+                                  str(download_share_cnt) + "\n" +
+                                  str(1920 * 1080) + "\n" +
+                                  str(int(time.time())) + "\n" +
+                                  str(download_username) + "\n" +
+                                  str(download_head_url) + "\n" +
+                                  str(download_video_url) + "\n" +
+                                  str(download_cover_url) + "\n" +
+                                  "shipinhao"+str(int(time.time())))
+                    Common.logger(log_type).info("==========视频信息已保存至info.txt==========")
+
+                    Common.logger(log_type).info("开始上传视频:{}".format(download_title))
+                    our_video_id = Publish.upload_and_publish(log_type, env, "play")
+                    if env == 'dev':
+                        our_video_link = "https://testadmin.piaoquantv.com/cms/post-detail/" + str(our_video_id) + "/info"
+                    else:
+                        our_video_link = "https://admin.piaoquantv.com/cms/post-detail/" + str(our_video_id) + "/info"
+                    Common.logger(log_type).info("视频上传完成:{}", our_video_link)
+
+                    # # 视频ID工作表，插入首行
+                    # Feishu.insert_columns(log_type, "shipinhao", "c77cf9", "ROWS", 1, 2)
+                    # # 视频ID工作表，首行写入数据
+                    # upload_time = int(time.time())
+                    # values = [[time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(upload_time)),
+                    #            "话题榜",
+                    #            str(download_title),
+                    #            our_video_link,
+                    #            download_duration,
+                    #            download_like_cnt,
+                    #            download_share_cnt,
+                    #            download_favorite_cnt,
+                    #            download_comment_cnt,
+                    #            download_username,
+                    #            str(download_head_url),
+                    #            str(download_cover_url),
+                    #            str(download_video_url)]]
+                    # time.sleep(1)
+                    # Feishu.update_values(log_type, "shipinhao", "c77cf9", "F2:V2", values)
+
+                    # 删除行或列，可选 ROWS、COLUMNS
+                    time.sleep(1)
+                    Feishu.dimension_range(log_type, "shipinhao", "FSDlBy", "ROWS", i + 1, i + 1)
+                    Common.logger(log_type).info("下载/上传成功:{}\n", download_title)
+                    return
+        except Exception as e:
+            Feishu.dimension_range(log_type, "shipinhao", "FSDlBy", "ROWS", 2, 2)
+            Common.logger(log_type).error('download_publish异常，删除视频信息成功:{}\n', e)
+
+    @classmethod
+    def run_download_publish(cls, log_type, env):
+        while True:
+            feeds_sht = Feishu.get_values_batch(log_type, 'shipinhao', 'FSDlBy')
+            if len(feeds_sht) == 1:
+                Common.logger(log_type).info('下载/上传完成\n')
+                break
             else:
-                print("no")
-
-
-# 创建日志
-def create_log():
-    # 创建日志文件夹。如果不存在，则创建
-    log_dir = "../logs/"  # 日志文件夹名称
-    log_path = os.getcwd() + os.sep + log_dir  # 日志路径
-    if not os.path.isdir(log_path):
-        os.makedirs(log_path)
-
-    log_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-    date_format = "%Y-%m-%d %p %H:%M:%S"
-    # 初始化
-    logging.basicConfig(filename=log_path + 'logs.logs', level=logging.INFO, format=log_format, datefmt=date_format)
-
-    logger = logging.getLogger('shipinhao')
-
-    logger.info("这是第一条日志")
-
+                cls.download_publish(log_type, env)
 
-def split_title():
-    title = Feishu.get_values_batch('demo', 'shipinhao', 'FSDlBy')[3][2]
-    print(title)
-    print('\n')
-    # print(title.split('\n')[0].split('#')[0])
-    print(title.split('\n')[0].replace('#', ''))
+    @classmethod
+    def get_sht(cls):
+        sht = Feishu.get_values_batch('recommend', 'shipinhao', 'c77cf9')
+        print(len(sht))
 
 
 if __name__ == "__main__":
-    # print(float(str(1.2)))
-    # split_title()
-    print(str(datetime.datetime.now()).split('.')[0])
-    print(time.time())
+    # print(str(datetime.datetime.now()).split('.')[0])
+    # print(time.time())
+    # Demo.run_download_publish('topic', 'prod')
+    Demo.get_sht()
     pass

main/feishu_lib.py

@@ -309,6 +309,6 @@ class Feishu:
 
 
 if __name__ == "__main__":
-    print(Feishu.get_token('gzh'))
+    print(Feishu.get_token('shipinhao'))
 
     pass

main/shipinhao_get_url.py

@@ -6,31 +6,31 @@ import os
 import sys
 import time
 import psutil as psutil
-# import atomacos
 from appium import webdriver
 from selenium.webdriver.common.by import By
 sys.path.append(os.getcwd())
-# from crawler_shipinhao.main.common import Common
-# from crawler_shipinhao.main.feishu_lib import Feishu
 from main.feishu_lib import Feishu
 from main.common import Common
 
 
 class GetUrl:
     @classmethod
-    def kill_pid(cls):
-        os.system('chcp 65001')  # 将cmd的显示字符编码从默认的GBK改为UTF-8
-        list_process = list()
-        pid_list = psutil.pids()
-        for sub_pid in pid_list:
-            try:
-                process_info = psutil.Process(sub_pid)
-                if process_info.name() == 'WechatBrowser.exe' or process_info.name() == 'WeChatPlayer.exe':
-                    list_process.append(sub_pid)
-            except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
-                pass
-        for pid in list_process:
-            os.system('taskkill /f /pid ' + str(pid))
+    def kill_pid(cls, log_type):
+        try:
+            os.system('chcp 65001')  # 将cmd的显示字符编码从默认的GBK改为UTF-8
+            list_process = list()
+            pid_list = psutil.pids()
+            for sub_pid in pid_list:
+                try:
+                    process_info = psutil.Process(sub_pid)
+                    if process_info.name() == 'WechatBrowser.exe' or process_info.name() == 'WeChatPlayer.exe':
+                        list_process.append(sub_pid)
+                except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
+                    pass
+            for pid in list_process:
+                os.system('taskkill /f /pid ' + str(pid))
+        except Exception as e:
+            Common.logger(log_type).error('kill_pid异常:{}', e)
 
     @classmethod
     def click_video(cls, log_type):
@@ -111,60 +111,60 @@ class GetUrl:
                     return '未找到url'
 
         except Exception as e:
-            Common.logger(log_type).exception("get_url异常：{}", e)
+            Common.logger(log_type).exception("get_url异常：{}\n", e)
             return None
 
     @classmethod
     def write_url(cls, log_type):
-        # try:
-        while True:
-            if Feishu.get_values_batch(log_type, 'shipinhao', 'FSDlBy')[1][11] is None:
-                Common.del_charles_files('recommend')
-                cls.click_video(log_type)
-                Common.logger(log_type).info('等待 2s')
-                time.sleep(2)
-                Common.logger(log_type).info('获取视频头像/封面/播放地址')
-                # print('获取视频头像/封面/播放地址')
-                urls = cls.get_url(log_type)
-                if urls == '未找到url':
-                    time.sleep(1)
-                    cls.write_url(log_type)
-                elif urls is None:
-                    time.sleep(1)
-                    cls.write_url(log_type)
-                else:
-                    Feishu.update_values(log_type, 'shipinhao', 'FSDlBy', 'J2:L2',
-                                         [['https://'+urls[2], 'https://'+urls[1], 'https://'+urls[0]]])
-                    Common.logger(log_type).info('视频地址信息写入飞书成功\n')
+        try:
+            while True:
+                if Feishu.get_values_batch(log_type, 'shipinhao', 'FSDlBy')[1][11] is None:
                     Common.del_charles_files('recommend')
+                    cls.click_video(log_type)
+                    Common.logger(log_type).info('等待 2s')
+                    time.sleep(2)
+                    Common.logger(log_type).info('获取视频头像/封面/播放地址')
+                    # print('获取视频头像/封面/播放地址')
+                    urls = cls.get_url(log_type)
+                    if urls == '未找到url':
+                        time.sleep(1)
+                        cls.write_url(log_type)
+                    elif urls is None:
+                        time.sleep(1)
+                        cls.write_url(log_type)
+                    else:
+                        Feishu.update_values(log_type, 'shipinhao', 'FSDlBy', 'J2:L2',
+                                             [['https://'+urls[2], 'https://'+urls[1], 'https://'+urls[0]]])
+                        Common.logger(log_type).info('视频地址信息写入飞书成功\n')
+                        Common.del_charles_files('recommend')
+                        break
+                else:
+                    Common.logger(log_type).info('视频已有地址信息，休眠 10s')
+                    time.sleep(10)
                     break
-            else:
-                Common.logger(log_type).info('视频已有地址信息，休眠 10s')
-                time.sleep(10)
-                break
-        # except Exception as e:
-        #     # Feishu.dimension_range(log_type, 'shipinhao', 'FSDlBy', 'ROWS', 2, 2)
-        #     Common.logger(log_type).error('write_url异常:{}', e)
+        except Exception as e:
+            # Feishu.dimension_range(log_type, 'shipinhao', 'FSDlBy', 'ROWS', 2, 2)
+            Common.logger(log_type).error('write_url异常:{}\n', e)
 
     @classmethod
     def run_get_url(cls, log_type):
-        # try:
-        while True:
-            if len(Feishu.get_values_batch(log_type, 'shipinhao', 'FSDlBy')) == 1:
-                Common.logger(log_type).info('暂无需要获取地址的视频信息')
-                time.sleep(30)
-                break
-            else:
-                cls.write_url(log_type)
+        try:
+            while True:
+                if len(Feishu.get_values_batch(log_type, 'shipinhao', 'FSDlBy')) == 1:
+                    Common.logger(log_type).info('暂无需要获取地址的视频信息')
+                    time.sleep(30)
+                    break
+                else:
+                    cls.write_url(log_type)
 
-        # except Exception as e:
-        #     Common.logger(log_type).error('run_get_url异常:{}', e)
+        except Exception as e:
+            Common.logger(log_type).error('run_get_url异常:{}\n', e)
 
 
 if __name__ == '__main__':
-    # GetUrl.write_url('recommend')
-    # print(len(Feishu.get_values_batch('recommend', 'shipinhao', 'FSDlBy')))
     while True:
         GetUrl.run_get_url('recommend')
         Common.del_logs('recommend')
         time.sleep(1)
+
+    pass

main/shipinhao_recommend.py

@@ -36,7 +36,6 @@ class Recommend:
                 "automationName": "UiAutomator2"  # 使用引擎，默认为 Appium，
                 # 其中 Appium、UiAutomator2、Selendroid、Espresso 用于 Android，XCUITest 用于 iOS
             }
-            # global driver
             driver = webdriver.Remote("http://localhost:4723/wd/hub", caps)
             driver.implicitly_wait(10)