修改select方法

applications/longArticlesMysql.py

@@ -31,13 +31,14 @@ class longArticlesMySQL(object):
         return affected_rows
 
     @classmethod
-    def select(cls, sql):
+    def select(cls, sql, cursor_type=None):
         """
         查询
+        :param cursor_type: 查询出结果的类型，默认None
         :param sql:
         :return:
         """
-        cursor = cls.connection.cursor()
+        cursor = cls.connection.cursor(cursor_type)
         cursor.execute(sql)
         result = cursor.fetchall()
         return result
@@ -60,15 +61,3 @@ class longArticlesMySQL(object):
             cls.connection.rollback()
             raise e
 
-    @classmethod
-    def select_json(cls, sql):
-        """
-        查询
-        :param sql:
-        :return:
-        """
-        cursor = cls.connection.cursor(DictCursor)
-        cursor.execute(sql)
-        result = cursor.fetchall()
-        return result
-

coldStartTasks/crawler/weixin_account_crawler.py

@@ -6,6 +6,7 @@ import traceback
 from typing import List, Set, Dict, Tuple
 
 from tqdm import tqdm
+from pymysql.cursors import DictCursor
 
 from applications import WeixinSpider, longArticlesMySQL, log, bot
 from applications.const import WeixinVideoCrawlerConst
@@ -31,7 +32,7 @@ class WeixinAccountCrawler(object):
         :return:
         """
         sql = "select distinct account_name from datastat_sort_strategy;"
-        account_name_list = self.db_client.select_json(sql)
+        account_name_list = self.db_client.select(sql, cursor_type=DictCursor)
         account_name_set = set()
         for account_name_obj in account_name_list:
             account_name_set.add(account_name_obj['account_name'])
@@ -47,7 +48,7 @@ class WeixinAccountCrawler(object):
             FROM publish_single_video_source
             WHERE source_account = {const.NEED_SCAN_SOURCE_ACCOUNT};
         """
-        article_url_list = self.db_client.select_json(sql)
+        article_url_list = self.db_client.select(sql, cursor_type=DictCursor)
         return article_url_list
 
     def update_crawler_article_status(self, article_id_tuple: Tuple[int, ...]) -> int:
@@ -74,7 +75,7 @@ class WeixinAccountCrawler(object):
             WHERE read_rate > {const.READ_AVG_MULTIPLE} and view_count > {const.MIN_READ_COUNT} and publish_timestamp > {publish_timestamp_threshold};
             ORDER BY read_rate DESC;
         """
-        title_list = self.db_client.select_json(sql)
+        title_list = self.db_client.select(sql, cursor_type=DictCursor)
         title_list = [i['title'] for i in title_list]
         return title_list
 

coldStartTasks/crawler/weixin_video_crawler.py

@@ -8,6 +8,7 @@ import traceback
 from typing import List, Dict
 
 from tqdm import tqdm
+from pymysql.cursors import DictCursor
 
 from applications import bot
 from applications import log
@@ -60,7 +61,7 @@ class WeixinVideoCrawler(object):
             FROM weixin_account_for_videos
             WHERE status = {const.ACCOUNT_CRAWL_STATUS};
         """
-        response = self.db_client.select_json(select_sql)
+        response = self.db_client.select(select_sql, DictCursor)
         return response
 
     def crawler_article_video_list(self, account_obj: Dict, cursor=None):