piaoquan_crawler/gongzhonghao/gongzhonghao_main/run_test.py

# -*- coding: utf-8 -*-
# @Author: wangkun
# @Time: 2023/6/30
import datetime
from multiprocessing import Process


def get_author_videos(user_list):
    print(f"{datetime.datetime.now()}:{user_list}")


def test_list():
    user_list = ["a", "b", "c", "d", "e", "f", 1]
    user_num = len(user_list)
    chunk_size = 2  # 每个进程处理的用户数量
    crawler_num = int(user_num // chunk_size)  # 向下取整
    if user_num % chunk_size != 0:
        crawler_num += 1
    print(f"crawler_num:{crawler_num}")

    processes = []
    for i in range(crawler_num):
        start = i * chunk_size
        end = min((i + 1) * chunk_size, user_num + 1)
        process = Process(target=get_author_videos, args=(user_list[start:end],))
        process.start()
        processes.append(process)

    for process in processes:
        process.join()

if __name__ == "__main__":
    test_list()
    pass