|
@@ -50,6 +50,7 @@ async def consume_single_message(spider):
|
|
|
consumer = get_consumer(topic, group)
|
|
consumer = get_consumer(topic, group)
|
|
|
try:
|
|
try:
|
|
|
messages = consumer.consume_message(wait_seconds=10, batch_size=1)
|
|
messages = consumer.consume_message(wait_seconds=10, batch_size=1)
|
|
|
|
|
+ print(messages)
|
|
|
if messages:
|
|
if messages:
|
|
|
# 在这里消费消息，做一些数据处理分析
|
|
# 在这里消费消息，做一些数据处理分析
|
|
|
for single_message in messages:
|
|
for single_message in messages:
|
|
@@ -88,7 +89,7 @@ async def main():
|
|
|
"""
|
|
"""
|
|
|
spider_list = TopicGroup().produce()
|
|
spider_list = TopicGroup().produce()
|
|
|
while spider_list:
|
|
while spider_list:
|
|
|
- print(spider_list)
|
|
|
|
|
|
|
+ # print(spider_list)
|
|
|
tasks = [consume_single_message(spider) for spider in spider_list]
|
|
tasks = [consume_single_message(spider) for spider in spider_list]
|
|
|
await asyncio.gather(*tasks)
|
|
await asyncio.gather(*tasks)
|
|
|
await asyncio.sleep(20)
|
|
await asyncio.sleep(20)
|
