|
|
@@ -13,7 +13,6 @@ from common.scheduling_db import MysqlHelper
|
|
|
from benshanzhufu.benshanzhufu_recommend.benshanzhufu_recommend_scheduling import BenshanzhufuRecommend
|
|
|
|
|
|
|
|
|
-
|
|
|
def main(log_type, crawler, topic_name, group_id, env):
|
|
|
consumer = get_consumer(topic_name, group_id)
|
|
|
# 长轮询表示如果Topic没有消息，则客户端请求会在服务端挂起3秒，3秒内如果有消息可以消费则立即返回响应。
|
|
|
@@ -30,6 +29,7 @@ def main(log_type, crawler, topic_name, group_id, env):
|
|
|
# 长轮询消费消息。
|
|
|
recv_msgs = consumer.consume_message(batch, wait_seconds)
|
|
|
for msg in recv_msgs:
|
|
|
+ benshanzhufu_start_time = int(time.time())
|
|
|
Common.logger(log_type, crawler).info(f"Receive\n"
|
|
|
f"MessageId:{msg.message_id}\n"
|
|
|
f"MessageBodyMD5:{msg.message_body_md5}\n"
|
|
|
@@ -63,7 +63,11 @@ def main(log_type, crawler, topic_name, group_id, env):
|
|
|
rule_dict=rule_dict,
|
|
|
env=env)
|
|
|
# Common.del_logs(log_type, crawler)
|
|
|
- Common.logger(log_type, crawler).info('抓取一轮结束\n')
|
|
|
+ Common.logger(log_type, crawler).info('抓取一轮结束')
|
|
|
+ benshanzhufu_end_time = int(time.time())
|
|
|
+ benshanzhufu_duration = benshanzhufu_end_time - benshanzhufu_start_time
|
|
|
+ Common.logger(log_type, crawler).info(f"duration {benshanzhufu_duration}\n")
|
|
|
+ Common.logging(log_type, crawler, env, f"duration {benshanzhufu_duration}\n")
|
|
|
|
|
|
except MQExceptionBase as err:
|
|
|
# Topic中没有消息可消费。
|
