|
|
@@ -127,7 +127,7 @@ def h_timer_check():
|
|
|
Timer(60, h_timer_check).start()
|
|
|
else:
|
|
|
log_.info("上游数据就绪，count={}，开始读取数据表".format(table_data_cnt))
|
|
|
- #2 读取数据表
|
|
|
+ #2 读取数据表+区域过滤
|
|
|
data = get_table_data(project, table, partition)
|
|
|
data_new = []
|
|
|
for one in data:
|
|
|
@@ -139,11 +139,11 @@ def h_timer_check():
|
|
|
log_.info("{}被过滤掉了".format(one["region"]))
|
|
|
|
|
|
log_.info("数据处理完成，数据数量={}，所有的地域：".format(len(data_new), ",".join([
|
|
|
- one["region"] for one in data
|
|
|
+ one["region"] for one in data_new
|
|
|
])))
|
|
|
log_.info("开始处理和写入redis")
|
|
|
#3 写入redis
|
|
|
- records_process_for_list(data, process_and_store, max_size=10, num_workers=5)
|
|
|
+ records_process_for_list(data_new, process_and_store, max_size=10, num_workers=5)
|
|
|
|
|
|
except Exception as e:
|
|
|
log_.error(f"4小时地域-趋势统计数据更新失败, exception: {e}, traceback: {traceback.format_exc()}")
|
zhangbo