server-deploy/longvideo_remove_failed_instances.py

import requests
import utils
import longvideo_config
import logging

logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
                    datefmt='%a, %d %b %Y %H:%M:%S')


def get_failed_sever_instances(slb_client, instance_ids):
    """获取探活失败的机器"""
    failed_instances = []
    for instance_id in instance_ids:
        ip_address = utils.get_ip_address(client=slb_client, instance_id=instance_id)
        health_check_url = f"http://{ip_address}:8080/longvideoapi/test"
        try:
            http_code = requests.get(health_check_url).status_code
        except:
            failed_instances.append(instance_id)
            logging.info(f"failed instances: {instance_id}/{ip_address}")
            continue

        if http_code == 200:
            continue
        else:
            failed_instances.append(instance_id)
            logging.info(f"failed instances: {instance_id}/{ip_address}")
    return failed_instances


def main():
    """移除服务挂掉的机器"""
    slb_client = utils.connect_client(access_key_id=longvideo_config.slb_client_params['access_key_id'],
                                      access_key_secret=longvideo_config.slb_client_params['access_key_secret'],
                                      region_id=longvideo_config.slb_client_params['region_id'])

    # 1. 获取slb下所有机器
    online_instance_ids = utils.get_instance_ids(client=slb_client, slb_id=longvideo_config.slb_id_list[0])
    online_instance_count = len(online_instance_ids)
    logging.info(f"online instance count: {online_instance_count}.")
    logging.info(f"online instance ids: {online_instance_ids}")
    # 2. 获取探活失败的机器
    failed_instances = get_failed_sever_instances(slb_client=slb_client, instance_ids=online_instance_ids)
    logging.info(f"failed instances count: {len(failed_instances)}")
    logging.info(f"failed instance ids: {failed_instances}")

    # 3. 摘流量
    if len(failed_instances) > 0:
        utils.set_instance_weight_process_with_slbs(client=slb_client,
                                                    slb_id_list=longvideo_config.slb_id_list,
                                                    instance_id_list=failed_instances,
                                                    weight_list=[(0, 15)])
        logging.info(f"set weight with 0 finished.")
    else:
        logging.info("not have failed instances!")


if __name__ == '__main__':
    main()