import asyncio
import sys
import time
import requests
import utils
import logging
import os
import docker
import longvideo_config
from concurrent.futures import ThreadPoolExecutor
health_instances = []
ess_instances = []
remove_container_instances = []
def server_health_check(client, instance_id):
"""
服务健康检查
:param client: 客户端连接
:param instance_id: instanceId
:return:
"""
global health_instances
ip_address = utils.get_ip_address(client=client, instance_id=instance_id)
while True:
health_check_url = f"http://{ip_address}:8080/longvideoapi/test"
try:
http_code = requests.get(health_check_url).status_code
except:
logging.info(f"images is downloading ip：{ip_address}")
http_code = 0
if http_code == 200:
health_instances.append((instance_id, ip_address))
logging.info(f"health check success, instance: {instance_id}/{ip_address}")
break
else:
time.sleep(10)
def remove_container_image(client, instance_id, container_name_list):
"""
移除旧容器并删除旧镜像
:param client: 客户端连接
:param instance_id: instanceId type-string
:param container_name_list: 容器名称 type-list
:return:
"""
ip_address = utils.get_ip_address(client=client, instance_id=instance_id)
logging.info(f"服务器信息：{instance_id}/{ip_address}")
client = docker.DockerClient(base_url=f'tcp://{ip_address}:2375', timeout=60)
# 移除旧的容器
container_remove_retry = 3
i = 0
while True:
if i >= container_remove_retry:
logging.error(f"容器不存在或者无法删除当前容器, instance = {instance_id}/{ip_address}")
sys.exit()
try:
flag = False
for container_name in container_name_list:
try:
container_id = client.containers.get(container_name)
container_id.remove(force=True)
flag = True
break
except:
continue
if flag:
break
except Exception as e:
i += 1
# 删除旧镜像
images_remove_retry = 3
j = 0
while True:
if j >= images_remove_retry:
logging.error(f"镜像不存在，无法获取到镜像ID, instance = {instance_id}/{ip_address}")
sys.exit()
try:
images = client.images.list()
for image in images:
client.images.remove(force=True, image=image.tags[0])
time.sleep(2)
global remove_container_instances
remove_container_instances.append(instance_id)
break
except Exception as e:
i += 1
def update_instance(create_client, slb_client, instance_id, version):
"""
线上机器更新
:param create_client:
:param slb_client: slb客户端连接
:param instance_id: instanceId
:param version: 版本标记
:return:
"""
logging.info(f"update instance: {instance_id}")
# 1. 摘流量
utils.set_instance_weight_process_with_slbs(client=slb_client,
slb_id_list=longvideo_config.slb_id_list,
instance_id_list=[instance_id],
weight_list=[(0, 15)])
logging.info(f"set weight with 0 finished, instance: {instance_id}")
# 2. 移除旧容器并删除旧镜像
global remove_container_instances
remove_container_instances = []
container_name_list = ['vlogapi', 'longvideoapi']
remove_container_image(slb_client, instance_id, container_name_list)
logging.info(f"remove container & images finished, instance: {remove_container_instances},"
f" count: {len(remove_container_instances)}")
if len(remove_container_instances) == 0:
logging.error(f"remove container image failed|")
sys.exit()
# 3. 发送启动脚本到机器上
utils.send_file_to_ecs(client=create_client, instance_id_list=[instance_id], **longvideo_config.start_sh)
logging.info(f"send start shell file finished, instance: {instance_id}")
# 4. 启动服务
server_start_sh = os.path.join(longvideo_config.start_sh['target_dir'], longvideo_config.start_sh['name'])
server_start_commend = f"sh {server_start_sh} {version}"
utils.run_command(client=create_client, instance_ids=[instance_id], command=server_start_commend)
# 5. 探活
global health_instances
health_instances = []
server_health_check(slb_client, instance_id)
logging.info(f"health instances: {health_instances}, count: {len(health_instances)}")
# 6. 挂载流量
if len(health_instances) == 0:
logging.info(f"health instances: {health_instances}, count: {len(health_instances)}")
sys.exit()
# 机器探活成功
time.sleep(10)
utils.add_backend_servers_with_slbs(client=slb_client,
slb_id_list=longvideo_config.slb_id_list,
instances=health_instances)
health_instance_ids = [instance_id for instance_id, _ in health_instances]
add_weight_list = [(10, 5), (20, 5), (40, 5), (60, 5), (80, 5), (100, 5)]
utils.set_instance_weight_process_with_slbs(client=slb_client,
slb_id_list=longvideo_config.slb_id_list,
instance_id_list=health_instance_ids,
weight_list=add_weight_list)
logging.info(f"finished instances: {health_instances}, count: {len(health_instances)}")
def main():
try:
version = sys.argv[1]
num = 1
if len(sys.argv) > 2:
num = int(sys.argv[2])
slb_client = utils.connect_client(access_key_id=longvideo_config.slb_client_params['access_key_id'],
access_key_secret=longvideo_config.slb_client_params['access_key_secret'],
region_id=longvideo_config.slb_client_params['region_id'])
create_client = utils.connect_client(access_key_id=longvideo_config.create_client_params['access_key_id'],
access_key_secret=longvideo_config.create_client_params['access_key_secret'],
region_id=longvideo_config.create_client_params['region_id'])
# 1. 获取slb下所有机器
online_instance_ids = utils.get_instance_ids(client=slb_client, slb_id=longvideo_config.slb_id_list[0])
online_instance_count = len(online_instance_ids)
logging.info(f"online instance count: {online_instance_count}.")
logging.info(f"online instance ids: {online_instance_ids}")
# 2. 对其中一部分机器进行更新
instance_ids = online_instance_ids[:num]
print(len(instance_ids))
logging.info(f"update {num} instances start ...")
for instance_id in instance_ids:
update_instance(create_client=create_client, slb_client=slb_client, instance_id=instance_id, version=version)
logging.info(f"update {num} instances end!")
except Exception as e:
logging.error(e)
sys.exit()
if __name__ == '__main__':
main()