|
@@ -0,0 +1,301 @@
|
|
|
|
+#!/bin/env python
|
|
|
|
+# coding=utf-8
|
|
|
|
+import logging
|
|
|
|
+import time
|
|
|
|
+import docker
|
|
|
|
+import sys
|
|
|
|
+import requests
|
|
|
|
+import json
|
|
|
|
+import queue
|
|
|
|
+import threading
|
|
|
|
+from aliyunsdkcore import client
|
|
|
|
+from aliyunsdkecs.request.v20140526.DescribeNetworkInterfacesRequest import DescribeNetworkInterfacesRequest
|
|
|
|
+from aliyunsdkcore.request import CommonRequest
|
|
|
|
+from aliyunsdkslb.request.v20140515.DescribeLoadBalancerAttributeRequest import DescribeLoadBalancerAttributeRequest
|
|
|
|
+from concurrent.futures import ThreadPoolExecutor
|
|
|
|
+
|
|
|
|
+logging.basicConfig(level=logging.INFO,
|
|
|
|
+ format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
|
|
|
|
+ datefmt='%a, %d %b %Y %H:%M:%S')
|
|
|
|
+
|
|
|
|
+# ##### 修改负载均衡权限
|
|
|
|
+AccessKey = 'LTAIuPbTPL3LDDKN'
|
|
|
|
+AccessSecret = 'ORcNedKwWuwVtcq4IRFtUDZgS0b1le'
|
|
|
|
+RegionId = 'cn-hangzhou'
|
|
|
|
+clt = client.AcsClient(AccessKey, AccessSecret, RegionId)
|
|
|
|
+
|
|
|
|
+slb_id_1 = 'lb-bp1werfophtsjzfr76njm'
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+def getInstanceId(slb_id):
|
|
|
|
+ """获取slb下所有服务器信息"""
|
|
|
|
+ request = DescribeLoadBalancerAttributeRequest()
|
|
|
|
+ request.set_accept_format('json')
|
|
|
|
+ request.set_LoadBalancerId(slb_id)
|
|
|
|
+ response = clt.do_action_with_exception(request)
|
|
|
|
+ return json.loads(response)
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+def setWeight(slb_id, instance_id, weight):
|
|
|
|
+ """
|
|
|
|
+ 设置slb权重
|
|
|
|
+ :param slb_id: slb_id
|
|
|
|
+ :param instance_id: 服务器id
|
|
|
|
+ :param weight: 权重值
|
|
|
|
+ :return: None
|
|
|
|
+ """
|
|
|
|
+ BackendServers = [{"ServerId": instance_id, "Weight": weight}]
|
|
|
|
+ request = CommonRequest()
|
|
|
|
+ request.set_accept_format('json')
|
|
|
|
+ request.set_domain('slb.aliyuncs.com')
|
|
|
|
+ request.set_version('2014-05-15')
|
|
|
|
+ request.set_method('POST')
|
|
|
|
+ request.set_action_name('SetBackendServers')
|
|
|
|
+ request.add_query_param('BackendServers', BackendServers)
|
|
|
|
+ request.add_query_param('LoadBalancerId', slb_id)
|
|
|
|
+ try:
|
|
|
|
+ response = clt.do_action_with_exception(request)
|
|
|
|
+ except Exception as e:
|
|
|
|
+ logging.error(e)
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+def set_weight_for_more(slb_id, instance_id_list, weight):
|
|
|
|
+ """
|
|
|
|
+ 同时设置多台服务器的slb权重,权重一样
|
|
|
|
+ :param slb_id: slb_id
|
|
|
|
+ :param instance_id_list: 服务器id list
|
|
|
|
+ :param weight: 权重值
|
|
|
|
+ :return: None
|
|
|
|
+ """
|
|
|
|
+ BackendServers = [{"ServerId": instance_id, "Weight": weight} for instance_id in instance_id_list]
|
|
|
|
+ request = CommonRequest()
|
|
|
|
+ request.set_accept_format('json')
|
|
|
|
+ request.set_domain('slb.aliyuncs.com')
|
|
|
|
+ request.set_version('2014-05-15')
|
|
|
|
+ request.set_method('POST')
|
|
|
|
+ request.set_action_name('SetBackendServers')
|
|
|
|
+ request.add_query_param('BackendServers', BackendServers)
|
|
|
|
+ request.add_query_param('LoadBalancerId', slb_id)
|
|
|
|
+ try:
|
|
|
|
+ response = clt.do_action_with_exception(request)
|
|
|
|
+ except Exception as e:
|
|
|
|
+ logging.error(e)
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+def getIpadd(instance_id):
|
|
|
|
+ """获取实例IP地址"""
|
|
|
|
+ request = DescribeNetworkInterfacesRequest()
|
|
|
|
+ request.set_accept_format('json')
|
|
|
|
+ request.set_InstanceId(instance_id)
|
|
|
|
+ response = clt.do_action_with_exception(request)
|
|
|
|
+ request_content = json.loads(response)
|
|
|
|
+ IpAddr = request_content['NetworkInterfaceSets']['NetworkInterfaceSet'][0]['PrivateIpAddress']
|
|
|
|
+ return IpAddr
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+def checkHealth(ipadd):
|
|
|
|
+ """健康检查"""
|
|
|
|
+ while True:
|
|
|
|
+ health_url = 'http://%s:5001/healthcheck' %(ipadd)
|
|
|
|
+ header = {"Content-Type": "application/json"}
|
|
|
|
+ try:
|
|
|
|
+ health_code = requests.get(health_url).status_code
|
|
|
|
+ except Exception as e:
|
|
|
|
+ continue
|
|
|
|
+ if health_code == 200:
|
|
|
|
+ return False
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+def setInstanceWeightProcess(instance_id):
|
|
|
|
+ """服务更新完之后逐步修改服务器的权重值"""
|
|
|
|
+ # 直接加载100会出现502。权重值每次增加20,每5s修改一次
|
|
|
|
+ # for i in range(1, 6):
|
|
|
|
+ # weight = i * 20
|
|
|
|
+ for weight, sleep_time in [(10, 30), (20, 20), (40, 10), (60, 10), (80, 10), (100, 10)]:
|
|
|
|
+ flag = True
|
|
|
|
+ while flag:
|
|
|
|
+ try:
|
|
|
|
+ setWeight(slb_id_1, instance_id, weight)
|
|
|
|
+ time.sleep(sleep_time)
|
|
|
|
+ flag = False
|
|
|
|
+ except Exception as e:
|
|
|
|
+ time.sleep(10)
|
|
|
|
+ continue
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+def set_instance_weight_process(instance_id_list):
|
|
|
|
+ """服务更新完之后逐步修改服务器的权重值"""
|
|
|
|
+ for weight, sleep_time in [(10, 30), (20, 20), (40, 10), (60, 10), (80, 10), (100, 10)]:
|
|
|
|
+ flag = True
|
|
|
|
+ while flag:
|
|
|
|
+ try:
|
|
|
|
+ set_weight_for_more(slb_id=slb_id_1, instance_id_list=instance_id_list, weight=weight)
|
|
|
|
+ time.sleep(sleep_time)
|
|
|
|
+ flag = False
|
|
|
|
+ except Exception as e:
|
|
|
|
+ time.sleep(10)
|
|
|
|
+ continue
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+def update():
|
|
|
|
+ """更新服务"""
|
|
|
|
+ time.sleep(60)
|
|
|
|
+ global success_count, finished_instance_id_list, ipadd_list
|
|
|
|
+ apps = 'rov-server'
|
|
|
|
+ version = sys.argv[1]
|
|
|
|
+ # registry = 'registry-vpc.cn-hangzhou.aliyuncs.com/stuuudy/rov-server:{}'.format(version)
|
|
|
|
+ registry = 'registry.piaoquantv.com/piaoquan/rov-server:{}'.format(version)
|
|
|
|
+ while not q2.empty():
|
|
|
|
+ instance_id = q2.get()
|
|
|
|
+ ipadd = getIpadd(instance_id)
|
|
|
|
+ logging.info(f"服务器信息:{instance_id}/{ipadd}")
|
|
|
|
+ client = docker.DockerClient(base_url=f'tcp://{ipadd}:2375', timeout=60)
|
|
|
|
+ try:
|
|
|
|
+ # 更新前移除旧的容器
|
|
|
|
+ id = client.containers.get(apps)
|
|
|
|
+ id.remove(force=True)
|
|
|
|
+ except Exception as e:
|
|
|
|
+ print("容器不存在或者无法删除当前容器")
|
|
|
|
+
|
|
|
|
+ try:
|
|
|
|
+ # 更新前删除旧的镜像
|
|
|
|
+ images = client.images.list(all)
|
|
|
|
+ for i in range(len(images)):
|
|
|
|
+ if images[i]:
|
|
|
|
+ client.images.remove(force=True, image=images[i].tags[0])
|
|
|
|
+ time.sleep(2)
|
|
|
|
+ # images = client.images.get(registry)
|
|
|
|
+ # client.images.remove(force=True, image=registry)
|
|
|
|
+ except Exception as e:
|
|
|
|
+ print(e)
|
|
|
|
+ print("镜像不存在,无法获取到镜像ID")
|
|
|
|
+ sys.exit(0)
|
|
|
|
+
|
|
|
|
+ try:
|
|
|
|
+ # 登录镜像仓库
|
|
|
|
+ # client.login(username='stuuudys', password='Qingqu@2019', registry='registry-vpc.cn-hangzhou.aliyuncs.com')
|
|
|
|
+ client.login(username='admin', password='Harbor12345', registry='registry.piaoquantv.com')
|
|
|
|
+ # 启动一个容器
|
|
|
|
+ client.containers.run(registry, detach=True, cap_add='SYS_PTRACE', network_mode='host', name=apps,
|
|
|
|
+ volumes={'/datalog/': {'bind': '/datalog/', 'mode': 'rw'}})
|
|
|
|
+ checkHealth(ipadd)
|
|
|
|
+ time.sleep(30)
|
|
|
|
+ finished_instance_id_list.append(instance_id)
|
|
|
|
+ ipadd_list.append(ipadd)
|
|
|
|
+ except Exception as e:
|
|
|
|
+ raise e
|
|
|
|
+ sys.exit() # 容器启动失败立即退出更新
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+class MyThread(threading.Thread):
|
|
|
|
+ def __init__(self, func):
|
|
|
|
+ threading.Thread.__init__(self)
|
|
|
|
+ self.func = func
|
|
|
|
+
|
|
|
|
+ def run(self):
|
|
|
|
+ self.func()
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+def update_sever(instance_id):
|
|
|
|
+ """更新服务"""
|
|
|
|
+ # time.sleep(20)
|
|
|
|
+ # print(slb_id_1, instance_id)
|
|
|
|
+
|
|
|
|
+ # 设置slb权重为0
|
|
|
|
+ setWeight(slb_id_1, instance_id, 0)
|
|
|
|
+ # 等待10s,处理剩余请求
|
|
|
|
+ time.sleep(10)
|
|
|
|
+
|
|
|
|
+ apps = 'rov-server'
|
|
|
|
+ version = sys.argv[1]
|
|
|
|
+ registry = f'registry-vpc.cn-hangzhou.aliyuncs.com/stuuudy/rov-server:{version}'
|
|
|
|
+ ipadd = getIpadd(instance_id)
|
|
|
|
+ logging.info(f"服务器信息:{instance_id}/{ipadd}")
|
|
|
|
+ client = docker.DockerClient(base_url=f'tcp://{ipadd}:2375', timeout=60)
|
|
|
|
+ try:
|
|
|
|
+ # 更新前移除旧的容器
|
|
|
|
+ id = client.containers.get(apps)
|
|
|
|
+ id.remove(force=True)
|
|
|
|
+ except Exception as e:
|
|
|
|
+ print("容器不存在或者无法删除当前容器")
|
|
|
|
+
|
|
|
|
+ try:
|
|
|
|
+ # 更新前删除旧的镜像
|
|
|
|
+ images = client.images.list(all)
|
|
|
|
+ for i in range(len(images)):
|
|
|
|
+ if images[i]:
|
|
|
|
+ client.images.remove(force=True, image=images[i].tags[0])
|
|
|
|
+ time.sleep(2)
|
|
|
|
+ # images = client.images.get(registry)
|
|
|
|
+ # client.images.remove(force=True, image=registry)
|
|
|
|
+ except Exception as e:
|
|
|
|
+ print(e)
|
|
|
|
+ print("镜像不存在,无法获取到镜像ID")
|
|
|
|
+ sys.exit(0)
|
|
|
|
+
|
|
|
|
+ try:
|
|
|
|
+ # 登录镜像仓库
|
|
|
|
+ client.login(username='stuuudys', password='Qingqu@2019', registry='registry-vpc.cn-hangzhou.aliyuncs.com')
|
|
|
|
+ # 启动一个容器
|
|
|
|
+ client.containers.run(registry, detach=True, cap_add='SYS_PTRACE', network_mode='host', name=apps,
|
|
|
|
+ volumes={'/datalog/': {'bind': '/datalog/', 'mode': 'rw'}})
|
|
|
|
+ checkHealth(ipadd)
|
|
|
|
+ print(f"{ipadd}: 权重修改中......")
|
|
|
|
+ # 修改权重
|
|
|
|
+ setInstanceWeightProcess(instance_id)
|
|
|
|
+ global finished_instance_id_list
|
|
|
|
+ finished_instance_id_list.append(instance_id)
|
|
|
|
+ except Exception as e:
|
|
|
|
+ raise e
|
|
|
|
+ sys.exit() # 容器启动失败立即退出更新
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+if __name__ == '__main__':
|
|
|
|
+ # 更新完成计数
|
|
|
|
+ success_count = 0
|
|
|
|
+ threads = []
|
|
|
|
+ # 线程数
|
|
|
|
+ thread_num = 2
|
|
|
|
+ # 获取slb下所有服务器信息
|
|
|
|
+ res = getInstanceId(slb_id_1)
|
|
|
|
+ # slb下服务器总数
|
|
|
|
+ total = len(res["BackendServers"]["BackendServer"])
|
|
|
|
+ q1 = queue.Queue()
|
|
|
|
+ q2 = queue.Queue()
|
|
|
|
+ # 获取slb下所有服务器的 instance_id
|
|
|
|
+ if res["BackendServers"]["BackendServer"]:
|
|
|
|
+ for i in range((len(res["BackendServers"]["BackendServer"]))):
|
|
|
|
+ instance_id = res["BackendServers"]["BackendServer"][i]["ServerId"]
|
|
|
|
+ q1.put(instance_id)
|
|
|
|
+
|
|
|
|
+ while not q1.empty():
|
|
|
|
+ # 摘流量
|
|
|
|
+ for i in range(thread_num):
|
|
|
|
+ if q1.empty():
|
|
|
|
+ break
|
|
|
|
+ instance_id = q1.get()
|
|
|
|
+ # 设置slb权重为0
|
|
|
|
+ setWeight(slb_id_1, instance_id, 0)
|
|
|
|
+ q2.put(instance_id)
|
|
|
|
+ time.sleep(5)
|
|
|
|
+
|
|
|
|
+ # 挂流量
|
|
|
|
+ finished_instance_id_list = []
|
|
|
|
+ ipadd_list = []
|
|
|
|
+ for i in range(thread_num):
|
|
|
|
+ # 开启线程
|
|
|
|
+ thread = MyThread(update)
|
|
|
|
+ thread.start()
|
|
|
|
+ threads.append(thread)
|
|
|
|
+ for thread in threads:
|
|
|
|
+ thread.join()
|
|
|
|
+
|
|
|
|
+ print(f"{ipadd_list}: 权重修改中......")
|
|
|
|
+ # 修改权重
|
|
|
|
+ set_instance_weight_process(instance_id_list=finished_instance_id_list)
|
|
|
|
+ success_count = success_count + thread_num
|
|
|
|
+ print(f"更新进度: {success_count}/{total}")
|
|
|
|
+
|
|
|
|
+ # 权重补充(一秒内同时请求两个服务器修改权重,可能会有一个失败)
|
|
|
|
+ # print(finished_instance_id_list)
|
|
|
|
+ # set_weight_for_more(slb_id=slb_id_1, instance_id_list=finished_instance_id_list, weight=100)
|