From f86c9ffaa892794c815adb986ae9b577ee26b1c0 Mon Sep 17 00:00:00 2001 From: hainingzhang Date: Sun, 11 Mar 2018 10:07:20 +0800 Subject: [PATCH] [CE-297] fix the incompatibility with vSphere host type 1.Update to support vCenter params. 2.Prolong the healcheck sleep time in case of other type of host need more time to create a Fabric cluster. Change-Id: Ib7e53cfc022998d0618b555237481677bc0f8492 Signed-off-by: hainingzhang --- docs/setup_worker_vsphere.md | 2 +- src/agent/vsphere/host.py | 4 +- src/agent/vsphere/host_operations.py | 68 +++++++++++++++++----- src/common/utils.py | 10 +++- src/modules/cluster.py | 13 +---- src/modules/host.py | 85 ++++++++++------------------ src/modules/models/host.py | 1 + 7 files changed, 102 insertions(+), 81 deletions(-) diff --git a/docs/setup_worker_vsphere.md b/docs/setup_worker_vsphere.md index 7a93f47e..91a01aad 100644 --- a/docs/setup_worker_vsphere.md +++ b/docs/setup_worker_vsphere.md @@ -49,7 +49,7 @@ Upload the template OS OVA to vCenter before create vSphere type host in Cello. 1. Login to vSphere Client. 2. Right-Click on ESX host on which you want to deploy template. 3. Select Deploy OVF template. -4. Copy and paste URL for [OVA for Cello](https://drive.google.com/file/d/0B4Ioua6jjCH9b0ROOE14SUlqUk0/view?usp=sharing) +4. Copy and paste URL for [OVA for Cello](https://drive.google.com/file/d/1VN4FDlgd0Q7tqBtGNBN5trP7upaswUWS/view?usp=sharing) 5. Please deploy the ova on the same cluster which will be planned to be used by cello later 6. **Check the name of the VM created** , this will be used to create vSphere type host in Cello later. (Should default to PhotonOSTemplate.ova) diff --git a/src/agent/vsphere/host.py b/src/agent/vsphere/host.py index 3f1b7d57..246e5dd8 100644 --- a/src/agent/vsphere/host.py +++ b/src/agent/vsphere/host.py @@ -29,7 +29,7 @@ class VsphereHost(HostBase): def __init__(self): self.collection = db["host"] - def create(self, vcip, username, pwd, port, params): + def create(self, vcip, username, pwd, port, params, hid): """ Create a new vSphere host :param vcip : vCenter address :param username: vCenter username @@ -131,7 +131,7 @@ def create(self, vcip, username, pwd, port, params): else: vc_resources[NETWORK] = network - operation.create_vm(connection, params) + operation.create_vm(connection, params, hid) return True def delete(self, vmuuid, vcip, username, pwd, port=443): diff --git a/src/agent/vsphere/host_operations.py b/src/agent/vsphere/host_operations.py index 7d28da5b..8feaf0b3 100644 --- a/src/agent/vsphere/host_operations.py +++ b/src/agent/vsphere/host_operations.py @@ -18,6 +18,7 @@ sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..')) from common import log_handler, LOG_LEVEL, db, utils from agent import setup_container_host +from modules.models import Host as HostModel context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) context.verify_mode = ssl.CERT_NONE @@ -46,6 +47,7 @@ def pull_images(self, worker_api): try: self.pull_and_tag_fabric_images(client) self.pull_and_tag_fabric_base_images(client) + self.pull_and_tag_blockchain_explorer_images(client) return True except Exception as e: logger.error('{}'.format(e)) @@ -73,6 +75,26 @@ def pull_and_tag_fabric_base_images(self, docker_client): self.tag_image(docker_client, image, utils.ARCH, utils.BASEIMAGE_RELEASE) + def pull_and_tag_blockchain_explorer_images(self, docker_client): + """ + pull base fabric explorer images, blockchain-explorer, mysql5.7. + :param docker_client: DockerClient object + :return: + """ + try: + image = utils.BLOCKCHAIN_EXPLORER_IMAGE + tag = utils.BLOCKCHAIN_EXPLORER_TAG + logger.info("pulling image: {}".format(image)) + docker_client.images.pull(image, tag) + image = utils.MYSQL_IMAGE + tag = utils.MYSQL_TAG + logger.info("pulling image: {}".format(image)) + docker_client.images.pull(image, tag) + except Exception as e: + logger.error("Docker client error msg: {}".format(e)) + error_msg = "Cannot pull image:{}".format("explorer images") + raise Exception(error_msg) + def pull_image(self, client, image, arch, version): """ pull specific fabric images @@ -82,7 +104,7 @@ def pull_image(self, client, image, arch, version): :param version: fabric image version """ try: - image_to_be_pull = utils.FABRIC_IMAGE_TAG.format(image) + image_to_be_pull = utils.FABRIC_IMAGE.format(image) name = utils.FABRIC_IMAGE_FULL.format(image, arch, version) logger.info("pulling image: {}".format(name)) client.images.pull(image_to_be_pull, tag=arch + '-' + version) @@ -101,7 +123,7 @@ def tag_image(self, client, image, arch, version): """ try: name = utils.FABRIC_IMAGE_FULL.format(image, arch, version) - tag = utils.FABRIC_IMAGE_TAG.format(image) + tag = utils.FABRIC_IMAGE_TAG.format(image, version) image_to_be_tag = client.images.get(name) logger.info("tag {} => {}".format(name, tag)) image_to_be_tag.tag(tag) @@ -172,7 +194,7 @@ def initializesi(self, vcip, username, pwd, port): atexit.register(Disconnect, si) return si - def create_vm(self, connection, params): + def create_vm(self, connection, params, hid): """ start a thread to create vm and will update db. :param connection: vc connection return by initializesi @@ -180,11 +202,11 @@ def create_vm(self, connection, params): :return: """ t = threading.Thread(target=self.setup_vm, - args=(connection, params), + args=(connection, params, hid), name="setupvm") t.start() - def setup_vm(self, connection, params): + def setup_vm(self, connection, params, hid): """ setup a new vritualmachine. :param connection: vc connection return by initializesi @@ -206,6 +228,10 @@ def setup_vm(self, connection, params): vmgateway = vm.get(utils.VMGATEWAY) # Get vc params + vcip = vc.get(utils.VCIP) + vcusername = vc.get(utils.VCUSERNAME) + vcpwd = vc.get(utils.VCPWD) + vcport = vc.get(utils.VCPORT) template = vc.get(utils.TEMPLATE) datacenter = vc.get(utils.VC_DATACENTER) cluster = vc.get(utils.VC_CLUSTER) @@ -257,7 +283,13 @@ def setup_vm(self, connection, params): clonespec.config = vmconf clonespec.powerOn = True - host = { + try: + host = HostModel.objects.get(id=hid) + except Exception: + logger.error("No vsphere host found with id=" + hid) + return + + host.vcparam = { utils.VMUUID: '', utils.VMIP: vmip, utils.VMNETMASK: vmnetmask, @@ -268,6 +300,10 @@ def setup_vm(self, connection, params): utils.VC_CLUSTER: cluster.name, utils.VC_DATASTORE: datastore.name, utils.VC_DATACENTER: datacenter.name, + utils.VCIP: vcip, + utils.VCUSERNAME: vcusername, + utils.VCPWD: vcpwd, + utils.VCPORT: vcport } try: task = template.Clone(folder=destfolder, name=vmname, @@ -276,28 +312,34 @@ def setup_vm(self, connection, params): self.wait_for_task(task) vm = self.check_object(connection, [vim.VirtualMachine], vmname) workerapi = "tcp://" + vmip + ":2375" + host.worker_api = workerapi uuid = vm.summary.config.uuid - host.update({utils.VMUUID: uuid}) + host.vcparam[utils.VMUUID] = uuid + host.save() if self.check_isport_open(vmip, utils.WORKER_API_PORT, utils.DEFAULT_TIMEOUT): if (self.pull_images(workerapi) and setup_container_host(utils.WORKER_TYPE_DOCKER, workerapi)): - host['status'] = 'active' + host.status = 'active' logger.info(host) else: - host["status"] = 'error' + host.status = 'error' logger.error("Failed to setup container host") else: - host["status"] = 'error' + host.status = 'error' logger.error("Failed to ping docker daemon:{}:{}" .format(vmip, "2375")) - self.col.find_one_and_update({utils.VMNAME: vmname}, - {"$set": host}) + host.save() # Should be safe to delete vm though VsphereHost layer. except Exception as e: logger.error(e) - self.col.delete_one({utils.VMNAME: vmname}) + if self.check_isport_open(vmip, utils.WORKER_API_PORT, + utils.DEFAULT_TIMEOUT): + host = HostModel.objects.get(id=hid) + uuid = host.vcparam[utils.VMUUID] + self.delete_vm(vcip, vcusername, vcpwd, vcport, uuid) + host.delete() return def wait_for_task(self, task): diff --git a/src/common/utils.py b/src/common/utils.py index 91996a34..30825aed 100644 --- a/src/common/utils.py +++ b/src/common/utils.py @@ -22,11 +22,19 @@ VERSION = '1.0.5' BASEIMAGE_RELEASE = '0.3.2' FABRIC_IMAGE_FULL = 'hyperledger/fabric-{}:{}-{}' -FABRIC_IMAGE_TAG = 'hyperledger/fabric-{}' +FABRIC_IMAGE_TAG = 'hyperledger/fabric-{}:{}' +FABRIC_IMAGE = 'hyperledger/fabric-{}' FABRIC_IMAGES = ['peer', 'tools', 'orderer', 'ca', 'ccenv', 'kafka', 'zookeeper'] FABRIC_BASE_IMAGES = ['baseimage', 'baseos'] +# explorer images +BLOCKCHAIN_EXPLORER_IMAGE = 'yeasy/blockchain-explorer' +BLOCKCHAIN_EXPLORER_TAG = '0.1.0-preview' + +MYSQL_IMAGE = 'mysql' +MYSQL_TAG = '5.7' + # host status HOST_STATUS = 'status' HOST_STATUS_ACTIVE = 'active' diff --git a/src/modules/cluster.py b/src/modules/cluster.py index ae2aaba0..9406a729 100644 --- a/src/modules/cluster.py +++ b/src/modules/cluster.py @@ -19,7 +19,7 @@ from agent import get_swarm_node_ip -from common import db, log_handler, LOG_LEVEL +from common import db, log_handler, LOG_LEVEL, utils from common import CLUSTER_PORT_START, CLUSTER_PORT_STEP, \ NETWORK_TYPE_FABRIC_PRE_V1, NETWORK_TYPE_FABRIC_V1, \ CONSENSUS_PLUGINS_FABRIC_V1, \ @@ -236,12 +236,6 @@ def create(self, name, host_id, config, start_port=0, logger.warning("host {} is already full".format(host_id)) return None - if worker.type == WORKER_TYPE_VSPHERE: - vm_params = self.host_handler.get_vm_params_by_id(host_id) - docker_daemon = vm_params.get(VMIP) + ":2375" - worker.update({"worker_api": "tcp://" + docker_daemon}) - logger.info(worker) - peer_num = int(config.get_data().get("size", 4)) ca_num = 2 if peer_num > 1 else 1 @@ -312,7 +306,7 @@ def create(self, name, host_id, config, start_port=0, ) def check_health_work(cid): - time.sleep(5) + time.sleep(60) self.refresh_health(cid) t = Thread(target=check_health_work, args=(cid,)) t.start() @@ -695,8 +689,7 @@ def _get_service_ip(self, cluster_id, node='peer0'): cluster_id, node)) logger.debug("swarm host, ip = {}".format(host_ip)) elif host_type == WORKER_TYPE_VSPHERE: - vm_params = self.host_handler.get_vm_params_by_id(host_id) - host_ip = vm_params.get(VMIP) + host_ip = host.vcparam[utils.VMIP] logger.debug(" host, ip = {}".format(host_ip)) else: logger.error("Unknown host type = {}".format(host_type)) diff --git a/src/modules/host.py b/src/modules/host.py index 4f5fc1bb..083c962c 100644 --- a/src/modules/host.py +++ b/src/modules/host.py @@ -17,7 +17,7 @@ sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..')) from common import \ log_handler, \ - FabricV1NetworkConfig, \ + FabricV1NetworkConfig, utils, \ LOG_LEVEL, CLUSTER_LOG_TYPES, CLUSTER_LOG_LEVEL, \ NETWORK_SIZE_FABRIC_V1, \ CLUSTER_PORT_START, CLUSTER_PORT_STEP, \ @@ -107,6 +107,19 @@ def create(self, name, worker_api, host_type, capacity=1, logger.warning("Host {} cannot be setup".format(name)) return {} + hid = uuid4().hex + host = HostModel(id=hid, + name=name, + worker_api=worker_api, + capacity=capacity, + type=host_type, + log_level=log_level, + log_type=log_type, + log_server=log_server, + autofill=autofill == "true", + schedulable=schedulable == "true" + ) + if (host_type == WORKER_TYPE_DOCKER or host_type == WORKER_TYPE_SWARM): if not self.host_agents[host_type].create(worker_api): @@ -118,7 +131,7 @@ def create(self, name, worker_api, host_type, capacity=1, vc = params.get(VCENTER) vm = params.get(VIRTUAL_MACHINE) - worker_api = vc.get(VCIP) + vc_ip = vc.get(VCIP) vc_username = vc.get(VCUSERNAME) vc_passwd = vc.get(VCPWD) vc_port = vc.get(VCPORT) @@ -142,11 +155,12 @@ def create(self, name, worker_api, host_type, capacity=1, HOST_STATUS: HOST_STATUS_PENDING } logger.debug("update {}".format(h_update)) + host.status = HOST_STATUS_PENDING try: - if self.host_agents[host_type].create(worker_api, + if self.host_agents[host_type].create(vc_ip, vc_username, vc_passwd, vc_port, - params): + params, hid): logger.info("Creating vSphere host{}".format(name)) except Exception as e: # Catch failure while connecting to vc. @@ -154,18 +168,6 @@ def create(self, name, worker_api, host_type, capacity=1, logger.error("{}".format(e)) return {"msg": "{}".format(e)} - hid = uuid4().hex - host = HostModel(id=hid, - name=name, - worker_api=worker_api, - capacity=capacity, - type=host_type, - log_level=log_level, - log_type=log_type, - log_server=log_server, - autofill=autofill == "true", - schedulable=schedulable == "true" - ) host.save() if capacity > 0 and autofill == "true": # should autofill it @@ -187,33 +189,6 @@ def get_by_id(self, id): return ins - # def get_vc_params_by_id(self, id): - # """ Get vCenter params while host type is vsphere - # - # :param id: id of the doc - # :return: serialized result or obj - # """ - # ins = self.col.find_one({"id": id}) - # if not ins: - # logger.warning("No host found with id=" + id) - # return {} - # return self._serialize(ins, keys=[VCUSERNAME, - # VCPWD, VCPORT]) - # - # def get_vm_params_by_id(self, id): - # """ Get VM params while host type is vsphere - # - # :param id: id of the doc - # :return: serialized result or obj - # """ - # ins = self.col.find_one({"id": id}) - # if not ins: - # logger.warning("No host found with id=" + id) - # return {} - # return self._serialize(ins, keys=[VMUUID, - # VMIP, - # VMNAME]) - def update(self, id, d): """ Update a host's property @@ -286,17 +261,19 @@ def delete(self, id): host_type == WORKER_TYPE_SWARM): self.host_agents[host_type].delete(h.worker_api) - # elif host_type == WORKER_TYPE_VSPHERE: - # if h.status == "pending": - # return False - # vc_params = self.get_vc_params_by_id(id) - # vm_params = self.get_vm_params_by_id(id) - # logger.info(vc_params) - # self.host_agents[host_type].delete(vm_params.get(VMUUID), - # h.worker_api, - # vc_params.get(VCUSERNAME), - # vc_params.get(VCPWD), - # vc_params.get(VCPORT)) + elif host_type == WORKER_TYPE_VSPHERE: + if h.status == "pending": + return False + vmuuid = h.vcparam[utils.VMUUID] + vcip = h.vcparam[utils.VCIP] + vcusername = h.vcparam[utils.VCUSERNAME] + vcpwd = h.vcparam[utils.VCPWD] + vcport = h.vcparam[utils.VCPORT] + self.host_agents[host_type].delete(vmuuid, + vcip, + vcusername, + vcpwd, + vcport) h.delete() return True diff --git a/src/modules/models/host.py b/src/modules/models/host.py index c7aa78e3..505826c6 100644 --- a/src/modules/models/host.py +++ b/src/modules/models/host.py @@ -35,6 +35,7 @@ class Host(Document): schedulable = BooleanField(default=False) capacity = IntField(default=0) clusters = ListField(default=[]) + vcparam = DictField(default={}) class Cluster(Document):