Skip to content

Commit

Permalink
[CE-297] fix the incompatibility with vSphere host type
Browse files Browse the repository at this point in the history 
1.Update to support vCenter params.
2.Prolong the healcheck sleep time in case of other type
  of host need more time to create a Fabric cluster.

Change-Id: Ib7e53cfc022998d0618b555237481677bc0f8492
Signed-off-by: hainingzhang <haininghenryzh@vmware.com>
  • Loading branch information
@hainingzhang
hainingzhang committed Mar 11, 2018
1 parent 7799713 commit f86c9ff
Show file tree
Hide file tree
Showing 7 changed files with 102 additions and 81 deletions.
2 changes: 1 addition & 1 deletion docs/setup_worker_vsphere.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ Upload the template OS OVA to vCenter before create vSphere type host in Cello.
1. Login to vSphere Client.
2. Right-Click on ESX host on which you want to deploy template.
3. Select Deploy OVF template.
4. Copy and paste URL for [OVA for Cello](https://drive.google.com/file/d/0B4Ioua6jjCH9b0ROOE14SUlqUk0/view?usp=sharing)
4. Copy and paste URL for [OVA for Cello](https://drive.google.com/file/d/1VN4FDlgd0Q7tqBtGNBN5trP7upaswUWS/view?usp=sharing)
5. Please deploy the ova on the same cluster which will be planned to be used by cello later
6. **Check the name of the VM created** , this will be used to create vSphere type host in Cello later. (Should default to PhotonOSTemplate.ova)

Expand Down
4 changes: 2 additions & 2 deletions src/agent/vsphere/host.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ class VsphereHost(HostBase):
def __init__(self):
self.collection = db["host"]

def create(self, vcip, username, pwd, port, params):
def create(self, vcip, username, pwd, port, params, hid):
""" Create a new vSphere host
:param vcip : vCenter address
:param username: vCenter username
Expand Down Expand Up @@ -131,7 +131,7 @@ def create(self, vcip, username, pwd, port, params):
else:
vc_resources[NETWORK] = network

operation.create_vm(connection, params)
operation.create_vm(connection, params, hid)
return True

def delete(self, vmuuid, vcip, username, pwd, port=443):
Expand Down
68 changes: 55 additions & 13 deletions src/agent/vsphere/host_operations.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..'))
from common import log_handler, LOG_LEVEL, db, utils
from agent import setup_container_host
from modules.models import Host as HostModel

context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
context.verify_mode = ssl.CERT_NONE
Expand Down Expand Up @@ -46,6 +47,7 @@ def pull_images(self, worker_api):
try:
self.pull_and_tag_fabric_images(client)
self.pull_and_tag_fabric_base_images(client)
self.pull_and_tag_blockchain_explorer_images(client)
return True
except Exception as e:
logger.error('{}'.format(e))
Expand Down Expand Up @@ -73,6 +75,26 @@ def pull_and_tag_fabric_base_images(self, docker_client):
self.tag_image(docker_client, image, utils.ARCH,
utils.BASEIMAGE_RELEASE)

def pull_and_tag_blockchain_explorer_images(self, docker_client):
"""
pull base fabric explorer images, blockchain-explorer, mysql5.7.
:param docker_client: DockerClient object
:return:
"""
try:
image = utils.BLOCKCHAIN_EXPLORER_IMAGE
tag = utils.BLOCKCHAIN_EXPLORER_TAG
logger.info("pulling image: {}".format(image))
docker_client.images.pull(image, tag)
image = utils.MYSQL_IMAGE
tag = utils.MYSQL_TAG
logger.info("pulling image: {}".format(image))
docker_client.images.pull(image, tag)
except Exception as e:
logger.error("Docker client error msg: {}".format(e))
error_msg = "Cannot pull image:{}".format("explorer images")
raise Exception(error_msg)

def pull_image(self, client, image, arch, version):
"""
pull specific fabric images
Expand All @@ -82,7 +104,7 @@ def pull_image(self, client, image, arch, version):
:param version: fabric image version
"""
try:
image_to_be_pull = utils.FABRIC_IMAGE_TAG.format(image)
image_to_be_pull = utils.FABRIC_IMAGE.format(image)
name = utils.FABRIC_IMAGE_FULL.format(image, arch, version)
logger.info("pulling image: {}".format(name))
client.images.pull(image_to_be_pull, tag=arch + '-' + version)
Expand All @@ -101,7 +123,7 @@ def tag_image(self, client, image, arch, version):
"""
try:
name = utils.FABRIC_IMAGE_FULL.format(image, arch, version)
tag = utils.FABRIC_IMAGE_TAG.format(image)
tag = utils.FABRIC_IMAGE_TAG.format(image, version)
image_to_be_tag = client.images.get(name)
logger.info("tag {} => {}".format(name, tag))
image_to_be_tag.tag(tag)
Expand Down Expand Up @@ -172,19 +194,19 @@ def initializesi(self, vcip, username, pwd, port):
atexit.register(Disconnect, si)
return si

def create_vm(self, connection, params):
def create_vm(self, connection, params, hid):
"""
start a thread to create vm and will update db.
:param connection: vc connection return by initializesi
:param params:
:return:
"""
t = threading.Thread(target=self.setup_vm,
args=(connection, params),
args=(connection, params, hid),
name="setupvm")
t.start()

def setup_vm(self, connection, params):
def setup_vm(self, connection, params, hid):
"""
setup a new vritualmachine.
:param connection: vc connection return by initializesi
Expand All @@ -206,6 +228,10 @@ def setup_vm(self, connection, params):
vmgateway = vm.get(utils.VMGATEWAY)

# Get vc params
vcip = vc.get(utils.VCIP)
vcusername = vc.get(utils.VCUSERNAME)
vcpwd = vc.get(utils.VCPWD)
vcport = vc.get(utils.VCPORT)
template = vc.get(utils.TEMPLATE)
datacenter = vc.get(utils.VC_DATACENTER)
cluster = vc.get(utils.VC_CLUSTER)
Expand Down Expand Up @@ -257,7 +283,13 @@ def setup_vm(self, connection, params):
clonespec.config = vmconf
clonespec.powerOn = True

host = {
try:
host = HostModel.objects.get(id=hid)
except Exception:
logger.error("No vsphere host found with id=" + hid)
return

host.vcparam = {
utils.VMUUID: '',
utils.VMIP: vmip,
utils.VMNETMASK: vmnetmask,
Expand All @@ -268,6 +300,10 @@ def setup_vm(self, connection, params):
utils.VC_CLUSTER: cluster.name,
utils.VC_DATASTORE: datastore.name,
utils.VC_DATACENTER: datacenter.name,
utils.VCIP: vcip,
utils.VCUSERNAME: vcusername,
utils.VCPWD: vcpwd,
utils.VCPORT: vcport
}
try:
task = template.Clone(folder=destfolder, name=vmname,
Expand All @@ -276,28 +312,34 @@ def setup_vm(self, connection, params):
self.wait_for_task(task)
vm = self.check_object(connection, [vim.VirtualMachine], vmname)
workerapi = "tcp://" + vmip + ":2375"
host.worker_api = workerapi
uuid = vm.summary.config.uuid
host.update({utils.VMUUID: uuid})
host.vcparam[utils.VMUUID] = uuid
host.save()
if self.check_isport_open(vmip, utils.WORKER_API_PORT,
utils.DEFAULT_TIMEOUT):
if (self.pull_images(workerapi) and
setup_container_host(utils.WORKER_TYPE_DOCKER,
workerapi)):
host['status'] = 'active'
host.status = 'active'
logger.info(host)
else:
host["status"] = 'error'
host.status = 'error'
logger.error("Failed to setup container host")
else:
host["status"] = 'error'
host.status = 'error'
logger.error("Failed to ping docker daemon:{}:{}"
.format(vmip, "2375"))
self.col.find_one_and_update({utils.VMNAME: vmname},
{"$set": host})
host.save()
# Should be safe to delete vm though VsphereHost layer.
except Exception as e:
logger.error(e)
self.col.delete_one({utils.VMNAME: vmname})
if self.check_isport_open(vmip, utils.WORKER_API_PORT,
utils.DEFAULT_TIMEOUT):
host = HostModel.objects.get(id=hid)
uuid = host.vcparam[utils.VMUUID]
self.delete_vm(vcip, vcusername, vcpwd, vcport, uuid)
host.delete()
return

def wait_for_task(self, task):
Expand Down
10 changes: 9 additions & 1 deletion src/common/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,19 @@
VERSION = '1.0.5'
BASEIMAGE_RELEASE = '0.3.2'
FABRIC_IMAGE_FULL = 'hyperledger/fabric-{}:{}-{}'
FABRIC_IMAGE_TAG = 'hyperledger/fabric-{}'
FABRIC_IMAGE_TAG = 'hyperledger/fabric-{}:{}'
FABRIC_IMAGE = 'hyperledger/fabric-{}'
FABRIC_IMAGES = ['peer', 'tools', 'orderer', 'ca', 'ccenv', 'kafka',
'zookeeper']
FABRIC_BASE_IMAGES = ['baseimage', 'baseos']

# explorer images
BLOCKCHAIN_EXPLORER_IMAGE = 'yeasy/blockchain-explorer'
BLOCKCHAIN_EXPLORER_TAG = '0.1.0-preview'

MYSQL_IMAGE = 'mysql'
MYSQL_TAG = '5.7'

# host status
HOST_STATUS = 'status'
HOST_STATUS_ACTIVE = 'active'
Expand Down
13 changes: 3 additions & 10 deletions src/modules/cluster.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

from agent import get_swarm_node_ip

from common import db, log_handler, LOG_LEVEL
from common import db, log_handler, LOG_LEVEL, utils
from common import CLUSTER_PORT_START, CLUSTER_PORT_STEP, \
NETWORK_TYPE_FABRIC_PRE_V1, NETWORK_TYPE_FABRIC_V1, \
CONSENSUS_PLUGINS_FABRIC_V1, \
Expand Down Expand Up @@ -236,12 +236,6 @@ def create(self, name, host_id, config, start_port=0,
logger.warning("host {} is already full".format(host_id))
return None

if worker.type == WORKER_TYPE_VSPHERE:
vm_params = self.host_handler.get_vm_params_by_id(host_id)
docker_daemon = vm_params.get(VMIP) + ":2375"
worker.update({"worker_api": "tcp://" + docker_daemon})
logger.info(worker)

peer_num = int(config.get_data().get("size", 4))
ca_num = 2 if peer_num > 1 else 1

Expand Down Expand Up @@ -312,7 +306,7 @@ def create(self, name, host_id, config, start_port=0,
)

def check_health_work(cid):
time.sleep(5)
time.sleep(60)
self.refresh_health(cid)
t = Thread(target=check_health_work, args=(cid,))
t.start()
Expand Down Expand Up @@ -695,8 +689,7 @@ def _get_service_ip(self, cluster_id, node='peer0'):
cluster_id, node))
logger.debug("swarm host, ip = {}".format(host_ip))
elif host_type == WORKER_TYPE_VSPHERE:
vm_params = self.host_handler.get_vm_params_by_id(host_id)
host_ip = vm_params.get(VMIP)
host_ip = host.vcparam[utils.VMIP]
logger.debug(" host, ip = {}".format(host_ip))
else:
logger.error("Unknown host type = {}".format(host_type))
Expand Down
85 changes: 31 additions & 54 deletions src/modules/host.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..'))
from common import \
log_handler, \
FabricV1NetworkConfig, \
FabricV1NetworkConfig, utils, \
LOG_LEVEL, CLUSTER_LOG_TYPES, CLUSTER_LOG_LEVEL, \
NETWORK_SIZE_FABRIC_V1, \
CLUSTER_PORT_START, CLUSTER_PORT_STEP, \
Expand Down Expand Up @@ -107,6 +107,19 @@ def create(self, name, worker_api, host_type, capacity=1,
logger.warning("Host {} cannot be setup".format(name))
return {}

hid = uuid4().hex
host = HostModel(id=hid,
name=name,
worker_api=worker_api,
capacity=capacity,
type=host_type,
log_level=log_level,
log_type=log_type,
log_server=log_server,
autofill=autofill == "true",
schedulable=schedulable == "true"
)

if (host_type == WORKER_TYPE_DOCKER or
host_type == WORKER_TYPE_SWARM):
if not self.host_agents[host_type].create(worker_api):
Expand All @@ -118,7 +131,7 @@ def create(self, name, worker_api, host_type, capacity=1,
vc = params.get(VCENTER)
vm = params.get(VIRTUAL_MACHINE)

worker_api = vc.get(VCIP)
vc_ip = vc.get(VCIP)
vc_username = vc.get(VCUSERNAME)
vc_passwd = vc.get(VCPWD)
vc_port = vc.get(VCPORT)
Expand All @@ -142,30 +155,19 @@ def create(self, name, worker_api, host_type, capacity=1,
HOST_STATUS: HOST_STATUS_PENDING
}
logger.debug("update {}".format(h_update))
host.status = HOST_STATUS_PENDING
try:
if self.host_agents[host_type].create(worker_api,
if self.host_agents[host_type].create(vc_ip,
vc_username,
vc_passwd, vc_port,
params):
params, hid):
logger.info("Creating vSphere host{}".format(name))

except Exception as e: # Catch failure while connecting to vc.
logger.error("Host {} cannot be setup".format(name))
logger.error("{}".format(e))
return {"msg": "{}".format(e)}

hid = uuid4().hex
host = HostModel(id=hid,
name=name,
worker_api=worker_api,
capacity=capacity,
type=host_type,
log_level=log_level,
log_type=log_type,
log_server=log_server,
autofill=autofill == "true",
schedulable=schedulable == "true"
)
host.save()

if capacity > 0 and autofill == "true": # should autofill it
Expand All @@ -187,33 +189,6 @@ def get_by_id(self, id):

return ins

# def get_vc_params_by_id(self, id):
# """ Get vCenter params while host type is vsphere
#
# :param id: id of the doc
# :return: serialized result or obj
# """
# ins = self.col.find_one({"id": id})
# if not ins:
# logger.warning("No host found with id=" + id)
# return {}
# return self._serialize(ins, keys=[VCUSERNAME,
# VCPWD, VCPORT])
#
# def get_vm_params_by_id(self, id):
# """ Get VM params while host type is vsphere
#
# :param id: id of the doc
# :return: serialized result or obj
# """
# ins = self.col.find_one({"id": id})
# if not ins:
# logger.warning("No host found with id=" + id)
# return {}
# return self._serialize(ins, keys=[VMUUID,
# VMIP,
# VMNAME])

def update(self, id, d):
""" Update a host's property
Expand Down Expand Up @@ -286,17 +261,19 @@ def delete(self, id):
host_type == WORKER_TYPE_SWARM):
self.host_agents[host_type].delete(h.worker_api)

# elif host_type == WORKER_TYPE_VSPHERE:
# if h.status == "pending":
# return False
# vc_params = self.get_vc_params_by_id(id)
# vm_params = self.get_vm_params_by_id(id)
# logger.info(vc_params)
# self.host_agents[host_type].delete(vm_params.get(VMUUID),
# h.worker_api,
# vc_params.get(VCUSERNAME),
# vc_params.get(VCPWD),
# vc_params.get(VCPORT))
elif host_type == WORKER_TYPE_VSPHERE:
if h.status == "pending":
return False
vmuuid = h.vcparam[utils.VMUUID]
vcip = h.vcparam[utils.VCIP]
vcusername = h.vcparam[utils.VCUSERNAME]
vcpwd = h.vcparam[utils.VCPWD]
vcport = h.vcparam[utils.VCPORT]
self.host_agents[host_type].delete(vmuuid,
vcip,
vcusername,
vcpwd,
vcport)
h.delete()
return True

Expand Down
1 change: 1 addition & 0 deletions src/modules/models/host.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ class Host(Document):
schedulable = BooleanField(default=False)
capacity = IntField(default=0)
clusters = ListField(default=[])
vcparam = DictField(default={})


class Cluster(Document):
Expand Down

0 comments on commit f86c9ff

Please sign in to comment.