vmcatcher_image

#!/usr/bin/env python
import sys
if sys.version_info < (2, 4):
    print "Your python interpreter is too old. Please consider upgrading."
    sys.exit(1)

if sys.version_info < (2, 5):
    import site
    import os.path
    from distutils.sysconfig import get_python_lib
    found = False
    module_dir = get_python_lib()
    for name in os.listdir(module_dir):
        lowername = name.lower()
        if lowername[0:10] == 'sqlalchemy' and 'egg' in lowername:
            sqlalchemy_dir = os.path.join(module_dir, name)
            if os.path.isdir(sqlalchemy_dir):
                site.addsitedir(sqlalchemy_dir)
                found = True
                break
    if not found:
        print "Could not find SQLAlchemy installed."
        sys.exit(1)

from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
import vmcatcher.databaseDefinition as model
import os
import logging
import optparse
from smimeX509validation import TrustStore, LoadDirChainOfTrust,smimeX509validation, smimeX509ValidationError
from vmcatcher.__version__ import version
import vmcatcher
import urllib2
import urllib
import hashlib
import datetime
from hepixvmitrust.vmitrustlib import VMimageListDecoder as VMimageListDecoder
from hepixvmitrust.vmitrustlib import time_format_definition as time_format_definition
from M2Crypto import SMIME, X509, BIO
try:
    import simplejson as json
except:
    import json

class db_actions:

    def __init__(self,session):
        self.session = session
        self.log = logging.getLogger("vmcatcher_subscribe.db_actions")


    def image_lister(self):
        outputlist = []

        imageDefs = self.session.query(model.ImageDefinition,model.Subscription).\
            filter(model.Subscription.id == model.ImageDefinition.subscription)

        for q_result in imageDefs:
            imagedef = q_result[0]
            subscription = q_result[1]
            subauthq = self.session.query(model.ImageInstance,model.ImageListInstance).\
                filter(model.ImageDefinition.id == imagedef.id).\
                filter(model.Subscription.id == subscription.id).\
                filter(model.ImageDefinition.subscription == model.Subscription.id).\
                filter(model.ImageInstance.fkIdentifier == model.ImageDefinition.id).\
                filter(model.ImageInstance.fkimagelistinstance == model.ImageListInstance.id).\
                filter(model.Subscription.imagelist_latest == model.ImageListInstance.id)
            bimappedOutput = imagedef.cache
            if subauthq.count() == 1:
                q_result = subauthq.one()
                imageInstance = q_result[0]
                listInstance = q_result[1]
                available = 0
                if ((imagedef.latest == imageInstance.id) and (subscription.authorised == 1) and
                    (subscription.imagelist_latest == listInstance.id) and (listInstance.expired == None) and
                    (imageInstance.fkimagelistinstance != None) and (imagedef.latest !=None)):
                    available = 1
                bimappedOutput = imagedef.cache + (available << 1)
            outputlist.append("%s\t%s\t%s" % (imagedef.identifier,bimappedOutput,subscription.identifier))

        seperator = '\n'
        return seperator.join(outputlist)

    def image_by_sha512_writefile_imagelist(self,treess,sha512,path):
        query_image = self.session.query(model.ImageInstance).\
            filter(model.ImageInstance.fkimagelistinstance == model.ImageListInstance.id).\
            filter(model.ImageInstance.sha512 == str(sha512))
        if query_image.count() == 0:
            self.log.warning('Message not found')
            return False
        fp = open(path,'w')
        for image in query_image:
            fp.write(image.data)
        fp.close()
        return True


    def image_by_sha512_writefile_json(self,treess,sha512,path):
        query_image = self.session.query(model.ImageInstance,model.ImageListInstance).\
            filter(model.ImageInstance.fkimagelistinstance == model.ImageListInstance.id).\
            filter(model.ImageInstance.sha512 == sha512)
        if query_image.count() == 0:
            self.log.warning('Message not found')
            return False
        fp = open(path,'w')
        for touple_set in query_image:
            image = touple_set[0]
            image_list = touple_set[1]
            buf = BIO.MemoryBuffer(str(image_list.data))
            sk = X509.X509_Stack()
            p7, data = SMIME.smime_load_pkcs7_bio(buf)
            data_str = data.read()
            fp.write(data_str)
        fp.close()
    def image_by_sha512_display_info(self,ancore,sha512):
        output = []
        query_image = self.session.query(model.ImageInstance,model.ImageListInstance).\
            filter(model.ImageInstance.fkimagelistinstance == model.ImageListInstance.id).\
            filter(model.ImageInstance.sha512 == str(sha512))
        if query_image.count() == 0:
            self.log.warning('Message not found')
            return False
        for query_row in query_image:
            image = query_row[0]
            imagelist = query_row[1]

            print ('dc:identifier=%s' % (image.identifier))
            print ('dc:description=%s' % (image.description))
            print ('hv:hypervisor=%s' % (image.hypervisor))
            print ('sl:checksum:sha512=%s' % (image.sha512))
            print ('hv:uri=%s' % (image.uri))
            print ('sl:os=%s' % (image.os))
            print ('sl:osversion=%s' % (image.osversion ))
            print ('sl:arch=%s' % (image.arch))
            print ('hv:version=%s' % (image.version))
            print ('hv:size=%s' % (image.size))
            print ('dc:title=%s' % (image.title))
            print ('sl:comments=%s' % (image.comments))
            print ('dc:date:imported=%s' % (imagelist.imported.strftime(time_format_definition)))
            print ('dc:date:created=%s' % (imagelist.created.strftime(time_format_definition)))
            print ('dc:date:expires=%s' % (imagelist.expires.strftime(time_format_definition)))
            #validated_data = ancore.validate_text(str(image.data))
            #data = validated_data['data']
            #dn = validated_data['signer_dn']
            #ca = validated_data['issuer_dn']
        return True
class queryby_base:
    """"Base class for querying subscriptions"""
    def __init__(self,session):
        self.session = session
    def subscription_by_id(self,private_id):
        subscriptionlist = self.session.query(model.Subscription).\
                filter(model.Subscription.id==private_id)
        return subscriptionlist
    def subscription_by_uri(self,url):
        subscriptionlist = self.session.query(model.Subscription).\
                filter(model.Subscription.url==url)
        return subscriptionlist
    def subscription_by_uuid(self,uuid):
        subscriptionlist = self.session.query(model.Subscription).\
                filter(model.Subscription.identifier==uuid)
        return subscriptionlist
    def imagelist_by_id(self,private_id):
        subscriptionlist = self.session.query(model.ImageListInstance).\
                filter(model.ImageListInstance.id==private_id)
        return subscriptionlist

        # Now the virtual class
    def subscription_get(self,by_id):
        return self.subscription_by_id(by_id)

    def imageDefinition_by_imageUuid(self,uuid):
        imagelistImage = self.session.query(model.ImageDefinition).\
                filter(model.ImageDefinition.identifier == uuid)


        return imagelistImage
    def imageDefinition_by_imageSha512(self,sha512):
        imagelistImage = self.session.query(model.ImageDefinition).\
                filter(model.ImageListInstance.id==model.ImageInstance.fkimagelistinstance).\
                filter(model.ImageInstance.sha512 == sha512).\
                filter(model.ImageInstance.fkIdentifier == model.ImageDefinition.id)
        return imagelistImage


class queryby_sha512(queryby_base):
    def imageDefinition_get(self,sha512):
        return self.imageDefinition_by_imageSha512(sha512)

class queryby_uuid(queryby_base):
    def imageDefinition_get(self,uuid):
        return self.imageDefinition_by_imageUuid(uuid)

class output_driver_base:
    def __init__(self,file_pointer,session,anchor):
        self.session = session
        self.log = logging.getLogger("db_actions")
        self.file_pointer = file_pointer
        self.anchor = anchor
    def display_subscription_imagelist(self,subscription,imagelist):
        status = None

        self.display_subscription(subscription)
        if not self.display_imagelist(imagelist):
            return False
        return True
    def display_subscription(self,subscription):
        pass
    def display_imagelist(self,imagelist):
        return True
    def subscriptions_lister(self):
        pass

class output_driver_smime(output_driver_base):
    def display_subscription(self,subscription):
        pass
    def display_imagelist(self,imagelist):
        self.file_pointer.write (imagelist.data)
        return True
    def display_imagelistImage(self,imagedef,imagelist,image):
        if not self.display_imagelist(imagelist):
            return False
        return True

class output_driver_message(output_driver_base):
    def display_subscription(self,subscription):
        pass
    def display_imagelist(self,imagelist):
        smimeProcessor =  smimeX509validation(self.anchor)
        try:
            smimeProcessor.Process(str(imagelist.data))
        except smimeX509ValidationError,E:
            self.log.error("Failed to validate text for '%s' produced error '%s'" % (imagelist,E))
            return False
        if not smimeProcessor.verified:
            self.log.error("Failed to validate text for '%s' produced error '%s'" % (subscriptionKey,E))
            return False
        self.file_pointer.write (smimeProcessor.InputDaraStringIO.getvalue())
        return True

    def display_imagelistImage(self,imagedef,imagelist,image):
        if not self.display_imagelist(imagelist):
            return False
        return True

class output_driver_lines(output_driver_base):
    def display_subscription(self,image):
        self.file_pointer.write ('image.dc:identifier=%s\n' % (image.identifier))
        self.file_pointer.write ('image.dc:description=%s\n' % (image.description))
        self.file_pointer.write ('image.hv:uri=%s\n' % (image.uri))
        return True
    def display_imagelist(self,imagelist):
        smimeProcessor =  smimeX509validation(self.anchor)
        try:
            smimeProcessor.Process(str(imagelist.data))
        except smimeX509ValidationError,E:
            self.log.error("Failed to validate text for '%s' produced error '%s'" % (imagelist,E))
            return False
        self.file_pointer.write (smimeProcessor.InputDaraStringIO.getvalue())
        return True
    def display_imagelistImage(self,imagedef,imagelist,image):
        self.file_pointer.write ('imagelist.dc:identifier=%s\n' % (imagedef.identifier))
        self.file_pointer.write ('imagelist.dc:date:imported=%s\n' % (imagelist.imported.strftime(time_format_definition)))
        self.file_pointer.write ('imagelist.dc:date:created=%s\n' % (imagelist.created.strftime(time_format_definition)))
        self.file_pointer.write ('imagelist.dc:date:expires=%s\n' % (imagelist.expires.strftime(time_format_definition)))
        self.file_pointer.write ('imagedef.dc:identifier=%s\n' % (imagedef.identifier))
        self.file_pointer.write ('imagedef.cache=%s\n' % (imagedef.cache))
        self.file_pointer.write ('image.dc:description=%s\n' % (image.description))
        self.file_pointer.write ('image.dc:title=%s\n' % (image.title))
        self.file_pointer.write ('image.hv:hypervisor=%s\n' % (image.hypervisor))
        self.file_pointer.write ('image.hv:size=%s\n' % (image.size))
        self.file_pointer.write ('image.hv:uri=%s\n' % (image.uri))
        self.file_pointer.write ('image.hv:version=%s\n' % (image.version))
        self.file_pointer.write ('image.sl:arch=%s\n' % (image.hypervisor))
        self.file_pointer.write ('image.sl:checksum:sha512=%s\n' % (image.sha512))
        self.file_pointer.write ('image.sl:comments=%s\n' % (image.comments))
        self.file_pointer.write ('image.sl:os=%s\n' % (image.os))
        self.file_pointer.write ('image.sl:osversion=%s\n' % (image.osversion))
        return True




    def subscriptions_lister(self):

        subauthq = self.session.query(model.Subscription).all()
        for item in subauthq:
            self.file_pointer.write ("%s\t%s\t%s\n" % (item.uuid,item.authorised,item.url))

class db_controler:
    def __init__(self,dboptions, dblog):
        self.log = logging.getLogger("vmcatcher_image.controler")
        self.engine = create_engine(dboptions, echo=dblog)
        model.init(self.engine)
        self.SessionFactory = sessionmaker(bind=self.engine)
        self.anchor = None
    def setup_trust_anchor(self,directory):
        self.anchor = LoadDirChainOfTrust(directory)
    def setup_selector_factory(self,factory):
        self.factory_selector = factory
    def setup_view_factory(self,factory):
        self.factory_view = factory
    def images_info(self,images_selected,outputfiles):
        pairs, extra_selectors ,extra_paths = pairsNnot(images_selected,outputfiles)

        for item in extra_selectors:
            pairs.append([item,None])

        NoErrorHappened = True
        Session = self.SessionFactory()
        selector = self.factory_selector(Session)
        for pair in pairs:
            selector_filter = pair[0]
            output_file_name = pair[1]
            output_fileptr = sys.stdout
            if output_file_name != None:
                output_fileptr = open(output_file_name,'w+')
                output_fileptr.flush()

            queryImageDef = selector.imageDefinition_get(selector_filter)
            if queryImageDef.count() == 0:
                self.log.warning("Selections '%s' does not match any known images." % (selector_filter))
                continue
            view = self.factory_view(output_fileptr,Session,self.anchor)
            for imagedef in queryImageDef:
                details = Session.query(model.ImageListInstance,model.ImageInstance).\
                    filter(model.ImageListInstance.id==model.ImageInstance.fkimagelistinstance).\
                    filter(model.ImageDefinition.id == imagedef.id).\
                    filter(model.ImageInstance.fkIdentifier == model.ImageDefinition.id).\
                    filter(model.Subscription.id == imagedef.subscription).\
                    filter(model.Subscription.imagelist_latest == model.ImageListInstance.id)
                if details.count() == 0:
                    self.log.warning("Image '%s' has expired." % (selector_filter))
                for item in details:
                    imagelist = item[0]
                    image = item[1]
                    if not view.display_imagelistImage(imagedef,imagelist,image):
                        NoErrorHappened = False
            if output_file_name != None:
                output_fileptr.close()
        return NoErrorHappened

    def image_list(self):
        Session = self.SessionFactory()
        db = db_actions(Session)
        print db.image_lister()
    def images_subscribe(self,images_selected,subscribe):
        Session = self.SessionFactory()
        selector = self.factory_selector(Session)
        for image in images_selected:
            queryImageDef = selector.imageDefinition_get(image)
            for ImageDef in queryImageDef:
                ImageDef.cache = subscribe
                Session.add(ImageDef)
                Session.commit()

# User interface

def pairsNnot(list_a,list_b):
    len_generate_list = len(list_a)
    len_image_list = len(list_b)
    ocupies_generate_list = set(range(len_generate_list))
    ocupies_image_list = set(range(len_image_list))
    ocupies_pairs = ocupies_image_list.intersection(ocupies_generate_list)
    diff_a = ocupies_generate_list.difference(ocupies_image_list)
    diff_b = ocupies_image_list.difference(ocupies_generate_list)
    arepairs = []
    for i in ocupies_pairs:
        arepairs.append([list_a[i],list_b[i]])
    notpairs_a = []
    for i in diff_a:
        notpairs_a.append(list_a[i])
    notpairs_b = []
    for i in diff_b:
        notpairs_b.append(list_b[i])

    return arepairs,notpairs_a,notpairs_b


def main():
    """Runs program and handles command line options"""
    p = optparse.OptionParser(version = "%prog " + version)
    p.add_option('-l', '--list', action ='store_true',help='list all images.')
    p.add_option('-d', '--database', action ='store', help='Database Initiation string')
    p.add_option('-c', '--cert-dir', action ='store',help='Certificate directory.', metavar='INPUTDIR',
        default='/etc/grid-security/certificates/')
    p.add_option('-a', '--add', action ='store_true',help='Add image to Cache.')
    p.add_option('-r', '--remove', action ='store_true',help='Remove image from Cache.')
    p.add_option('-s', '--sha512', action ='append',help='Select images by sha512.', metavar='SHA512')
    p.add_option('-u', '--uuid', action ='append',help='Select images by uuid.', metavar='UUID')
    p.add_option('-i', '--info', action ='store_true',help='Info on selected images')
    p.add_option('-f', '--format', action ='store',help='Sets the output format')
    p.add_option('-o', '--output', action ='append',help='Export File.', metavar='OUTPUTFILE')
    p.add_option('-L', '--logfile', action ='store',help='Logfile configuration file.', metavar='LOGFILE')
    p.add_option('-v', '--verbose', action ='count',help='Change global log level, increasing log output.', metavar='LOGFILE')
    p.add_option('-q', '--quiet', action ='count',help='Change global log level, decreasing log output.', metavar='LOGFILE')
    p.add_option('--log-config', action ='store',help='Logfile configuration file, (overrides command line).', metavar='LOGFILE')
    p.add_option('--log-sql-info', action ='store_true',help='Echo all SQL commands.')
    outputformats = set(['SMIME','message','lines'])
    output_format_selected = set([])
    options, arguments = p.parse_args()
    anchor_needed = False
    format_needed = False
    anchor =  None
    actions = set([])
    images_selected = []
    messages_path = []
    subscription_url_list = []
    outputfiles = []
    input_format_selected = set([])
    actionsrequiring_selections = set(['output','delete','info','add','remove'])
    inputformats = set(['uuid','sha512'])
    input_format_selected = set([])
    databaseConnectionString = None
    certDir = None
    logFile = None
    debugSqlEcho = False
    if 'VMCATCHER_RDBMS' in os.environ:
        databaseConnectionString = os.environ['VMCATCHER_RDBMS']
    if 'VMCATCHER_LOG_CONF' in os.environ:
        logFile = os.environ['VMCATCHER_LOG_CONF']
    if 'VMCATCHER_DIR_CERT' in os.environ:
        certDir = os.environ['VMCATCHER_DIR_CERT']
    
    
    # Set up log file
    LoggingLevel = logging.WARNING
    LoggingLevelCounter = 2
    if options.verbose:
        LoggingLevelCounter = LoggingLevelCounter - options.verbose
        if options.verbose == 1:
            LoggingLevel = logging.INFO
        if options.verbose == 2:
            LoggingLevel = logging.DEBUG
    if options.quiet:
        LoggingLevelCounter = LoggingLevelCounter + options.quiet
    if LoggingLevelCounter <= 0:
        LoggingLevel = logging.DEBUG
    if LoggingLevelCounter == 1:
        LoggingLevel = logging.INFO
    if LoggingLevelCounter == 2:
        LoggingLevel = logging.WARNING
    if LoggingLevelCounter == 3:
        LoggingLevel = logging.ERROR
    if LoggingLevelCounter == 4:
        LoggingLevel = logging.FATAL
    if LoggingLevelCounter >= 5:
        LoggingLevel = logging.CRITICAL
    
    if options.log_config:
        logFile = options.log_config
    if logFile != None:
        if os.path.isfile(str(options.log_config)):
            logging.config.fileConfig(options.log_config)
        else:
            logging.basicConfig(level=LoggingLevel)
            log = logging.getLogger("main")
            log.error("Logfile configuration file '%s' was not found." % (options.log_config))
            sys.exit(1)
    else:
        
        logging.basicConfig(level=LoggingLevel)
    log = logging.getLogger("main")
    # Now logging is set up process other options
    if options.log_sql_info:
        debugSqlEcho = True
    if options.cert_dir:
        certDir = options.cert_dir
    if options.list:
        actions.add('list')
    if options.sha512:
        images_selected = options.sha512
        input_format_selected.add('sha512')
    if options.uuid:
        images_selected = options.uuid
        input_format_selected.add('uuid')
    if options.info:
        actions.add('info')
        anchor_needed = True
    if options.output:
        format_needed = True
        outputfiles = options.output
    if options.add:
        actions.add('add')
    if options.remove:
        actions.add('remove')
    if options.database:
        databaseConnectionString = options.database
    # 1 So we have some command line validation

    if databaseConnectionString == None:
        databaseConnectionString = 'sqlite:///vmcatcher.db'
        log.info("Defaulting DB connection to '%s'" % (databaseConnectionString))
    if len(actions) == 0:
        log.error("No actions selected")
        sys.exit(1)
    if len(actions) > 1:
        log.error("More than one action selected.")
        sys.exit(1)
    if actions.issubset(actionsrequiring_selections) and len(images_selected) == 0:
        action_askedfor = actions.pop()
        log.error("Action '%s' requires an image to be selected." % action_askedfor)
        sys.exit(1)
    if format_needed and len(output_format_selected) == 0:
        log.error("No output format selected")
        sys.exit(1)

    if options.format:
        if options.format in outputformats:
            output_format_selected.add(options.format)
            anchor_needed = True
        else:
            log.error("Invalid format '%s' allowed formats are '%s'" % (options.format,outputformats))
            sys.exit(1)
    # 1 So we have some actions to process

    # 1.1 Initate DB
    database = db_controler(databaseConnectionString,debugSqlEcho)
    # 2 Initate CA's to manage files
    if anchor_needed:
        if certDir == None:
            certDir = "/etc/grid-security/certificates/"
            log.info("Defaulting Certificate directory to '%s'" % (certDir))
        database.setup_trust_anchor(certDir)
    # Handle conflicting identifiers
    selectors_types = inputformats.intersection(input_format_selected)
    selectors_types_len = len(selectors_types)
    if selectors_types_len > 1:
        log.error('Conflicting selectors.')
        sys.exit(1)
    selector_str = 'uuid'
    if selectors_types_len == 1:
        selector_str = selectors_types.pop()

    mapper = {'uuid' : queryby_uuid,
            'sha512' : queryby_sha512,
        }
    database.setup_selector_factory(mapper[selector_str])
    # Handle the output_view

    outputformats_selections = outputformats.intersection(output_format_selected)
    outputformats_selections_len = len(outputformats_selections)
    if outputformats_selections_len > 1:
        log.error('Conflicting output formats.')
        sys.exit(1)
    selector_str = 'lines'
    if outputformats_selections_len == 1:
        selector_str = outputformats_selections.pop()
    mapper = {'lines' : output_driver_lines,
        'SMIME' : output_driver_smime,
        'message' : output_driver_message,
    }
    database.setup_view_factory(mapper[selector_str])
    # Handle actions
    if 'subscribe' in actions:
        Session = SessionFactory()
        db = db_actions(Session)
        for uri in subscription_url_list:
            db.subscribe_file(anchor,uri)
    if 'list' in actions:
        database.image_list()
    if 'update' in actions:
        Session = SessionFactory()
        db = db_actions(Session)
        db.subscriptions_update(anchor)
    if 'dump' in actions:
        if not 'select' in actions:
            log.error('No subscriptions selected.')
        if 'json' in actions:
            pairs, extra_uuid ,extra_paths = pairsNnot(images_selected,messages_path)
            if len(extra_paths) > 0:
                log.warning('Extra paths will be ignored.')
                for path in extra_paths:
                    log.info('ignoring path %s' % (path))
            if len(extra_uuid) > 0:
                log.warning('sha512 ignored.')
                for path in extra_uuid:
                    log.info('ignoring sha512 %s' % (path))
            Session = SessionFactory()
            db = db_actions(Session)
            for item in pairs:
                db.image_by_sha512_writefile_json(anchor,item[0],item[1])
        if 'message' in actions:
            pairs, extra_uuid ,extra_paths = pairsNnot(images_selected,messages_path)
            if len(extra_paths) > 0:
                log.warning('Extra paths will be ignored.')
                for path in extra_paths:
                    log.info('ignoring path %s' % (path))
            if len(extra_uuid) > 0:
                log.warning('sha512 ignored.')
                for path in extra_uuid:
                    log.info('Ignoring sha512 %s' % (path))

            Session = SessionFactory()
            db = db_actions(Session)
            for item in pairs:
                db.image_by_sha512_writefile_imagelist(anchor,item[0],item[1])
    if 'info' in actions:
        if not database.images_info(images_selected,outputfiles):
            sys.exit(1)

    if 'add' in actions:
        database.images_subscribe(images_selected,1)
    if 'remove' in actions:
        database.images_subscribe(images_selected,0)
if __name__ == "__main__":
    main()