sample-lstm2.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"sample-lstm2.ipynb","version":"0.3.2","provenance":[],"collapsed_sections":[]},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.6"},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"metadata":{"scrolled":true,"id":"HtYHjm0FuK9X","colab_type":"code","outputId":"05e5f210-87e7-4752-ce88-45d75f32a3e1","executionInfo":{"status":"ok","timestamp":1556272367455,"user_tz":-330,"elapsed":3777,"user":{"displayName":"Chandeesh Kumar","photoUrl":"https://lh4.googleusercontent.com/--MT8mrFpMqE/AAAAAAAAAAI/AAAAAAAAP2E/43g8ggDCRdk/s64/photo.jpg","userId":"17069823236948718831"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"cell_type":"code","source":["from numpy import array\n","from pickle import load,dump\n","import pandas as pd\n","from keras.preprocessing.text import Tokenizer\n","from keras.preprocessing.sequence import pad_sequences\n","from keras.utils import to_categorical\n","from keras.utils import plot_model\n","from keras.models import Model\n","from keras.layers import Input\n","from keras.layers import Dense\n","from keras.layers import LSTM\n","from keras.layers import Embedding\n","from keras.layers import Dropout\n","from keras.layers import Bidirectional\n","from keras.layers import *\n","from keras.layers.merge import add\n","from keras.callbacks import ModelCheckpoint\n","import pydot\n","from numpy import argmax\n","from nltk.translate.bleu_score import corpus_bleu"],"execution_count":1,"outputs":[{"output_type":"stream","text":["Using TensorFlow backend.\n"],"name":"stderr"}]},{"metadata":{"id":"2oQm2dOGvunN","colab_type":"code","outputId":"4a99a958-ed79-4282-b8a8-97b074888303","executionInfo":{"status":"ok","timestamp":1556272397201,"user_tz":-330,"elapsed":23844,"user":{"displayName":"Chandeesh Kumar","photoUrl":"https://lh4.googleusercontent.com/--MT8mrFpMqE/AAAAAAAAAAI/AAAAAAAAP2E/43g8ggDCRdk/s64/photo.jpg","userId":"17069823236948718831"}},"colab":{"base_uri":"https://localhost:8080/","height":122}},"cell_type":"code","source":["# Load the Drive helper and mount\n","from google.colab import drive\n","\n","# This will prompt for authorization.\n","drive.mount('/content/drive')"],"execution_count":2,"outputs":[{"output_type":"stream","text":["Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n","\n","Enter your authorization code:\n","··········\n","Mounted at /content/drive\n"],"name":"stdout"}]},{"metadata":{"scrolled":true,"id":"eo4QCpfauK9g","colab_type":"code","outputId":"40bdd08d-7316-4acc-fa70-344ab908312f","executionInfo":{"status":"ok","timestamp":1556272427167,"user_tz":-330,"elapsed":3925,"user":{"displayName":"Chandeesh Kumar","photoUrl":"https://lh4.googleusercontent.com/--MT8mrFpMqE/AAAAAAAAAAI/AAAAAAAAP2E/43g8ggDCRdk/s64/photo.jpg","userId":"17069823236948718831"}},"colab":{"base_uri":"https://localhost:8080/","height":102}},"cell_type":"code","source":["# load doc into memory\n","def load_doc(filename):\n","\t# open the file as read only\n","\tfile = open(filename, 'r')\n","\t# read all text\n","\ttext = file.read()\n","\t# close the file\n","\tfile.close()\n","\treturn text\n"," \n","# load a pre-defined list of photo identifiers\n","def load_set(filename):\n","\tdoc = load_doc(filename)\n","\tdataset = list()\n","\t# process line by line\n","\tfor line in doc.split('\\n'):\n","\t\t# skip empty lines\n","\t\tif len(line) < 1:\n","\t\t\tcontinue\n","\t\t# get the image identifier\n","\t\tidentifier = line.split(' ')[0]\n","\t\tdataset.append(identifier)\n","\treturn set(dataset)\n"," \n","# load clean descriptions into memory\n","def load_clean_descriptions(filename, dataset):\n","\t# load document\n","\tdoc = load_doc(filename)\n","\tdescriptions = dict()\n","\tfor line in doc.split('\\n'):\n","\t\t# split line by white space\n","\t\ttokens = line.split()\n","\t\t# split id from description\n","\t\timage_id, image_desc = tokens[0], tokens[1:]\n","\t\t# skip images not in the set\n","\t\tif image_id in dataset:\n","\t\t\t# create list\n","\t\t\tif image_id not in descriptions:\n","\t\t\t\tdescriptions[image_id] = list()\n","\t\t\t# wrap description in tokens\n","\t\t\tdesc = 'startseq ' + ' '.join(image_desc) + ' endseq'\n","\t\t\t# store\n","\t\t\tdescriptions[image_id].append(desc)\n","\treturn descriptions\n"," \n","# load photo features\n","def load_photo_features(filename, dataset):\n","    # load all features\n","    all_features = load(open(filename, 'rb'))\n","    #print(all_features)\n","    # filter features\n","    features = {k: all_features[k] for k in dataset}\n","    return features\n"," \n","# covert a dictionary of clean descriptions to a list of descriptions\n","def to_lines(descriptions):\n","\tall_desc = list()\n","\tfor key in descriptions.keys():\n","\t\t[all_desc.append(d) for d in descriptions[key]]\n","\treturn all_desc\n"," \n","# fit a tokenizer given caption descriptions\n","def create_tokenizer(descriptions):\n","\tlines = to_lines(descriptions)\n","\ttokenizer = Tokenizer()\n","\ttokenizer.fit_on_texts(lines)\n","\treturn tokenizer\n"," \n","# calculate the length of the description with the most words\n","def max_length(descriptions):\n","\tlines = to_lines(descriptions)\n","\treturn max(len(d.split()) for d in lines)\n"," \n","# create sequences of images, input sequences and output words for an image\n","def create_sequences(domain, tokenizer, max_length, desc_list, photo):\n","    X1, X2, X3, y = list(), list(), list(), list()\n","    # walk through each description for the image\n","    for desc in desc_list:\n","        # encode the sequence\n","        seq = tokenizer.texts_to_sequences([desc])[0]\n","        # split one sequence into multiple X,y pairs\n","        for i in range(1, len(seq)):\n","            # split into input and output pair\n","            in_seq, out_seq = seq[:i], seq[i]\n","            # pad input sequence\n","            in_seq = pad_sequences([in_seq], maxlen=max_length)[0]\n","            # encode output sequence\n","            out_seq = to_categorical([out_seq], num_classes=vocab_size)[0]\n","            # store\n","            X1.append(photo)\n","            X2.append(domain)\n","            X3.append(in_seq)\n","            y.append(out_seq)\n","    return array(X1), array(X2), array(X3), array(y)\n"," \n","#data generator, intended to be used in a call to model.fit_generator()\n","def data_generator(domain_features, descriptions, photos, tokenizer, max_length):\n","    # loop for ever over images\n","    while 1:\n","        for key, desc_list in descriptions.items():\n","            # retrieve the photo feature\n","            domain = domain_features[key]\n","            photo = photos[key][0]\n","            in_img, in_domain, in_seq, out_word = create_sequences(domain, tokenizer, max_length, desc_list, photo)\n","            yield [[in_img, in_domain, in_seq], out_word]\n","\n","# load training dataset (6K)\n","filename = 'drive/My Drive/Final Year Project/Descriptions/trainuc.txt'\n","train = load_set(filename)\n","#print(sorted(train))\n","print('Dataset: %d' % len(train))\n","# descriptions\n","train_descriptions = load_clean_descriptions('drive/My Drive/Final Year Project/Descriptions/trainuc.txt', train)\n","print('Descriptions: train=%d' % len(train_descriptions))\n","# photo features\n","#print(train)\n","#all_features = load(open('features_inceptionv3_uc.pkl', 'rb'))\n","#print(all_features)\n","\n","train_features = load_photo_features('drive/My Drive/Final Year Project/Features/features_uc_resnet152_updated.pkl', train)\n","print('Photos: train=%d' % len(train_features))\n","# prepare tokenizer\n","tokenizer = create_tokenizer(train_descriptions)\n","#dump(tokenizer,open('tokenizer_resnet152.pkl','wb'))\n","vocab_size = len(tokenizer.word_index) + 1\n","print('Vocabulary Size: %d' % vocab_size)\n","## determine the maximum sequence length\n","max_length = max_length(train_descriptions)\n","print('Description Length: %d' % max_length)\n"," "],"execution_count":4,"outputs":[{"output_type":"stream","text":["Dataset: 1260\n","Descriptions: train=1260\n","Photos: train=1260\n","Vocabulary Size: 293\n","Description Length: 24\n"],"name":"stdout"}]},{"metadata":{"id":"o2TJ2Kk4uK9k","colab_type":"code","outputId":"f1facb01-9f29-40ca-be9c-373ab510deed","colab":{}},"cell_type":"code","source":["features=load(open('features_resnet152_newids.pkl','rb'))\n","keys  = features.keys()\n","l = list()\n","for i in train:\n","    if i not in keys:\n","        #print(i)\n","        l.append(i)\n","#print(sorted(l))\n","print(len(l))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["1260\n"],"name":"stdout"}]},{"metadata":{"id":"7AWArE9ruK9o","colab_type":"code","colab":{}},"cell_type":"code","source":["# map an integer to a word\n","def word_for_id(integer, tokenizer):\n","\tfor word, index in tokenizer.word_index.items():\n","\t\tif index == integer:\n","\t\t\treturn word\n","\treturn None\n"," \n","# generate a description for an image\n","def generate_desc(domain, model, tokenizer, photo, max_length):\n","    # seed the generation process\n","    in_text = 'startseq'\n","    # iterate over the whole length of the sequence\n","    for i in range(max_length):\n","        # integer encode input sequence\n","        sequence = tokenizer.texts_to_sequences([in_text])[0]\n","        # pad input\n","        domain1 = np.array([domain])\n","        sequence = pad_sequences([sequence], maxlen=max_length)\n","        # predict next word\n","        yhat = model.predict([photo,domain1,sequence], verbose=0)\n","        # convert probability to integer\n","        yhat = argmax(yhat)\n","        # map integer to word\n","        word = word_for_id(yhat, tokenizer)\n","        # stop if we cannot map the word\n","        if word is None:\n","            break\n","        # append as input for generating the next word\n","        in_text += ' ' + word\n","        # stop if we predict the end of the sequence\n","        if word == 'endseq':\n","            break\n","    return in_text\n"," \n","# evaluate the skill of the model\n","def evaluate_model(domain, model, descriptions, photos, tokenizer, max_length,filename):\n","    actual, predicted = list(), list()\n","    # step over the whole set\n","    lines = list()\n","    for key, desc_list in descriptions.items():\n","        yhat = generate_desc(domain[key], model, tokenizer, photos[key], max_length)\n","        ex=yhat\n","        a=yhat.split('startseq')\n","        b=a[1].split('endseq')\n","        lines.append('beam_size_1'+'\\t'+key + '\\t' + b[0])\n","        references = [d.split() for d in desc_list]\n","        actual.append(references)\n","        predicted.append(yhat.split())\n","        #\n","    data = '\\n'.join(lines)\n","    file = open(filename, 'w')\n","    file.write(data)\n","    file.close()\n","    bleu=np.zeros(4)\n","    # calculate BLEU score\n","    bleu[0]=corpus_bleu(actual, predicted, weights=(1.0, 0, 0, 0))\n","    bleu[1]=corpus_bleu(actual, predicted, weights=(0.5, 0.5, 0, 0))\n","    bleu[2]=corpus_bleu(actual, predicted, weights=(0.3, 0.3, 0.3, 0))\n","    bleu[3]=corpus_bleu(actual, predicted, weights=(0.25, 0.25, 0.25, 0.25))\n","    print('BLEU-1: %f' % bleu[0])\n","    print('BLEU-2: %f' % bleu[1])\n","    print('BLEU-3: %f' % bleu[2])\n","    print('BLEU-4: %f' % bleu[3])\n","    return bleu"],"execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","outputId":"2a76680f-b6b5-44ca-bd2a-819948cd1e7b","executionInfo":{"status":"ok","timestamp":1556272450620,"user_tz":-330,"elapsed":1600,"user":{"displayName":"Chandeesh Kumar","photoUrl":"https://lh4.googleusercontent.com/--MT8mrFpMqE/AAAAAAAAAAI/AAAAAAAAP2E/43g8ggDCRdk/s64/photo.jpg","userId":"17069823236948718831"}},"id":"hvlCi3wHBO05","colab":{"base_uri":"https://localhost:8080/","height":68}},"cell_type":"code","source":["# load test set\n","filename = 'drive/My Drive/Final Year Project/Descriptions/testuc.txt'\n","test = load_set(filename)\n","print('Dataset: %d' % len(test))\n","# descriptions\n","test_descriptions = load_clean_descriptions('drive/My Drive/Final Year Project/Descriptions/testuc.txt', test)\n","print('Descriptions: test=%d' % len(test_descriptions))\n","# photo features\n","#print(test)\n","test_features = load_photo_features('drive/My Drive/Final Year Project/Features/features_uc_resnet152_updated.pkl', test)\n","print('Photos: test=%d' % len(test_features))"],"execution_count":6,"outputs":[{"output_type":"stream","text":["Dataset: 420\n","Descriptions: test=420\n","Photos: test=420\n"],"name":"stdout"}]},{"metadata":{"id":"k9m_DaZguK9x","colab_type":"code","outputId":"5beab32d-625e-4c96-f0ae-4f3d5e0aacad","colab":{}},"cell_type":"code","source":["features = load(open('features_resnet152_newids.pkl','rb'))\n","keys = features.keys()\n","for i in test:\n","    if i not in keys:\n","        print(i)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["13_080\n","03_061\n","18_078\n","11_078\n","13_075\n","05_066\n","15_076\n","12_069\n","18_079\n","15_062\n","18_064\n","05_076\n","00_063\n","01_067\n","09_078\n","10_067\n","16_080\n","18_066\n","10_076\n","01_061\n","09_070\n","16_078\n","13_079\n","02_072\n","00_074\n","04_065\n","08_069\n","17_070\n","05_079\n","09_062\n","07_077\n","02_074\n","01_074\n","10_064\n","07_078\n","13_061\n","08_075\n","15_067\n","18_076\n","03_067\n","04_073\n","02_075\n","09_074\n","09_063\n","11_065\n","16_076\n","19_071\n","19_074\n","02_065\n","13_063\n","02_079\n","19_066\n","14_075\n","16_063\n","12_066\n","19_075\n","15_066\n","11_079\n","04_070\n","09_067\n","11_080\n","14_070\n","05_061\n","08_068\n","08_064\n","11_070\n","16_069\n","10_079\n","18_077\n","20_077\n","07_067\n","07_076\n","01_070\n","11_061\n","04_071\n","04_062\n","00_061\n","13_077\n","17_072\n","18_073\n","15_075\n","06_070\n","00_077\n","14_073\n","08_072\n","19_076\n","06_065\n","03_078\n","02_080\n","05_070\n","09_061\n","09_075\n","16_066\n","17_077\n","04_067\n","12_073\n","05_077\n","00_070\n","02_069\n","19_069\n","08_076\n","13_074\n","20_063\n","12_063\n","16_074\n","06_071\n","03_080\n","04_077\n","02_073\n","14_078\n","07_065\n","14_068\n","02_062\n","03_071\n","10_070\n","19_077\n","00_067\n","05_069\n","10_080\n","03_072\n","03_062\n","19_061\n","03_079\n","12_076\n","11_071\n","07_080\n","03_073\n","19_078\n","05_075\n","09_064\n","10_075\n","11_063\n","15_074\n","18_072\n","02_070\n","07_074\n","04_080\n","14_080\n","07_071\n","07_068\n","10_073\n","20_076\n","12_072\n","01_075\n","12_070\n","10_074\n","10_072\n","14_076\n","12_080\n","12_079\n","13_064\n","19_073\n","09_066\n","09_068\n","12_068\n","20_064\n","07_079\n","15_071\n","09_080\n","04_064\n","20_066\n","00_065\n","15_068\n","17_073\n","15_064\n","06_080\n","03_069\n","19_063\n","05_063\n","00_069\n","18_074\n","08_078\n","19_080\n","00_080\n","11_069\n","14_079\n","20_069\n","04_068\n","20_073\n","16_062\n","06_077\n","08_067\n","10_063\n","18_062\n","00_073\n","20_065\n","16_072\n","00_075\n","14_062\n","10_065\n","12_062\n","20_068\n","10_069\n","13_070\n","17_076\n","03_068\n","13_066\n","08_062\n","16_071\n","04_069\n","13_076\n","00_071\n","10_068\n","02_077\n","08_071\n","06_062\n","15_063\n","09_077\n","08_077\n","01_073\n","09_076\n","05_068\n","20_075\n","00_072\n","06_072\n","20_078\n","19_067\n","18_070\n","04_063\n","15_079\n","17_068\n","04_078\n","03_065\n","13_073\n","16_075\n","19_072\n","14_077\n","03_075\n","01_079\n","17_074\n","04_076\n","17_064\n","01_077\n","05_078\n","06_061\n","08_061\n","12_077\n","02_064\n","20_080\n","18_068\n","05_073\n","06_073\n","03_077\n","14_071\n","17_075\n","05_074\n","20_072\n","01_065\n","02_061\n","11_067\n","14_072\n","09_065\n","01_069\n","14_065\n","20_070\n","01_072\n","07_069\n","04_072\n","07_063\n","11_068\n","16_070\n","11_073\n","16_064\n","18_075\n","18_080\n","13_071\n","10_077\n","07_062\n","02_076\n","14_066\n","02_066\n","18_061\n","00_066\n","03_063\n","06_063\n","01_064\n","17_080\n","17_079\n","17_063\n","02_068\n","06_067\n","20_062\n","15_061\n","09_072\n","03_074\n","09_071\n","01_080\n","11_075\n","04_075\n","10_071\n","11_072\n","08_065\n","06_068\n","06_064\n","02_063\n","19_070\n","14_061\n","12_064\n","18_071\n","14_063\n","11_077\n","07_072\n","02_071\n","00_079\n","05_062\n","00_078\n","07_070\n","15_070\n","13_068\n","13_069\n","07_064\n","12_074\n","07_066\n","14_074\n","06_076\n","02_067\n","18_067\n","03_066\n","15_069\n","01_078\n","20_074\n","13_078\n","19_079\n","05_065\n","04_061\n","11_066\n","07_073\n","00_062\n","10_061\n","15_077\n","05_067\n","06_074\n","01_071\n","17_067\n","00_076\n","05_080\n","15_080\n","09_069\n","08_080\n","16_061\n","16_077\n","11_062\n","06_078\n","11_076\n","01_076\n","19_068\n","17_062\n","17_061\n","02_078\n","03_070\n","11_064\n","03_064\n","12_065\n","12_078\n","07_075\n","17_071\n","10_062\n","12_067\n","01_068\n","08_066\n","14_064\n","18_069\n","19_064\n","12_061\n","18_063\n","20_079\n","01_063\n","00_068\n","20_061\n","06_075\n","13_067\n","10_078\n","15_072\n","00_064\n","15_065\n","17_069\n","09_079\n","13_065\n","12_075\n","15_073\n","04_066\n","15_078\n","11_074\n","07_061\n","04_079\n","14_067\n","17_065\n","20_067\n","19_065\n","06_069\n","05_071\n","16_068\n","03_076\n","16_079\n","12_071\n","08_070\n","05_072\n","09_073\n","16_073\n","06_066\n","19_062\n","14_069\n","18_065\n","05_064\n","16_067\n","01_066\n","06_079\n","16_065\n","17_066\n","13_072\n","17_078\n","20_071\n","04_074\n","01_062\n","10_066\n","08_079\n","08_063\n","08_074\n","08_073\n","13_062\n"],"name":"stdout"}]},{"metadata":{"id":"mCOfGXcuuK90","colab_type":"code","colab":{}},"cell_type":"code","source":["from keras import backend as K\n","from keras.engine.topology import Layer\n","from keras import initializers, regularizers, constraints\n","\n","\n","class Attention(Layer):\n","    def __init__(self, step_dim,\n","                 W_regularizer=None, b_regularizer=None,\n","                 W_constraint=None, b_constraint=None,\n","                 bias=True, **kwargs):\n","        self.supports_masking = True\n","        self.init = initializers.get('glorot_uniform')\n","\n","        self.W_regularizer = regularizers.get(W_regularizer)\n","        self.b_regularizer = regularizers.get(b_regularizer)\n","\n","        self.W_constraint = constraints.get(W_constraint)\n","        self.b_constraint = constraints.get(b_constraint)\n","\n","        self.bias = bias\n","        self.step_dim = step_dim\n","        self.features_dim = 0\n","        super(Attention, self).__init__(**kwargs)\n","\n","    def build(self, input_shape):\n","        assert len(input_shape) == 3\n","\n","        self.W = self.add_weight((input_shape[-1],),\n","                                 initializer=self.init,\n","                                 name='{}_W'.format(self.name),\n","                                 regularizer=self.W_regularizer,\n","                                 constraint=self.W_constraint)\n","        self.features_dim = input_shape[-1]\n","\n","        if self.bias:\n","            self.b = self.add_weight((input_shape[1],),\n","                                     initializer='zero',\n","                                     name='{}_b'.format(self.name),\n","                                     regularizer=self.b_regularizer,\n","                                     constraint=self.b_constraint)\n","        else:\n","            self.b = None\n","\n","        self.built = True\n","\n","    def compute_mask(self, input, input_mask=None):\n","        return None\n","\n","    def call(self, x, mask=None):\n","        features_dim = self.features_dim\n","        step_dim = self.step_dim\n","\n","        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n","                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n","\n","        if self.bias:\n","            eij += self.b\n","\n","        eij = K.tanh(eij)\n","\n","        a = K.exp(eij)\n","\n","        if mask is not None:\n","            a *= K.cast(mask, K.floatx())\n","\n","        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n","\n","        a = K.expand_dims(a)\n","        weighted_input = x * a\n","        return K.sum(weighted_input, axis=1)\n","\n","    def compute_output_shape(self, input_shape):\n","        return input_shape[0],  self.features_dim\n"],"execution_count":0,"outputs":[]},{"metadata":{"id":"iZz12TaouK94","colab_type":"code","colab":{}},"cell_type":"code","source":["# load a word embedding\n","def load_embedding(tokenizer, vocab_size, max_length):\n","\t# load the tokenizer\n","\tembedding = load(open('UCMDataset1/Features/word2vec_embedding.pkl', 'rb'))\n","\tdimensions = 100\n","\ttrainable = False\n","\t# create a weight matrix for words in training docs\n","\tweights = np.zeros((vocab_size, dimensions))\n","\t# walk words in order of tokenizer vocab to ensure vectors are in the right index\n","\tfor word, i in tokenizer.word_index.items():\n","\t\tif word not in embedding:\n","\t\t\tcontinue\n","\t\tweights[i] = embedding[word]\n","\tlayer = Embedding(vocab_size, dimensions, weights=[weights], input_length=max_length, trainable=trainable, mask_zero=True)\n","\treturn layer"],"execution_count":0,"outputs":[]},{"metadata":{"id":"U4Kjr8zpuK9-","colab_type":"code","colab":{}},"cell_type":"code","source":["# define the captioning model\n","loc = 'drive/My Drive/Final Year Project/Modelcall/'\n","def define_model(vocab_size, max_length):\n","    # feature extractor model\n","    inputs1 = Input(shape=(2048,))\n","    #fe1 = Dropout(0.5)(inputs1)\n","    fe2 = Dense(256, activation='relu')(inputs1)\n","    \n","    inputs2 = Input(shape=(21,))\n","    merged = concatenate([fe2,inputs2])\n","    \n","    fe3 = RepeatVector(max_length)(merged)\n","    \n","    # sequence model\n","    inputs3 = Input(shape=(max_length,))\n","    #se1 = load_embedding(tokenizer, vocab_size, max_length)(inputs3)\n","    se1 = Embedding(vocab_size, 256, mask_zero=True)(inputs3)\n","    #print(se1.shape)\n","    se2 = LSTM(256,return_sequences=True)(se1)\n","    #print(se2.shape)\n","    #x = Attention(max_length)(se2) \n","    #print(x.shape)\n","    se3 = TimeDistributed(Dense(256,activation='relu'))(se2)\n","    #print(se3.shape)\n","    \n","    # decoder model\n","    decoder1 = concatenate([fe3, se3])\n","    decoder2 = Bidirectional(LSTM(150,return_sequences=True))(decoder1)\n","    decoder3 = Attention(max_length)(decoder2)\n","    outputs = Dense(vocab_size, activation='softmax')(decoder3)\n","    # tie it together [image, seq] [word]\n","    model = Model(inputs=[inputs1, inputs2, inputs3], outputs=outputs)\n","    # compile model\n","    model.compile(loss='categorical_crossentropy', optimizer='adam')\n","    # summarize model\n","    model.summary()\n","    plot_model(model, to_file=loc+'model.png', show_shapes=True)\n","    return model"],"execution_count":0,"outputs":[]},{"metadata":{"id":"lMFx02ZSuK-C","colab_type":"code","outputId":"51fe1c07-0e9c-4f77-d1c2-43333467fcd1","executionInfo":{"status":"ok","timestamp":1556272509472,"user_tz":-330,"elapsed":2674,"user":{"displayName":"Chandeesh Kumar","photoUrl":"https://lh4.googleusercontent.com/--MT8mrFpMqE/AAAAAAAAAAI/AAAAAAAAP2E/43g8ggDCRdk/s64/photo.jpg","userId":"17069823236948718831"}},"colab":{"base_uri":"https://localhost:8080/","height":612}},"cell_type":"code","source":["model = define_model(vocab_size, max_length)"],"execution_count":13,"outputs":[{"output_type":"stream","text":["__________________________________________________________________________________________________\n","Layer (type)                    Output Shape         Param #     Connected to                     \n","==================================================================================================\n","input_7 (InputLayer)            (None, 2048)         0                                            \n","__________________________________________________________________________________________________\n","input_9 (InputLayer)            (None, 24)           0                                            \n","__________________________________________________________________________________________________\n","dense_7 (Dense)                 (None, 256)          524544      input_7[0][0]                    \n","__________________________________________________________________________________________________\n","input_8 (InputLayer)            (None, 21)           0                                            \n","__________________________________________________________________________________________________\n","embedding_3 (Embedding)         (None, 24, 256)      75008       input_9[0][0]                    \n","__________________________________________________________________________________________________\n","concatenate_5 (Concatenate)     (None, 277)          0           dense_7[0][0]                    \n","                                                                 input_8[0][0]                    \n","__________________________________________________________________________________________________\n","lstm_5 (LSTM)                   (None, 24, 256)      525312      embedding_3[0][0]                \n","__________________________________________________________________________________________________\n","repeat_vector_3 (RepeatVector)  (None, 24, 277)      0           concatenate_5[0][0]              \n","__________________________________________________________________________________________________\n","time_distributed_3 (TimeDistrib (None, 24, 256)      65792       lstm_5[0][0]                     \n","__________________________________________________________________________________________________\n","concatenate_6 (Concatenate)     (None, 24, 533)      0           repeat_vector_3[0][0]            \n","                                                                 time_distributed_3[0][0]         \n","__________________________________________________________________________________________________\n","bidirectional_3 (Bidirectional) (None, 24, 300)      820800      concatenate_6[0][0]              \n","__________________________________________________________________________________________________\n","attention_3 (Attention)         (None, 300)          324         bidirectional_3[0][0]            \n","__________________________________________________________________________________________________\n","dense_9 (Dense)                 (None, 293)          88193       attention_3[0][0]                \n","==================================================================================================\n","Total params: 2,099,973\n","Trainable params: 2,099,973\n","Non-trainable params: 0\n","__________________________________________________________________________________________________\n"],"name":"stdout"}]},{"metadata":{"id":"caIH9ThQuK-G","colab_type":"code","colab":{}},"cell_type":"code","source":["embedding = load(open('UCMDataset1/Features/word2vec_embedding.pkl', 'rb'))\n","embedding"],"execution_count":0,"outputs":[]},{"metadata":{"id":"n0LjHWyYuK-L","colab_type":"code","outputId":"9ba8cb08-5db7-491f-b1f7-66bf56af3a1f","executionInfo":{"status":"error","timestamp":1556007837240,"user_tz":-330,"elapsed":5037,"user":{"displayName":"Chandeesh Kumar","photoUrl":"https://lh4.googleusercontent.com/--MT8mrFpMqE/AAAAAAAAAAI/AAAAAAAAP2E/43g8ggDCRdk/s64/photo.jpg","userId":"17069823236948718831"}},"colab":{"base_uri":"https://localhost:8080/","height":180}},"cell_type":"code","source":["domain_features = load(open('drive/My Drive/Final Year Project/Features/domain_features.pkl', 'rb'))\n","print(domain_features['100_077'].shape)"],"execution_count":0,"outputs":[{"output_type":"error","ename":"KeyError","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)","\u001b[0;32m<ipython-input-7-477a005bc1ad>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mdomain_features\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'drive/My Drive/Final Year Project/Features/domain_features.pkl'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdomain_features\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'100_077'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;31mKeyError\u001b[0m: '100_077'"]}]},{"metadata":{"id":"ZzUE-B5zuK-P","colab_type":"code","outputId":"79cbe096-c4e7-43a8-b75e-775de13edfa7","colab":{"base_uri":"https://localhost:8080/","height":734}},"cell_type":"code","source":["## train the model, run epochs manually and save after each epoch\n","epochs = 20\n","steps = len(train_descriptions)\n","mylist = list()\n","df=pd.DataFrame(index=mylist, columns=['model_no','Bleu_1','Bleu_2','Bleu_3','Bleu_4'])\n","for i in range(1,epochs):\n","    print(i)\n","    # create the data generator\n","    generator = data_generator(domain_features, train_descriptions, train_features, tokenizer, max_length)\n","    # fit for one epoch\n","    model.fit_generator(generator, epochs=1, steps_per_epoch=steps, verbose=1)\n","    # save model\n","    model.save(loc+'model_' + str(i) + '.h5')\n","    # load the mode    \n","    filename = loc+'result/text_predicted_' + str(i) + '.txt'\n","    # evaluate model\n","    bleu = evaluate_model(domain_features, model, test_descriptions, test_features, tokenizer, max_length,filename)\n","    #bleu = evaluate_model(model, test_descriptions, test_features,tokenizer, max_length,filename_new)\n","    df.at[i,'model_no']=i\n","    df.at[i,'Bleu_1']=bleu[0]\n","    df.at[i,'Bleu_2']=bleu[1]\n","    df.at[i,'Bleu_3']=bleu[2]\n","    df.at[i,'Bleu_4']=bleu[3]\n","    df.to_csv(loc+'result/results.csv', index=False)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["1\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use tf.cast instead.\n","Epoch 1/1\n","1260/1260 [==============================] - 293s 233ms/step - loss: 3.0170\n","BLEU-1: 0.653435\n","BLEU-2: 0.494737\n","BLEU-3: 0.428642\n","BLEU-4: 0.314513\n","2\n","Epoch 1/1\n","1260/1260 [==============================] - 525s 417ms/step - loss: 1.4429\n","BLEU-1: 0.805743\n","BLEU-2: 0.713885\n","BLEU-3: 0.668995\n","BLEU-4: 0.576434\n","3\n","Epoch 1/1\n","1260/1260 [==============================] - 444s 353ms/step - loss: 0.7590\n","BLEU-1: 0.809448\n","BLEU-2: 0.718706\n","BLEU-3: 0.676294\n","BLEU-4: 0.586651\n","4\n","Epoch 1/1\n","1260/1260 [==============================] - 435s 345ms/step - loss: 0.5573\n","BLEU-1: 0.804887\n","BLEU-2: 0.721594\n","BLEU-3: 0.687419\n","BLEU-4: 0.606165\n","5\n","Epoch 1/1\n","1260/1260 [==============================] - 434s 345ms/step - loss: 0.4668\n","BLEU-1: 0.802764\n","BLEU-2: 0.717660\n","BLEU-3: 0.681599\n","BLEU-4: 0.599443\n","6\n","Epoch 1/1\n"," 236/1260 [====>.........................] - ETA: 6:56 - loss: 0.4360"],"name":"stdout"}]},{"metadata":{"id":"Lo6LI-cTuK-T","colab_type":"code","colab":{}},"cell_type":"code","source":["## from keras.models import load_model\n","filename = 'UCMDataset/text_predicted.txt'\n","model = model.load_weights('Modelnewdataall/model_1.h5')\n","evaluate_model(model, test_descriptions, test_features, tokenizer, max_length,filename)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"RMOASgS6uK-W","colab_type":"code","colab":{}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]}]}