creating-the-model (1).ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import struct\nimport numpy as np\nfrom keras.layers import Conv2D\nfrom keras.layers import Input\nfrom keras.layers import BatchNormalization\nfrom keras.layers import LeakyReLU,Dense\nfrom keras.layers import ZeroPadding2D,Flatten\nfrom keras.layers import UpSampling2D,Lambda\n\nfrom keras.layers import add, Concatenate,GlobalAveragePooling2D,Softmax\nfrom keras.models import Model\nimport tensorflow as tf\nfrom keras.applications import vgg19\nimport tensorflow_datasets as tfds\nimport os\nimport cv2 \nfrom pathlib import Path\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nimport pickle\nimport seaborn as sns\nimport matplotlib.image as mpimg\nimport shutil\nimport torch\nimport torch.nn as nn\nimport time\n\nimport shutil\nshutil.rmtree('/your/folder/path/')\nshutil.copytree('../input/chechpoint-of-sisr', './checkpoint_dir')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-10T04:05:21.10271Z","iopub.execute_input":"2022-01-10T04:05:21.103195Z","iopub.status.idle":"2022-01-10T04:05:27.860917Z","shell.execute_reply.started":"2022-01-10T04:05:21.103147Z","shell.execute_reply":"2022-01-10T04:05:27.860268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_paths_to_HR_images= ['../input/patches/train/HR/'+name for name in os.listdir('../input/patches/train/HR')]\ntrain_paths_to_LR_images= ['../input/patches/train/LR/'+name for name in os.listdir('../input/patches/train/LR')]\n\ntest_paths_to_HR_images= ['../input/patches/test/HR/'+name for name in os.listdir('../input/patches/test/HR/')]\ntest_paths_to_LR_images= ['../input/patches/test/LR/'+name for name in os.listdir('../input/patches/test/LR/')]","metadata":{"execution":{"iopub.status.busy":"2022-01-10T04:05:43.072305Z","iopub.execute_input":"2022-01-10T04:05:43.072711Z","iopub.status.idle":"2022-01-10T04:05:43.134477Z","shell.execute_reply.started":"2022-01-10T04:05:43.072667Z","shell.execute_reply":"2022-01-10T04:05:43.13344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# visualizing the patches","metadata":{}},{"cell_type":"markdown","source":"# Creating the pipeline","metadata":{}},{"cell_type":"code","source":"def read(path_HR,path_LR):\n\n    # 1)reading the content of the path\n\n     # 3)Normalizing the image\n\n    # 1)reading the content of the path\n    content_HR=tf.io.read_file(path_HR)\n    content_LR=tf.io.read_file(path_LR)\n     \n    # 2)decoding the content\n    image_HR= tf.cast(tf.io.decode_png(content_HR,channels=3),dtype= tf.float32)\n    image_LR= tf.io.decode_png(content_LR,channels=3)\n     \n    #The pixel value of HR images should be between [-1,1]\n    image_HR= image_HR*2.0/255.0-1.0\n\n    #The pixel value for LR images should be between [0,1]\n    image_LR = tf.image.convert_image_dtype(image_LR, tf.float32)\n    \n\n    return image_HR,image_LR\n\ndef pipeline(HR_pat,LR_pat):\n\n   dataset=tf.data.Dataset.from_tensor_slices((HR_pat,LR_pat))\n    \n   \n   #reading the images.....................................................\n   dataset=dataset.map(read,num_parallel_calls=tf.data.experimental.AUTOTUNE)\n\n   #Creating the batch\n   dataset=dataset.batch(batch_size=batch,drop_remainder=True)\n\n   #All the batches will be stored in the cache after the first iteration\n   dataset.cache()\n\n   dataset.prefetch(tf.data.experimental.AUTOTUNE)   \n   return dataset\n\n\n# #Dataset Parameter\n\n# train_dataset= pipeline(train_paths_to_HR_images[0:100],train_paths_to_LR_images[0:100])\n\n# for HR,LR in train_dataset:\n    \n#     plt.figure(figsize=[8,6])\n#     imgplot = plt.imshow(HR[0])\n#     plt.show()\n#     break","metadata":{"execution":{"iopub.status.busy":"2022-01-10T04:05:43.852268Z","iopub.execute_input":"2022-01-10T04:05:43.852526Z","iopub.status.idle":"2022-01-10T04:05:43.864609Z","shell.execute_reply.started":"2022-01-10T04:05:43.852498Z","shell.execute_reply":"2022-01-10T04:05:43.863629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model\n","metadata":{}},{"cell_type":"markdown","source":"*Generator*","metadata":{}},{"cell_type":"code","source":"Growth_rate=32\n\ndef dense_block(inpt):\n    \"\"\"\n    Dense block containes total 4 conv blocks with leakyRelu \n    activation, followed by post conv layer\n    Params: tensorflow layer\n    Returns: tensorflow layer\n    \"\"\"\n    b1 = Conv2D(Growth_rate, kernel_size=3, strides=1, padding='same')(inpt)\n    b1 = LeakyReLU(0.2)(b1)\n    b1 = Concatenate()([inpt,b1])\n\n    b2 = Conv2D(Growth_rate, kernel_size=3, strides=1, padding='same')(b1)\n    b2 = LeakyReLU(0.2)(b2)\n    b2 = Concatenate()([inpt,b1,b2]) \n\n    b3 = Conv2D(Growth_rate, kernel_size=3, strides=1, padding='same')(b2)\n    b3 = LeakyReLU(0.2)(b3)\n    b3 = Concatenate()([inpt,b1,b2,b3])\n\n    b4 = Conv2D(Growth_rate, kernel_size=3, strides=1, padding='same')(b3)\n    b4 = LeakyReLU(0.2)(b4)\n    b4 = Concatenate()([inpt,b1,b2,b3,b4])\n\n    b5 = Conv2D(Growth_rate, kernel_size=3, strides=1, padding='same')(b4)\n    b5 = Lambda(lambda x:x*0.2)(b5)\n#     print(b5.shape)\n    b5 = add([b5, inpt])\n    \n    return b5\n\ndef RRDB(inpt,Beta= 0.2):\n    \n    \"\"\"\n    RRDB(residual in residual dense block) contained three dense  \n    block, each block followed by beta contant multiplication(0.2) \n    and addition with dense block input layer.\n    Params: tensorflow layer\n    Returns: tensorflow layer\n    \"\"\"\n    x = dense_block(inpt)\n\n    x = dense_block(x)\n\n    x = dense_block(x)\n\n    x = Lambda(lambda x:x*Beta)(x) # Beta constant\n\n    out = add([x,inpt])\n    return out","metadata":{"execution":{"iopub.status.busy":"2022-01-10T04:05:44.29327Z","iopub.execute_input":"2022-01-10T04:05:44.293516Z","iopub.status.idle":"2022-01-10T04:05:44.303407Z","shell.execute_reply.started":"2022-01-10T04:05:44.293488Z","shell.execute_reply":"2022-01-10T04:05:44.302711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Creating GMGAN\n\ndef Generator_model(growth_rate=64, rrdb= 16):\n input=Input(shape=(96,96,3),name='Input')\n\n#First convolution layer\n x1= Conv2D(filters=Growth_rate,kernel_size=3,padding='Same',name='conv1')(input) \n#  print(x1.shape)\n    \n for i in range(rrdb):\n#     print(i)\n    r= RRDB(x1)\n    x1= r\n\n#convolution layer after RRDB block\n x2= Conv2D(filters=Growth_rate,kernel_size=3,padding='Same')(r) \n\n x=add([x2,x1]) #skip connection\n\n#Upsampling the LOW RESOLUTION IMAGE to increase the number of pixel by 4x in super resolution image\n x= UpSampling2D(size=(4,4))(x)\n\n x= Conv2D(filters=Growth_rate,kernel_size=3,padding='Same')(x) \n\n output= Conv2D(filters=3,activation= 'tanh',kernel_size=9,padding='Same')(x) \n\n\n model=Model(input,output)\n model.summary()\n return model\n\ngenerator=Generator_model(Growth_rate)","metadata":{"execution":{"iopub.status.busy":"2022-01-10T04:05:44.424237Z","iopub.execute_input":"2022-01-10T04:05:44.424693Z","iopub.status.idle":"2022-01-10T04:05:52.054274Z","shell.execute_reply.started":"2022-01-10T04:05:44.424657Z","shell.execute_reply":"2022-01-10T04:05:52.053549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Creating discriminator\ndef conv_block(input,filter,kernel_size,stride,padding):\n\n        x=Conv2D(filters=filter,kernel_size=kernel_size,strides=stride,padding=padding)(input) \n        x=BatchNormalization()(x)\n        x=LeakyReLU()(x)\n        return x\n\n\ninput=Input(shape=(96*4,96*4,3),name='Input')\n\n\nx= conv_block(input,filter= 64,kernel_size= 3,stride= 1,padding='same')\nx=LeakyReLU()(x)\n\nx= conv_block(x,filter= 64,kernel_size= 3,stride= 2,padding='same')\n\nx= conv_block(x,filter= 128,kernel_size= 3,stride= 1,padding='same')\n\nx= conv_block(x,filter= 128,kernel_size= 3,stride= 2,padding='same')\n\nx= conv_block(x,filter= 256,kernel_size= 3,stride= 1,padding='same')\n\nx= conv_block(x,filter= 256,kernel_size= 3,stride= 2,padding='same')\n\nx= conv_block(x,filter= 512,kernel_size= 3,stride= 1,padding='same')\n\nx= conv_block(x,filter= 512,kernel_size= 3,stride= 2,padding='same')\n\nx= conv_block(x,filter= 1,kernel_size= 3,stride= 1,padding='same')\n\nx= Flatten()(x)\n\nx= Dense(1024)(x)\nx= LeakyReLU(alpha=0.2)(x)\n\noutput=Dense(1)(x)\n\ndiscriminator=Model(input,output)\ndiscriminator.summary()\n","metadata":{"execution":{"iopub.status.busy":"2022-01-10T04:05:52.055982Z","iopub.execute_input":"2022-01-10T04:05:52.056237Z","iopub.status.idle":"2022-01-10T04:05:52.248531Z","shell.execute_reply.started":"2022-01-10T04:05:52.056203Z","shell.execute_reply":"2022-01-10T04:05:52.247827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Losses","metadata":{}},{"cell_type":"code","source":"#For perpectual loss\nVGG_19 = tf.keras.applications.VGG19(input_shape=(96*4,96*4,3),include_top=False,weights='imagenet')\n\n\n#Generator Loss\ndef GMSD(HR,SR):\n\n#    print('The shape of input image',HR.shape)\n\n   sobel_HR=tf.image.sobel_edges(HR) #[batch_size,H,W,d,2] \n#    print('The  shape of sobel_HR:',sobel_HR.shape)\n#    print('The output after sobel_HR',sobel_HR[0,0,0])\n\n   sobel_HR= sobel_HR**2 #[H-2,2-2,3,2]\n   G_HR=tf.math.sqrt(tf.math.reduce_sum(sobel_HR,axis=-1)) #[H-2,W-2,3,1] \n#    print('The output m_HR',G_HR)\n#    print('The  shape of m_HR:',G_HR.shape)\n#    print('\\n')\n#    print('*****************************************************')\n\n\n   #calculating the gradient matrix value for super resolution image\n \n   sobel_SR=tf.image.sobel_edges(SR) #[H-2,W-2,3,2]\n#    print('The  shape of sobel_SR:',sobel_SR.shape)\n#    print('The output after sobel_GR',sobel_SR[0,0,0])\n\n\n   sobel_SR= sobel_SR**2 #[H-2,2-2,3,2]\n   G_SR=tf.math.sqrt(tf.math.reduce_sum(sobel_SR,axis=-1)) #[H-2,W-2,3,1] \n#    print('The output after joining m_SR',G_HR[0,0,0])\n#    print('The  shape of m_SR:',G_SR.shape)\n#    print('\\n')\n#    print('******/***********************************************')\n\n\n    #DEFINING THE PAREAMETER C\n   c=tf.math.multiply(2*G_SR,G_HR)+0.000001\n   d= tf.math.square(G_SR)+tf.math.square(G_HR)+0.000001\n   GMS=c/d \n#    print('The  shape of GMS:',GMS)\n\n   GMSM=tf.math.reduce_mean(GMS) #will get a scalar value\n#    print('The GMSM value is :',GMSM)\n\n   GMSD=tf.math.sqrt(tf.math.reduce_mean(tf.math.square((GMS-GMSM))))\n#    print('The GMSD value is :',GMSD)\n   return GMSD\n\n\ndef MSE(HR,SR):\n    a= tf.keras.losses.MSE(SR,HR) #[batch,height,width]\n    return tf.math.reduce_mean(a)\n    \n\ndef perpectual_loss(HR,SR):\n    VGG_SR= VGG_19(SR)\n    VGG_HR= VGG_19(HR)\n    perpectual= MSE(VGG_SR,VGG_HR)\n    return perpectual\n\n\ndef generator_loss(HR,HR_pred,SR,SR_pred):\n    \n    I_GA = -tf.reduce_mean(SR_pred)\n    I_Q= GMSD(HR,SR)\n    I_M= MSE(HR,SR)\n    I_P= perpectual_loss(HR,SR)\n    generator_los= 0.005*I_GA+I_Q+0.01*I_M+I_P\n\n\n    return generator_los\n","metadata":{"execution":{"iopub.status.busy":"2022-01-10T04:05:52.249816Z","iopub.execute_input":"2022-01-10T04:05:52.250084Z","iopub.status.idle":"2022-01-10T04:05:53.265247Z","shell.execute_reply.started":"2022-01-10T04:05:52.250051Z","shell.execute_reply":"2022-01-10T04:05:53.264478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n@tf.function\ndef discriminator_train(HR,LR,batch_size,step):\n    '''\n\n        Reference: https://www.tensorflow.org/tutorials/generative/dcgan\n    '''\n\n    epsilon = tf.random.uniform(shape=[batch_size, 1, 1, 1], minval=0, maxval=1)\n    ###################################\n    # Train D\n    ###################################\n    with tf.GradientTape(persistent=True) as d_tape:\n        with tf.GradientTape() as gp_tape:\n            SR = generator(LR, training=True)\n            fake_image_mixed = epsilon * tf.dtypes.cast(HR, tf.float32) + ((1 - epsilon) * SR)\n            fake_mixed_pred = discriminator(fake_image_mixed, training=True)\n            \n        # Compute gradient penalty\n#         print('Computing Gradient penalty')\n        grads = gp_tape.gradient(fake_mixed_pred, fake_image_mixed)\n        grad_norms = tf.sqrt(tf.reduce_sum(tf.square(grads), axis=[1, 2, 3]))\n        gradient_penalty = tf.reduce_mean(tf.square(grad_norms - 1))\n#         print('The value of gradient penalty',gradient_penalty)\n        \n#         print('calculating the loss for discriminator')\n        fake_pred = discriminator(SR, training=True)\n        real_pred = discriminator(HR, training=True)\n        \n        D_loss= 0.005*(tf.reduce_mean(real_pred) - tf.reduce_mean(fake_pred) + LAMBDA * gradient_penalty)\n\n    # Calculate the gradients for discriminator\n    D_gradients = d_tape.gradient(D_loss,discriminator.trainable_variables)\n    \n    # Apply the gradients to the optimizer\n    D_optimizer.apply_gradients(zip(D_gradients,discriminator.trainable_variables))\n\n    return D_loss\n    \n@tf.function\ndef generator_train(HR,LR, batch_size):\n    '''\n        One generator training step\n        \n        Reference: https://www.tensorflow.org/tutorials/generative/dcgan\n    '''\n#     print(\"retrace\")\n\n    ###################################\n    # Train G\n    ###################################\n    with tf.GradientTape() as g_tape:\n        SR = generator(LR, training=True)\n        SR_pred = discriminator(SR, training=False)\n        HR_pred= discriminator(HR, training=True)\n        \n        loss_gen= generator_loss(HR,HR_pred,SR,SR_pred)\n        \n        \n    # Calculate the gradients for generator\n    G_gradients = g_tape.gradient(loss_gen,generator.trainable_variables)\n    \n    # Apply the gradients to the optimizer\n    G_optimizer.apply_gradients(zip(G_gradients,generator.trainable_variables))\n    return loss_gen\n    \n@tf.function\ndef discriminator_test(HR,LR,batch_size):\n    '''\n\n        Reference: https://www.tensorflow.org/tutorials/generative/dcgan\n    '''\n#     print(\"retrace\")\n    epsilon = tf.random.uniform(shape=[batch_size, 1, 1, 1], minval=0, maxval=1)\n    ###################################\n    # Train D\n    ###################################\n\n    with tf.GradientTape() as gp_tape:\n        SR = generator(LR, training=True)\n        fake_image_mixed = epsilon * tf.dtypes.cast(HR, tf.float32) + ((1 - epsilon) * SR)\n        fake_mixed_pred = discriminator(fake_image_mixed, training=True)\n\n    # Compute gradient penalty\n#         print('Computing Gradient penalty')\n    grads = gp_tape.gradient(fake_mixed_pred, fake_image_mixed)\n    grad_norms = tf.sqrt(tf.reduce_sum(tf.square(grads), axis=[1, 2, 3]))\n    gradient_penalty = tf.reduce_mean(tf.square(grad_norms - 1))\n\n    fake_pred = discriminator(SR, training=True)\n    real_pred = discriminator(HR, training=True)\n\n    D_loss= 0.005*(tf.reduce_mean(real_pred) - tf.reduce_mean(fake_pred) + LAMBDA * gradient_penalty)\n\n\n    return D_loss\n    \n@tf.function\ndef generator_test(HR,LR, batch_size):\n    '''\n        One generator training step\n        \n        Reference: https://www.tensorflow.org/tutorials/generative/dcgan\n    '''\n#     print(\"retrace\")\n\n    ###################################\n    # Train G\n    ###################################\n    \n    SR = generator(LR, training=True)\n    SR_pred = discriminator(SR, training=False)\n    HR_pred= discriminator(HR, training=True)\n\n    loss_gen= generator_loss(HR,HR_pred,SR,SR_pred)\n\n    return loss_gen\n","metadata":{"execution":{"iopub.status.busy":"2022-01-10T04:05:53.267116Z","iopub.execute_input":"2022-01-10T04:05:53.267361Z","iopub.status.idle":"2022-01-10T04:05:53.286301Z","shell.execute_reply.started":"2022-01-10T04:05:53.267329Z","shell.execute_reply":"2022-01-10T04:05:53.285697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Create_checkpoints(load_previous_model=True):\n        #Creating an checkpoint object and manager to keep track of the checkpoint\n    !mkdir ./checkpoint_dir\n    checkpoint_path= './checkpoint_dir'\n    ckpt = tf.train.Checkpoint(generator=generator,\n                               discriminator=discriminator,\n                               G_optimizer=G_optimizer,\n                               D_optimizer=D_optimizer)\n\n    manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=5)\n    previous_epoch= 0\n\n    if load_previous_model== True:\n\n        #Loading the previous checkpoints so that to start the training from that epoch             \n        ckpt.restore(manager.latest_checkpoint)\n\n        if manager.latest_checkpoint:\n            previous_epoch= int(manager.latest_checkpoint.split('\\\\')[-1].split('-')[-1])\n            print('Loading the model parameters from previous epoch ie:',previous_epoch)\n            print('\\n')\n    else:\n\n        #Deleting the previous checkpoints directory\n        shutil.rmtree('./Checkpoints')\n        !mkdir './Checkpoints'\n\n    return manager,previous_epoch\n\n\n\n#optimizer\nD_optimizer = tf.keras.optimizers.Adam(0.0001)\nG_optimizer = tf.keras.optimizers.Adam(0.0001)\n\n#Creating and loading the checkpoint\nmanager,previous_epoch= Create_checkpoints(load_previous_model= True)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-10T04:05:53.287225Z","iopub.execute_input":"2022-01-10T04:05:53.287976Z","iopub.status.idle":"2022-01-10T04:05:54.011248Z","shell.execute_reply.started":"2022-01-10T04:05:53.28794Z","shell.execute_reply":"2022-01-10T04:05:54.010454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Parameters\nLAMBDA= 10\nepochs= 10\nN_CRITIC= 5\nbatch= 5\n\n#Dataset \ntrain_dataset= pipeline(train_paths_to_HR_images,train_paths_to_LR_images)\ntest_dataset= pipeline(test_paths_to_HR_images,test_paths_to_LR_images)\n\n#Steps\ntrain_steps= int(len(train_paths_to_HR_images)/batch)\ntest_steps= int(len(test_paths_to_HR_images)/batch)\nnu_to_train_dis= 0\n\nfor epoch in range(1,epochs):\n    start = time.time()\n    \n    \n    for step, [HR,LR] in enumerate(train_dataset):\n  \n        #Training the discriminator\n      \n        loss_dis= discriminator_train(HR,LR,batch,1)         \n        nu_to_train_dis+=1\n    \n        if nu_to_train_dis == N_CRITIC:\n            loss_gen= generator_train(HR,LR,batch)\n            nu_to_train_dis= 0\n            \n        if step%500==0 and step!=0:\n            print('Epoch: ',epoch+previous_epoch,'/',epochs+previous_epoch)\n            print(step,'/',train_steps,'/n')\n            print('loss of discriminator ========',loss_dis.numpy())\n            print('loss of generator ========',loss_gen.numpy())\n        \n        \n#     print('/nTESTING:')\n#     #For testing imagaes\n        \n#     for step, [HR,LR] in enumerate(test_dataset):\n  \n#         #Training the discriminator\n#         loss_dis= discriminator_train(HR,LR,batch)         \n#         nu_to_train_dis+=1\n    \n#         while nu_to_train_dis < N_CRITIC:\n#             loss_gen= WGAN_GP_train_g_step(HR,LR,batch)\n`\n#             nu_to_train_dis= 0\n        \n#         #Saving the checkpoints\n#         manager.save()\n\n#     print('Epoch: ',epoch+previous_epoch,'/',epochs+previous_epoch)\n#     print(step,'/',train_steps,'/n')\n#     print('loss of discriminator ========',loss_dis)\n#     print('loss of generator ========',loss_gen)\n    print('/******************************************************************/')\n    print('Time taken for epoch',epoch+previous_epoch,'is sec\\n',time.time()-start)\n    #Saving the model\n    manager.save()\n\n    ","metadata":{"execution":{"iopub.status.busy":"2022-01-09T15:54:34.684159Z","iopub.execute_input":"2022-01-09T15:54:34.684525Z","iopub.status.idle":"2022-01-09T15:54:38.685093Z","shell.execute_reply.started":"2022-01-09T15:54:34.684487Z","shell.execute_reply":"2022-01-09T15:54:38.683976Z"},"trusted":true},"execution_count":null,"outputs":[]}]}