Roteiro_classificacao_jgeotec.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "id": "cubL3DjOhOtT",
      "metadata": {
        "id": "cubL3DjOhOtT"
      },
      "source": [
        "# Bibliotecas"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "bd70565d-f16e-47d1-9534-c81f99fce4d5",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bd70565d-f16e-47d1-9534-c81f99fce4d5",
        "outputId": "8120890f-e4d0-452c-ac84-8eb9e7504293"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "from PIL import Image\n",
        "import itertools\n",
        "import time #Tempo de cada modelo\n",
        "import matplotlib.pyplot as plt\n",
        "import cv2\n",
        "\n",
        "from imblearn.pipeline import Pipeline\n",
        "from plotly import figure_factory as ff\n",
        "from matplotlib.colors import ListedColormap\n",
        "\n",
        "#Transformação em imagens\n",
        "from skimage.segmentation import felzenszwalb, slic, join_segmentations, mark_boundaries\n",
        "from skimage.feature import local_binary_pattern, hog, graycomatrix, graycoprops, shape_index\n",
        "from skimage.measure import label\n",
        "from skimage.util import img_as_ubyte\n",
        "#from scipy.stats import entropy\n",
        "from skimage.filters.rank import entropy\n",
        "\n",
        "#Métricas\n",
        "from tensorflow.keras.metrics import MeanIoU\n",
        "from sklearn.metrics import roc_curve, auc, roc_auc_score\n",
        "from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score\n",
        "\n",
        "#Modelos\n",
        "import lightgbm as lgb\n",
        "from xgboost import XGBClassifier\n",
        "from sklearn.model_selection import train_test_split, RandomizedSearchCV, GridSearchCV\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.cluster import KMeans\n",
        "from sklearn.naive_bayes import GaussianNB\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.neural_network import MLPClassifier\n",
        "\n",
        "#Acesso ao Drive\n",
        "#from google.colab import drive\n",
        "#drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "OMQwB-hIhTWT",
      "metadata": {
        "id": "OMQwB-hIhTWT"
      },
      "source": [
        "# Código"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "_98l2dQFmhVN",
      "metadata": {
        "id": "_98l2dQFmhVN"
      },
      "source": [
        "### Pacote Superpixel"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "id": "dc88c067-2cb3-430d-8506-5ef588ae9439",
      "metadata": {
        "id": "dc88c067-2cb3-430d-8506-5ef588ae9439"
      },
      "outputs": [],
      "source": [
        "from os import listdir\n",
        "from os.path import isfile, join\n",
        "from skimage.color import rgb2gray\n",
        "from skimage.filters import sobel\n",
        "from skimage.segmentation import felzenszwalb, slic, quickshift, watershed\n",
        "from skimage.segmentation import mark_boundaries\n",
        "from skimage.util import img_as_float\n",
        "from skimage.segmentation import join_segmentations\n",
        "from skimage import io\n",
        "import numpy as np\n",
        "import scipy as sp\n",
        "from scipy.spatial import distance\n",
        "import matplotlib.pyplot as plt\n",
        "from datetime import datetime\n",
        "from skimage.measure import label, regionprops\n",
        "import math\n",
        "\n",
        "import sys\n",
        "\n",
        "def mahalanobis(x=None, data=None, cov=None):\n",
        "    \"\"\"Compute the Mahalanobis Distance between each row of x and the data\n",
        "    x    : vector or matrix of data with, say, p columns.\n",
        "    data : ndarray of the distribution from which Mahalanobis distance of each observation of x is to be computed.\n",
        "    cov  : covariance matrix (p x p) of the distribution. If None, will be computed from data.\n",
        "    \"\"\"\n",
        "    x_minus_mu = x - np.mean(data,axis=0)\n",
        "    try:\n",
        "        if not cov:\n",
        "            cov = np.cov(data.T)\n",
        "        inv_covmat = sp.linalg.inv(cov)\n",
        "        left_term = np.dot(x_minus_mu, inv_covmat)\n",
        "        mahal = np.dot(left_term, x_minus_mu.T)\n",
        "        if type (mahal) is np.float64:\n",
        "            return mahal\n",
        "        return mahal.diagonal()\n",
        "    except:\n",
        "        return 9999999999999999999\n",
        "\n",
        "def get_orderedsegments(label_img, max_pixels=25):\n",
        "    uniques = np.unique(label_img, return_counts=True)\n",
        "    filtered = [[],[]]\n",
        "    filtered[0] = np.array(uniques[0][uniques[1]<=max_pixels])\n",
        "    inds = filtered[0].argsort()\n",
        "    sortedSegments = filtered[0][inds]\n",
        "    return sortedSegments\n",
        "\n",
        "def get_neighbors(label_img):\n",
        "    #print(label_img.shape)\n",
        "    props = regionprops(label_img)\n",
        "    vizinhos_original = []\n",
        "    vizinhos_original.append([])\n",
        "    len_props=len(props)\n",
        "    for i in range(0,len_props):\n",
        "        #if (i)%int(len_props/5) == 0:\n",
        "            #print(i)\n",
        "        vizinhos_original.append([])\n",
        "        for j in props[i].coords:\n",
        "            if j[0]-1>=0:\n",
        "                if props[i].label!=label_img[j[0]-1,j[1]]:\n",
        "                    temp=label_img[j[0]-1,j[1]]\n",
        "                    vizinhos_original[props[i].label].append(temp)\n",
        "            if j[1]-1>=0:\n",
        "                if props[i].label!=label_img[j[0],j[1]-1]:\n",
        "                    vizinhos_original[props[i].label].append(label_img[j[0],j[1]-1])\n",
        "            if j[0]+1<label_img.shape[0]:\n",
        "                if props[i].label!=label_img[j[0]+1,j[1]]:\n",
        "                    vizinhos_original[props[i].label].append(label_img[j[0]+1,j[1]])\n",
        "            if j[1]+1<label_img.shape[1]:\n",
        "                if props[i].label!=label_img[j[0],j[1]+1]:\n",
        "                    vizinhos_original[props[i].label].append(label_img[j[0],j[1]+1])\n",
        "        vizinhos_original[props[i].label] = list(np.unique(vizinhos_original[props[i].label]))\n",
        "    #print(len_props)\n",
        "    return vizinhos_original\n",
        "\n",
        "def join_segments(label_img, img, min_number_pixels=50, distance='mean', verbose=True):\n",
        "    if verbose:\n",
        "        print('gerando labels unicos')\n",
        "    label_img = label(label_img, connectivity=1)\n",
        "    if verbose:\n",
        "        print('gerando vizinhos unicos')\n",
        "    vizinhos = get_neighbors(label_img)\n",
        "    if verbose:\n",
        "        print('gerando segmentos de tamanho minimo',min_number_pixels)\n",
        "    t1=datetime.now()\n",
        "    counter=0\n",
        "    k=np.min(label_img)\n",
        "    label_img_max=np.max(label_img)\n",
        "    to_print = int(label_img_max/1000)\n",
        "    if to_print<1000:\n",
        "        to_print=1000\n",
        "    while (k <= label_img_max):\n",
        "        t2=datetime.now()\n",
        "        i=k\n",
        "\n",
        "        label_img_i_len = len(label_img[label_img==i])\n",
        "        if (label_img_i_len>=min_number_pixels) or (label_img_i_len==0):\n",
        "            k+=1\n",
        "            continue\n",
        "\n",
        "        minDistance = 9999999999999999999999\n",
        "        minDistanceSeg = -1\n",
        "\n",
        "        for v in vizinhos[i]:\n",
        "            if v==i:\n",
        "                continue\n",
        "            if minDistanceSeg == -1:\n",
        "                minDistanceSeg = v\n",
        "            m=99999999999999999999999\n",
        "            len_v = len(label_img[label_img==v])\n",
        "            if len_v==0:\n",
        "                continue\n",
        "            if len_v>label_img_i_len and len_v>2:\n",
        "                if distance=='mean':\n",
        "                    m = mahalanobis(np.mean(img[label_img==i], axis=0),img[label_img==v])\n",
        "                else:\n",
        "                    m = mahalanobis(np.median(img[label_img==i], axis=0),img[label_img==v])\n",
        "\n",
        "            elif label_img_i_len>2:\n",
        "                if distance=='mean':\n",
        "                    m = mahalanobis(np.mean(img[label_img==v], axis=0),img[label_img==i])\n",
        "                else:\n",
        "                    m = mahalanobis(np.median(img[label_img==v], axis=0),img[label_img==i])\n",
        "            if m<minDistance:\n",
        "                minDistance = m\n",
        "                minDistanceSeg = v\n",
        "\n",
        "        finalsegment = -1\n",
        "        removedsegment = -1\n",
        "        if minDistanceSeg<i:\n",
        "            finalsegment = minDistanceSeg\n",
        "            removedsegment = i\n",
        "        else:\n",
        "            finalsegment = i\n",
        "            removedsegment = minDistanceSeg\n",
        "\n",
        "        label_img[label_img==removedsegment] = finalsegment\n",
        "\n",
        "        finalsegment_len = len(label_img[label_img==finalsegment])\n",
        "\n",
        "        if finalsegment<=k and finalsegment_len<min_number_pixels:\n",
        "            k=finalsegment\n",
        "        else:\n",
        "            k+=1\n",
        "\n",
        "        vizinhos[finalsegment] = [*vizinhos[finalsegment],*vizinhos[removedsegment]]\n",
        "        vizinhos[finalsegment] = list(np.unique(vizinhos[finalsegment]))\n",
        "        temp = vizinhos[removedsegment].copy()\n",
        "        for j in temp:\n",
        "            try:\n",
        "                vizinhos[j].remove(removedsegment)\n",
        "            except:\n",
        "                print('ERRO removendo:',removedsegment)\n",
        "            vizinhos[j].append(finalsegment)\n",
        "\n",
        "            vizinhos[j] = list(np.unique(vizinhos[j]))\n",
        "\n",
        "        counter+=1\n",
        "        if counter%to_print==0 or k > label_img_max:\n",
        "            #print(label_img_max,'|',k,'-',i,'-',finalsegment, '-', removedsegment,\"|\", np.max(label_img), '|', datetime.now()-t1, datetime.now()-t2, \"-\", counter, '-', (datetime.now()-t1)/counter,\" | \",'final segment len: ',finalsegment_len)#, '|', len(vizinhos[finalsegment-1]))\n",
        "            if verbose:\n",
        "                print(label_img_max,'|',k,'|',distance,'|', datetime.now()-t1, datetime.now()-t2, \"-\", counter, '-', (datetime.now()-t1)/counter)\n",
        "        if k > label_img_max:\n",
        "            label_img = label(label_img, connectivity=1)\n",
        "            unique_labels = np.unique(label_img, return_counts=True)\n",
        "            smaller_than_min = len(unique_labels[1][unique_labels[1]<min_number_pixels])\n",
        "            if smaller_than_min>0:\n",
        "                if verbose:\n",
        "                    print(\"reprocessing: \",smaller_than_min,'|',unique_labels[1][unique_labels[1]<min_number_pixels],\"|\",unique_labels[0][unique_labels[1]<min_number_pixels])\n",
        "                k=np.min(label_img)\n",
        "                label_img_max=np.max(label_img)\n",
        "                vizinhos = get_neighbors(label_img)\n",
        "            else:\n",
        "                break\n",
        "\n",
        "    label_img = label(label_img, connectivity=1)\n",
        "    return label_img\n",
        "\n",
        "def segment_slic(img, save_image_name_path, filename):\n",
        "#     segments = slic(img, n_segments=int(img.shape[0]*img.shape[1]/1400), compactness=5, sigma=1, start_label=1)\n",
        "#     np.savez(save_image_name_path.replace('.tif','_segments_slic01.npz'),segments)\n",
        "#     segments = slic(img, n_segments=int(img.shape[0]*img.shape[1]/700), compactness=5, sigma=1, start_label=1)\n",
        "#     np.savez(save_image_name_path.replace('.tif','_segments_slic02.npz'),segments)\n",
        "#     segments = slic(img, n_segments=int(img.shape[0]*img.shape[1]/500), compactness=5, sigma=1, start_label=1)\n",
        "#     np.savez(save_image_name_path.replace('.tif','_segments_slic03.npz'),segments)\n",
        "#     segments = slic(img, n_segments=int(img.shape[0]*img.shape[1]/350), compactness=5, sigma=1, start_label=1)\n",
        "#     np.savez(save_image_name_path.replace('.tif','_segments_slic04.npz'),segments)\n",
        "#     segments = slic(img, n_segments=int(img.shape[0]*img.shape[1]/230), compactness=5, sigma=1, start_label=1)\n",
        "#     np.savez(save_image_name_path.replace('.tif','_segments_slic05.npz'),segments)\n",
        "    segments = slic(img, n_segments=int(img.shape[0]*img.shape[1]/170), compactness=5, sigma=1, start_label=1)\n",
        "    np.savez(save_image_name_path.replace('.tif','_segments_slic06.npz'),segments)\n",
        "    print('SLIC done', filename)\n",
        "\n",
        "def segment_quick(img, save_image_name_path, filename):\n",
        "#     segments = quickshift(img, kernel_size=2, max_dist=50, ratio=0.5)\n",
        "#     np.savez(save_image_name_path.replace('.tif','_segments_quick01.npz'),segments)\n",
        "    segments = quickshift(img, kernel_size=3, max_dist=50, ratio=0.5)\n",
        "    np.savez(save_image_name_path.replace('.tif','_segments_quick02.npz'),segments)\n",
        "#     segments = quickshift(img, kernel_size=4, max_dist=50, ratio=0.5)\n",
        "#     np.savez(save_image_name_path.replace('.tif','_segments_quick03.npz'),segments)\n",
        "#     segments = quickshift(img, kernel_size=5, max_dist=50, ratio=0.5)\n",
        "#     np.savez(save_image_name_path.replace('.tif','_segments_quick04.npz'),segments)\n",
        "    print('Quick done', filename)\n",
        "\n",
        "def segment_felz(img, save_image_name_path, filename):\n",
        "    segments = felzenszwalb(img, scale=50, sigma=0.5, min_size=50)\n",
        "    np.savez(save_image_name_path.replace('.tif','_segments_fz01.npz'),segments)\n",
        "    segments = felzenszwalb(img, scale=100, sigma=0.5, min_size=50)\n",
        "    np.savez(save_image_name_path.replace('.tif','_segments_fz02.npz'),segments)\n",
        "#     segments = felzenszwalb(img, scale=200, sigma=0.5, min_size=50)\n",
        "#     np.savez(save_image_name_path.replace('.tif','_segments_fz03.npz'),segments)\n",
        "    segments = felzenszwalb(img, scale=400, sigma=0.5, min_size=50)\n",
        "    np.savez(save_image_name_path.replace('.tif','_segments_fz04.npz'),segments)\n",
        "    segments = felzenszwalb(img, scale=50, sigma=0.5, min_size=100)\n",
        "    np.savez(save_image_name_path.replace('.tif','_segments_fz05.npz'),segments)\n",
        "    segments = felzenszwalb(img, scale=100, sigma=0.5, min_size=100)\n",
        "    np.savez(save_image_name_path.replace('.tif','_segments_fz06.npz'),segments)\n",
        "#     segments = felzenszwalb(img, scale=50, sigma=0.7, min_size=50)\n",
        "#     np.savez(save_image_name_path.replace('.tif','_segments_fz07.npz'),segments)\n",
        "#     segments = felzenszwalb(img, scale=100, sigma=0.7, min_size=50)\n",
        "#     np.savez(save_image_name_path.replace('.tif','_segments_fz08.npz'),segments)\n",
        "#     segments = felzenszwalb(img, scale=200, sigma=0.7, min_size=50)\n",
        "#     np.savez(save_image_name_path.replace('.tif','_segments_fz09.npz'),segments)\n",
        "#     segments = felzenszwalb(img, scale=400, sigma=0.7, min_size=50)\n",
        "#     np.savez(save_image_name_path.replace('.tif','_segments_fz10.npz'),segments)\n",
        "#     segments = felzenszwalb(img, scale=50, sigma=0.7, min_size=100)\n",
        "#     np.savez(save_image_name_path.replace('.tif','_segments_fz11.npz'),segments)\n",
        "#     segments = felzenszwalb(img, scale=100, sigma=0.7, min_size=100)\n",
        "#     np.savez(save_image_name_path.replace('.tif','_segments_fz12.npz'),segments)\n",
        "    print('Felzenszwalb done', filename)\n",
        "\n",
        "def segment_felz_slic(original_img, chunk_size,distance,min_number_pixels, dataset, filename, save_image_name_path):\n",
        "#     final_segments = np.zeros((original_img.shape[0],original_img.shape[1]),dtype=np.int32)\n",
        "#     for k in range(0,math.ceil(final_segments.shape[0]/chunk_size)):\n",
        "#         for l in range(0,math.ceil(final_segments.shape[1]/chunk_size)):\n",
        "#             img = original_img[k*chunk_size:(k+1)*chunk_size,l*chunk_size:(l+1)*chunk_size]\n",
        "#             segments = join_segmentations(slic(img, n_segments=int(img.shape[0]*img.shape[1]/2000), compactness=5, sigma=1, start_label=1), felzenszwalb(img, scale=200, sigma=0.7, min_size=200))\n",
        "#             segments = join_segments(segments, img, dataset, filename, min_number_pixels=min_number_pixels, distance=distance)\n",
        "#             final_segments[k*chunk_size:(k+1)*chunk_size,l*chunk_size:(l+1)*chunk_size] = segments+1+np.max(final_segments)\n",
        "#     segments = label(final_segments, connectivity=1)\n",
        "#     np.savez(save_image_name_path.replace('.tif','_segments_fz_slic01_'+distance+'_'+str(min_number_pixels)+'.npz'),segments)\n",
        "\n",
        "    final_segments = np.zeros((original_img.shape[0],original_img.shape[1]),dtype=np.int32)\n",
        "    for k in range(0,math.ceil(final_segments.shape[0]/chunk_size)):\n",
        "        for l in range(0,math.ceil(final_segments.shape[1]/chunk_size)):\n",
        "            img = original_img[k*chunk_size:(k+1)*chunk_size,l*chunk_size:(l+1)*chunk_size]\n",
        "            segments = join_segmentations(slic(img, n_segments=int(img.shape[0]*img.shape[1]/1500), compactness=5, sigma=1, start_label=1), felzenszwalb(img, scale=100, sigma=0.7, min_size=150))\n",
        "            segments = join_segments(segments, img, dataset, filename, min_number_pixels=min_number_pixels, distance=distance)\n",
        "            final_segments[k*chunk_size:(k+1)*chunk_size,l*chunk_size:(l+1)*chunk_size] = segments+1+np.max(final_segments)\n",
        "    segments = label(final_segments, connectivity=1)\n",
        "    np.savez(save_image_name_path.replace('.tif','_segments_fz_slic02_'+distance+'_'+str(min_number_pixels)+'.npz'),segments)\n",
        "\n",
        "#     final_segments = np.zeros((original_img.shape[0],original_img.shape[1]),dtype=np.int32)\n",
        "#     for k in range(0,math.ceil(final_segments.shape[0]/chunk_size)):\n",
        "#         for l in range(0,math.ceil(final_segments.shape[1]/chunk_size)):\n",
        "#             img = original_img[k*chunk_size:(k+1)*chunk_size,l*chunk_size:(l+1)*chunk_size]\n",
        "#             segments = join_segmentations(slic(img, n_segments=int(img.shape[0]*img.shape[1]/1000), compactness=5, sigma=1, start_label=1), felzenszwalb(img, scale=100, sigma=0.7, min_size=150))\n",
        "#             segments = join_segments(segments, img, dataset, filename, min_number_pixels=min_number_pixels, distance=distance)\n",
        "#             final_segments[k*chunk_size:(k+1)*chunk_size,l*chunk_size:(l+1)*chunk_size] = segments+1+np.max(final_segments)\n",
        "#     segments = label(final_segments, connectivity=1)\n",
        "#     np.savez(save_image_name_path.replace('.tif','_segments_fz_slic03_'+distance+'_'+str(min_number_pixels)+'.npz'),segments)\n",
        "\n",
        "    final_segments = np.zeros((original_img.shape[0],original_img.shape[1]),dtype=np.int32)\n",
        "    for k in range(0,math.ceil(final_segments.shape[0]/chunk_size)):\n",
        "        for l in range(0,math.ceil(final_segments.shape[1]/chunk_size)):\n",
        "            img = original_img[k*chunk_size:(k+1)*chunk_size,l*chunk_size:(l+1)*chunk_size]\n",
        "            segments = join_segmentations(slic(img, n_segments=int(img.shape[0]*img.shape[1]/500), compactness=5, sigma=1, start_label=1), felzenszwalb(img, scale=50, sigma=0.7, min_size=100))\n",
        "            segments = join_segments(segments, img, dataset, filename, min_number_pixels=min_number_pixels, distance=distance)\n",
        "            final_segments[k*chunk_size:(k+1)*chunk_size,l*chunk_size:(l+1)*chunk_size] = segments+1+np.max(final_segments)\n",
        "    segments = label(final_segments, connectivity=1)\n",
        "    np.savez(save_image_name_path.replace('.tif','_segments_fz_slic04_'+distance+'_'+str(min_number_pixels)+'.npz'),segments)\n",
        "\n",
        "    print('Felzenszwalb-SLIC done', filename)\n",
        "\n",
        "def segment_quick_slic(original_img, chunk_size,distance,min_number_pixels, dataset, filename, save_image_name_path):\n",
        "#     final_segments = np.zeros((original_img.shape[0],original_img.shape[1]),dtype=np.int32)\n",
        "#     for k in range(0,math.ceil(final_segments.shape[0]/chunk_size)):\n",
        "#         for l in range(0,math.ceil(final_segments.shape[1]/chunk_size)):\n",
        "#             img = original_img[k*chunk_size:(k+1)*chunk_size,l*chunk_size:(l+1)*chunk_size]\n",
        "#             segments = join_segmentations(slic(img, n_segments=int(img.shape[0]*img.shape[1]/2000), compactness=5, sigma=1, start_label=1),quickshift(img, kernel_size=5, max_dist=50, ratio=0.5))\n",
        "#             segments = join_segments(segments, img, dataset, filename, min_number_pixels=min_number_pixels, distance=distance)\n",
        "#             final_segments[k*chunk_size:(k+1)*chunk_size,l*chunk_size:(l+1)*chunk_size] = segments+1+np.max(final_segments)\n",
        "#     segments = label(final_segments, connectivity=1)\n",
        "#     np.savez(save_image_name_path.replace('.tif','_segments_quick_slic01_'+distance+'_'+str(min_number_pixels)+'.npz'),segments)\n",
        "\n",
        "#     final_segments = np.zeros((original_img.shape[0],original_img.shape[1]),dtype=np.int32)\n",
        "#     for k in range(0,math.ceil(final_segments.shape[0]/chunk_size)):\n",
        "#         for l in range(0,math.ceil(final_segments.shape[1]/chunk_size)):\n",
        "#             img = original_img[k*chunk_size:(k+1)*chunk_size,l*chunk_size:(l+1)*chunk_size]\n",
        "#             segments = join_segmentations(slic(img, n_segments=int(img.shape[0]*img.shape[1]/1500), compactness=5, sigma=1, start_label=1),quickshift(img, kernel_size=5, max_dist=50, ratio=0.5))\n",
        "#             segments = join_segments(segments, img, dataset, filename, min_number_pixels=min_number_pixels, distance=distance)\n",
        "#             final_segments[k*chunk_size:(k+1)*chunk_size,l*chunk_size:(l+1)*chunk_size] = segments+1+np.max(final_segments)\n",
        "#     segments = label(final_segments, connectivity=1)\n",
        "#     np.savez(save_image_name_path.replace('.tif','_segments_quick_slic02_'+distance+'_'+str(min_number_pixels)+'.npz'),segments)\n",
        "\n",
        "#     final_segments = np.zeros((original_img.shape[0],original_img.shape[1]),dtype=np.int32)\n",
        "#     for k in range(0,math.ceil(final_segments.shape[0]/chunk_size)):\n",
        "#         for l in range(0,math.ceil(final_segments.shape[1]/chunk_size)):\n",
        "#             img = original_img[k*chunk_size:(k+1)*chunk_size,l*chunk_size:(l+1)*chunk_size]\n",
        "#             segments = join_segmentations(slic(img, n_segments=int(img.shape[0]*img.shape[1]/1000), compactness=5, sigma=1, start_label=1),quickshift(img, kernel_size=5, max_dist=50, ratio=0.5))\n",
        "#             segments = join_segments(segments, img, dataset, filename, min_number_pixels=min_number_pixels, distance=distance)\n",
        "#             final_segments[k*chunk_size:(k+1)*chunk_size,l*chunk_size:(l+1)*chunk_size] = segments+1+np.max(final_segments)\n",
        "#     segments = label(final_segments, connectivity=1)\n",
        "#     np.savez(save_image_name_path.replace('.tif','_segments_quick_slic03_'+distance+'_'+str(min_number_pixels)+'.npz'),segments)\n",
        "\n",
        "#     final_segments = np.zeros((original_img.shape[0],original_img.shape[1]),dtype=np.int32)\n",
        "#     for k in range(0,math.ceil(final_segments.shape[0]/chunk_size)):\n",
        "#         for l in range(0,math.ceil(final_segments.shape[1]/chunk_size)):\n",
        "#             img = original_img[k*chunk_size:(k+1)*chunk_size,l*chunk_size:(l+1)*chunk_size]\n",
        "#             segments = join_segmentations(slic(img, n_segments=int(img.shape[0]*img.shape[1]/500), compactness=5, sigma=1, start_label=1),quickshift(img, kernel_size=5, max_dist=50, ratio=0.5))\n",
        "#             segments = join_segments(segments, img, dataset, filename, min_number_pixels=min_number_pixels, distance=distance)\n",
        "#             final_segments[k*chunk_size:(k+1)*chunk_size,l*chunk_size:(l+1)*chunk_size] = segments+1+np.max(final_segments)\n",
        "#     segments = label(final_segments, connectivity=1)\n",
        "#     np.savez(save_image_name_path.replace('.tif','_segments_quick_slic04_'+distance+'_'+str(min_number_pixels)+'.npz'), segments)\n",
        "    print('Quick-SLIC done', filename)\n",
        "\n",
        "\n",
        "def segment_felz_quick(original_img, chunk_size,distance,min_number_pixels, dataset, filename, save_image_name_path):\n",
        "#     final_segments = np.zeros((original_img.shape[0],original_img.shape[1]),dtype=np.int32)\n",
        "#     for k in range(0,math.ceil(final_segments.shape[0]/chunk_size)):\n",
        "#         for l in range(0,math.ceil(final_segments.shape[1]/chunk_size)):\n",
        "#             img = original_img[k*chunk_size:(k+1)*chunk_size,l*chunk_size:(l+1)*chunk_size]\n",
        "#             segments = join_segmentations(felzenszwalb(img, scale=200, sigma=0.7, min_size=200), quickshift(img, kernel_size=2, max_dist=50, ratio=0.5))\n",
        "#             segments = join_segments(segments, img, dataset, filename, min_number_pixels=min_number_pixels, distance=distance)\n",
        "#             final_segments[k*chunk_size:(k+1)*chunk_size,l*chunk_size:(l+1)*chunk_size] = segments+1+np.max(final_segments)\n",
        "#     segments = label(final_segments, connectivity=1)\n",
        "#     np.savez(save_image_name_path.replace('.tif','_segments_fz_quick01_'+distance+'_'+str(min_number_pixels)+'.npz'),segments)\n",
        "\n",
        "    final_segments = np.zeros((original_img.shape[0],original_img.shape[1]),dtype=np.int32)\n",
        "    for k in range(0,math.ceil(final_segments.shape[0]/chunk_size)):\n",
        "        for l in range(0,math.ceil(final_segments.shape[1]/chunk_size)):\n",
        "            img = original_img[k*chunk_size:(k+1)*chunk_size,l*chunk_size:(l+1)*chunk_size]\n",
        "            segments = join_segmentations(felzenszwalb(img, scale=200, sigma=0.7, min_size=200), quickshift(img, kernel_size=3, max_dist=50, ratio=0.5))\n",
        "            segments = join_segments(segments, img, dataset, filename, min_number_pixels=min_number_pixels, distance=distance)\n",
        "            final_segments[k*chunk_size:(k+1)*chunk_size,l*chunk_size:(l+1)*chunk_size] = segments+1+np.max(final_segments)\n",
        "    segments = label(final_segments, connectivity=1)\n",
        "    np.savez(save_image_name_path.replace('.tif','_segments_fz_quick02_'+distance+'_'+str(min_number_pixels)+'.npz'),segments)\n",
        "\n",
        "#     final_segments = np.zeros((original_img.shape[0],original_img.shape[1]),dtype=np.int32)\n",
        "#     for k in range(0,math.ceil(final_segments.shape[0]/chunk_size)):\n",
        "#         for l in range(0,math.ceil(final_segments.shape[1]/chunk_size)):\n",
        "#             img = original_img[k*chunk_size:(k+1)*chunk_size,l*chunk_size:(l+1)*chunk_size]\n",
        "#             segments = join_segmentations(felzenszwalb(img, scale=200, sigma=0.7, min_size=200), quickshift(img, kernel_size=4, max_dist=50, ratio=0.5))\n",
        "#             segments = join_segments(segments, img, dataset, filename, min_number_pixels=min_number_pixels, distance=distance)\n",
        "#             final_segments[k*chunk_size:(k+1)*chunk_size,l*chunk_size:(l+1)*chunk_size] = segments+1+np.max(final_segments)\n",
        "#     segments = label(final_segments, connectivity=1)\n",
        "#     np.savez(save_image_name_path.replace('.tif','_segments_fz_quick03_'+distance+'_'+str(min_number_pixels)+'.npz'),segments)\n",
        "\n",
        "    final_segments = np.zeros((original_img.shape[0],original_img.shape[1]),dtype=np.int32)\n",
        "    for k in range(0,math.ceil(final_segments.shape[0]/chunk_size)):\n",
        "        for l in range(0,math.ceil(final_segments.shape[1]/chunk_size)):\n",
        "            img = original_img[k*chunk_size:(k+1)*chunk_size,l*chunk_size:(l+1)*chunk_size]\n",
        "            segments = join_segmentations(felzenszwalb(img, scale=200, sigma=0.7, min_size=200), quickshift(img, kernel_size=5, max_dist=50, ratio=0.5))\n",
        "            segments = join_segments(segments, img, dataset, filename, min_number_pixels=min_number_pixels, distance=distance)\n",
        "            final_segments[k*chunk_size:(k+1)*chunk_size,l*chunk_size:(l+1)*chunk_size] = segments+1+np.max(final_segments)\n",
        "    segments = label(final_segments, connectivity=1)\n",
        "    np.savez(save_image_name_path.replace('.tif','_segments_fz_quick04_'+distance+'_'+str(min_number_pixels)+'.npz'),segments)\n",
        "    print('Felzenszwalb-Quick done', filename)\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "Xa0O131Cmn38",
      "metadata": {
        "id": "Xa0O131Cmn38"
      },
      "source": [
        "### Código"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "id": "0e05b00d-b11b-485b-a19b-34155baf8324",
      "metadata": {
        "id": "0e05b00d-b11b-485b-a19b-34155baf8324"
      },
      "outputs": [],
      "source": [
        "def plot_and_save(plots, grid, sub_folder=\"\", save_grid=False):\n",
        "  \"\"\"\n",
        "  Save plots in the /home/images folder as pdf files\n",
        "  plots: a dictionary with coords as keys (x,y)\n",
        "          and with items of the form:\n",
        "          {\"img\": your_image, \"title\": title_of_the_image, \"opts\", {dictionary}}\n",
        "        Obs.: the \"opts\" is an optional key that passes extra paramenters to\n",
        "          the \"imshow\" method\n",
        "\n",
        "  grid: the grid layout you want your plot to have:\n",
        "        Ex.: (2,2) -> 2 horizontal, 2 vertical,\n",
        "             (4,1) -> 4 horizontal, 1 vertical\n",
        "\n",
        "  sub_folder: if provided, a folder that will store your images\n",
        "              inside of the home/images/ folder\n",
        "              /home/images/folder\n",
        "\n",
        "  save_grid: if True also saves the final grid plot as grid.pdf\n",
        "  \"\"\"\n",
        "  import os\n",
        "  os.makedirs(f\"/home/images/{sub_folder}/\", exist_ok=True)\n",
        "\n",
        "  # helper function to handle custom ploting options\n",
        "  def make_plot(obj, plot_value):\n",
        "    if plot_value.get(\"opts\"):\n",
        "      obj.imshow(plot_value[\"img\"], **plot_value[\"opts\"])\n",
        "    else:\n",
        "      obj.imshow(plot_value[\"img\"])\n",
        "    obj.set_title(plot_value[\"title\"])\n",
        "\n",
        "  # saves each plot individually\n",
        "  for axis in plots:\n",
        "    fig2, ax2 = plt.subplots(figsize=(10, 10))\n",
        "    make_plot(ax2, plots[axis])\n",
        "    fig2.savefig(f\"/home/images/{sub_folder}/axis{axis}.pdf\", bbox_inches=\"tight\")\n",
        "    plt.close()\n",
        "\n",
        "  # arrange the plots in the correct grid for plotting\n",
        "  fig, ax = plt.subplots(*grid, figsize=(10, 10), sharex=True, sharey=True)\n",
        "  for axis in plots:\n",
        "    if grid == (1,1):\n",
        "      make_plot(ax, plots[axis])\n",
        "    elif 1 in grid:\n",
        "      make_plot(ax[sum(axis)], plots[axis])\n",
        "    else:\n",
        "      make_plot(ax[axis], plots[axis])\n",
        "\n",
        "    plt.tight_layout()\n",
        "\n",
        "  # optionally saves the grid image\n",
        "  if save_grid:\n",
        "    fig.savefig(f\"/home/images/{sub_folder}/grid.pdf\", bbox_inches=\"tight\")\n",
        "\n",
        "  # plots the image\n",
        "  plt.show()\n",
        "\n",
        "#----------------------------EXEMPLO---------------------------\n",
        "#axes = {\n",
        "#    (0,0): {\"img\": mark_boundaries(img, superpixel_fz),\"title\":\"Felzenszwalbs's method\"},\n",
        "#    (0,1): {\"img\": mark_boundaries(img, superpixel_slic), \"title\":'SLIC'},\n",
        "#    (1,0): {\"img\": mark_boundaries(img, superpixel_final), \"title\": 'Final'},\n",
        "#    (1,1): {\"img\": mask_img, \"title\": 'Máscara'}\n",
        "#}\n",
        "#\n",
        "#plot_and_save(plots=axes, grid=(2,2), sub_folder=\"teste\", save_grid=True)\n",
        "#--------------------------------------------------------------\n",
        "\n",
        "#Função para calcular as estatísticas do superpixel\n",
        "def stats_img(image,label):\n",
        "  superpixel_stats = []\n",
        "\n",
        "  im_rp  = image.reshape((image.shape[0]*image.shape[1],image.shape[2]))\n",
        "  sli_1d = np.reshape(label,-1)\n",
        "  uni    = np.unique(sli_1d)\n",
        "  uu_1 = uu_2 = uu_3 = uu_4 = uu_5 = np.zeros(im_rp.shape)\n",
        "\n",
        "  for i in uni:\n",
        "    loc        = np.where(sli_1d==i)[0]\n",
        "    mean_sp    = np.mean(im_rp[loc,:],axis=0)\n",
        "    uu_1[loc,:]  = mean_sp\n",
        "\n",
        "  superpixel_mean = np.reshape(uu_1,[image.shape[0],image.shape[1],image.shape[2]]).astype('uint8')\n",
        "  plt.imshow(superpixel_mean)\n",
        "  plt.show()\n",
        "\n",
        "  for i in uni:\n",
        "    loc        = np.where(sli_1d==i)[0]\n",
        "    std_sp     = np.std(im_rp[loc,:],axis=0)\n",
        "    uu_2[loc,:]  = std_sp\n",
        "\n",
        "  superpixel_std = np.reshape(uu_2,[image.shape[0],image.shape[1],image.shape[2]]).astype('uint8')\n",
        "\n",
        "  for i in uni:\n",
        "    loc        = np.where(sli_1d==i)[0]\n",
        "    median_sp  = np.median(im_rp[loc,:],axis=0)\n",
        "    uu_3[loc,:]  = median_sp\n",
        "\n",
        "  superpixel_median = np.reshape(uu_3,[image.shape[0],image.shape[1],image.shape[2]]).astype('uint8')\n",
        "\n",
        "  for i in uni:\n",
        "    loc        = np.where(sli_1d==i)[0]\n",
        "    max_sp     = np.max(im_rp[loc,:],axis=0)\n",
        "    uu_4[loc,:]  = max_sp\n",
        "\n",
        "  superpixel_max = np.reshape(uu_4,[image.shape[0],image.shape[1],image.shape[2]]).astype('uint8')\n",
        "\n",
        "  for i in uni:\n",
        "    loc        = np.where(sli_1d==i)[0]\n",
        "    min_sp     = np.min(im_rp[loc,:],axis=0)\n",
        "    uu_5[loc,:]  = min_sp\n",
        "\n",
        "  superpixel_min = np.reshape(uu_5,[image.shape[0],image.shape[1],image.shape[2]]).astype('uint8')\n",
        "\n",
        "  superpixel_stats.append(superpixel_mean)\n",
        "  superpixel_stats.append(superpixel_std)\n",
        "  superpixel_stats.append(superpixel_median)\n",
        "  superpixel_stats.append(superpixel_max)\n",
        "  superpixel_stats.append(superpixel_min)\n",
        "\n",
        "  return superpixel_stats\n",
        "\n",
        "def add_features(image,label_fz,label_SLIC):\n",
        "  image_gray  = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n",
        "  #image_gray = 255*image_gray\n",
        "  indices     = shape_index(image_gray)\n",
        "  stats_fz    = stats_img(image,label_fz)\n",
        "  stats_slic  = stats_img(image,label_SLIC)\n",
        "#  lbp_feature = local_binary_pattern(image_gray,5,2)\n",
        "#  feature_vector = np.dstack((feature_vector,lbp_feature))\n",
        "#  lbp_feature = local_binary_pattern(image_gray,20,15)\n",
        "#  feature_vector = np.dstack((feature_vector,lbp_feature))\n",
        "  lbp_feature = local_binary_pattern(image_gray,3,4)\n",
        "  fd,hog_feature = hog(image,orientations=12,pixels_per_cell=(3,3),cells_per_block=(1,1),channel_axis=-1,visualize=True)\n",
        "#  fd,hog_feature = hog(image,orientations=9,pixels_per_cell=(5,5),cells_per_block=(1,1),channel_axis=-1,visualize=True)\n",
        "#  feature_vector = np.dstack((feature_vector,hog_feature))\n",
        "#  fd,hog_feature = hog(image,orientations=9,pixels_per_cell=(7,7),cells_per_block=(1,1),channel_axis=-1,visualize=True)\n",
        "#  feature_vector = np.dstack((feature_vector,hog_feature))\n",
        "#  fd,hog_feature = hog(image,orientations=9,pixels_per_cell=(15,15),cells_per_block=(1,1),channel_axis=-1,visualize=True)\n",
        "#  feature_vector = np.dstack((feature_vector,hog_feature))\n",
        "\n",
        "  glcm = graycomatrix(image_gray,[4], [0],levels=256,symmetric=False,normed=True)\n",
        "  glcm_feature = np.zeros((image.shape[0],image.shape[1],4))\n",
        "\n",
        "  for j in range(0,image.shape[0]):\n",
        "    for k in range(0,image.shape[1]):\n",
        "      gray_level = image_gray[j,k]\n",
        "      if k+1 < image_gray.shape[1]:\n",
        "        gray_level2 = image_gray[j,k+1]\n",
        "        glcm_feature[j,k,0] = glcm[gray_level,gray_level2,0,0]\n",
        "      if k-1 >= 0:\n",
        "        gray_level2 = image_gray[j,k-1]\n",
        "        glcm_feature[j,k,1] = glcm[gray_level,gray_level2,0,0]\n",
        "      if j+1 < image_gray.shape[0]:\n",
        "        gray_level2 = image_gray[j+1,k]\n",
        "        glcm_feature[j,k,2] = glcm[gray_level,gray_level2,0,0]\n",
        "      if j-1 >= 0:\n",
        "        gray_level2 = image_gray[j-1,k]\n",
        "        glcm_feature[j,k,3] = glcm[gray_level,gray_level2,0,0]\n",
        "\n",
        "  feature_vector = np.dstack((image,indices))\n",
        "\n",
        "  feature_vector = np.dstack((feature_vector,lbp_feature))\n",
        "  feature_vector = np.dstack((feature_vector,hog_feature))\n",
        "\n",
        "  feature_vector = np.concatenate((feature_vector,glcm_feature), axis=2)\n",
        "\n",
        "  feature_vector = np.concatenate((feature_vector,stats_fz[0]), axis=2)\n",
        "  feature_vector = np.concatenate((feature_vector,stats_fz[1]), axis=2)\n",
        "  feature_vector = np.concatenate((feature_vector,stats_fz[2]), axis=2)\n",
        "  feature_vector = np.concatenate((feature_vector,stats_fz[3]), axis=2)\n",
        "  feature_vector = np.concatenate((feature_vector,stats_fz[4]), axis=2)\n",
        "\n",
        "  feature_vector = np.concatenate((feature_vector,stats_slic[0]), axis=2)\n",
        "  feature_vector = np.concatenate((feature_vector,stats_slic[1]), axis=2)\n",
        "  feature_vector = np.concatenate((feature_vector,stats_slic[2]), axis=2)\n",
        "  feature_vector = np.concatenate((feature_vector,stats_slic[3]), axis=2)\n",
        "  feature_vector = np.concatenate((feature_vector,stats_slic[4]), axis=2)\n",
        "\n",
        "  plt.imshow(indices)\n",
        "  plt.show()\n",
        "  plt.imshow(hog_feature, cmap='gray')\n",
        "  plt.show()\n",
        "  plt.imshow(lbp_feature)\n",
        "  plt.show()\n",
        "  plt.imshow(glcm_feature[:, :, 0], cmap='gray')\n",
        "  plt.show()\n",
        "\n",
        "  return feature_vector\n",
        "\n",
        "#Função para criar o Superpixel\n",
        "def superpixel_image_AllBands(image, min_number_pixels = 17, verbose=False):\n",
        "    work_image = np.zeros((image.shape[0],image.shape[1],3),dtype=float)\n",
        "    work_image[:,:,0] = image[:,:,0]\n",
        "    work_image[:,:,1] = image[:,:,1]\n",
        "    work_image[:,:,2] = image[:,:,2]\n",
        "\n",
        "    s2 = felzenszwalb(work_image, scale=1, sigma=0.8, min_size=int(min_number_pixels))\n",
        "    if verbose:\n",
        "        print(f'Felzenszwalb number of segments: {len(np.unique(s2))}')\n",
        "\n",
        "    s1 = slic(work_image, n_segments=int((work_image.shape[0]*work_image.shape[1]/(min_number_pixels*4))), compactness=3, sigma=1, start_label=1)\n",
        "    if verbose:\n",
        "        print(f'SLIC number of segments: {len(np.unique(s1))}')\n",
        "\n",
        "    return s1,s2,work_image\n",
        "\n",
        "def train_test(mat, X, y):\n",
        "    modelo = mat['modelo']\n",
        "    nome = mat['nome']\n",
        "    grid = mat['grid']\n",
        "\n",
        "    print(f'Começando {nome}.')\n",
        "    ini = time.time()\n",
        "    pipe = Pipeline([('modelo', modelo)])\n",
        "\n",
        "    gscv = GridSearchCV(\n",
        "        pipe,\n",
        "        param_grid = grid,\n",
        "        scoring =['f1','jaccard'],\n",
        "        refit='jaccard',\n",
        "        cv = 5,\n",
        "        verbose = True,\n",
        "        n_jobs = 10,\n",
        "    )\n",
        "\n",
        "    rscv = gscv.fit(X,y)\n",
        "    fim = time.time()\n",
        "\n",
        "    print(f'Término {nome} | Score: {round(rscv.best_score_,3)} | Time: {rscv.refit_time_}')\n",
        "    return {'name':nome, 'object': rscv}\n",
        "\n",
        "#exclui setores fora da amostra\n",
        "def exclui_setores_fora(image,setores,features,verbose=True):\n",
        "  features_final_lista = []\n",
        "  masks_final_lista = []\n",
        "\n",
        "  for i in range(0,len(imagens_lista)):\n",
        "    features_final = features[i][setores[i] == 0]\n",
        "    masks_final = gt_lista[i][setores[i] == 0]\n",
        "    features_final_lista.append(features_final)\n",
        "    masks_final_lista.append(masks_final)\n",
        "\n",
        "    if verbose == True:\n",
        "      print(f'Length Final Features: {len(features_final_lista[i])}',\n",
        "            f'Length Mask Vector: {len(masks_final_lista[i])}',\n",
        "            f'Positives in Mask: {masks_final_lista[i].sum()}')\n",
        "\n",
        "  return features_final_lista, masks_final_lista"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "7d0f940a-45ab-43e9-9e3f-d2831fe663c1",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "collapsed": true,
        "id": "7d0f940a-45ab-43e9-9e3f-d2831fe663c1",
        "outputId": "9f4c4cc1-083f-4914-f358-efb3677409cc"
      },
      "outputs": [],
      "source": [
        "mapa_cores = {\n",
        "    0: (0, 0, 0, 0),        # vazio\n",
        "    1: (1, 0, 0, 1)         # vermelho\n",
        "}\n",
        "\n",
        "lista_cores = [mapa_cores[x] for x in [0,1]]\n",
        "cmap_ = ListedColormap(lista_cores)\n",
        "\n",
        "imagens_lista  = []\n",
        "imagens_treino = []\n",
        "gt_lista       = []\n",
        "setores        = []\n",
        "\n",
        "path=\"D:/Big Data & ODS - IC/images/Zona Sul/\"\n",
        "\n",
        "for i in range(1,6):\n",
        "    img = Image.open(path+\"img_\"+str(i)+\".png\")\n",
        "    img = np.asarray(img)\n",
        "    imagens_lista.append(img)\n",
        "\n",
        "    img_treino = Image.open(path+\"img_treino_\"+str(i)+\".png\")\n",
        "    img_treino  = np.asarray(img_treino )\n",
        "    imagens_treino.append(img_treino )\n",
        "\n",
        "    gt = Image.open(path+\"mask_img_\"+str(i)+\".png\")\n",
        "    gt = np.asarray(gt).copy()\n",
        "    gt = gt[:,:,0]\n",
        "    gt[gt<100]=0\n",
        "    gt[gt>100]=1\n",
        "    gt_lista.append(gt)\n",
        "\n",
        "    set = Image.open(path+\"setoresAGNS_fora_img_\"+str(i)+\".png\")\n",
        "    set = np.asarray(set).copy()\n",
        "    set = set[:,:,0]\n",
        "    set[set>0]=1\n",
        "\n",
        "    print(\"setores\",set.shape)\n",
        "    print(np.unique(set))\n",
        "    setores.append(set)\n",
        "\n",
        "    plt.imshow(imagens_lista[-1])\n",
        "    plt.show()\n",
        "    plt.imshow(gt_lista[-1], cmap=cmap_)\n",
        "    plt.show()\n",
        "    plt.imshow(setores[-1],cmap=cmap_)\n",
        "    plt.show()\n",
        "    plt.imshow(imagens_treino[-1])\n",
        "    plt.show()\n",
        "    plt.imshow(imagens_treino[-1])\n",
        "    plt.imshow(gt_lista[-1], cmap=cmap_, alpha=0.6)\n",
        "    plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "4156e34e-80d9-45d2-99c0-15c6e00d6f71",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "collapsed": true,
        "id": "4156e34e-80d9-45d2-99c0-15c6e00d6f71",
        "outputId": "8af18ac1-c6c5-4346-c4bd-23bd76915718",
        "scrolled": true
      },
      "outputs": [],
      "source": [
        "superpixels_fz = []\n",
        "superpixels_SLIC = []\n",
        "features = []\n",
        "for img in imagens_lista:\n",
        "    superpixel_slic, superpixel_fz, img_segmentada = superpixel_image_AllBands(img,verbose=True)\n",
        "    plt.imshow(mark_boundaries(img, superpixel_fz, mode='subpixel'))\n",
        "    plt.show()\n",
        "    plt.imshow(mark_boundaries(img, superpixel_slic, mode='subpixel'))\n",
        "    plt.show()\n",
        "    superpixels_fz.append(superpixel_fz)\n",
        "    superpixels_SLIC.append(superpixel_slic)\n",
        "    features.append(add_features(img,superpixel_fz,superpixel_slic))\n",
        "for f in features:\n",
        "    print(len(f), len(f[0]))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "0IKRTKQU5cxs",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0IKRTKQU5cxs",
        "outputId": "57461527-7830-4a10-d35a-54b892fff0d3"
      },
      "outputs": [],
      "source": [
        "features_final_lista,masks_final_lista  = exclui_setores_fora(imagens_lista,setores,features,verbose=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "id": "W577jWdG96AQ",
      "metadata": {
        "id": "W577jWdG96AQ"
      },
      "outputs": [],
      "source": [
        "treino_X = []\n",
        "treino_Y = []\n",
        "teste_X = []\n",
        "teste_Y = []\n",
        "\n",
        "X_treino = list(itertools.chain(features_final_lista[0],features_final_lista[1],features_final_lista[2],features_final_lista[3]))\n",
        "X_treino = np.asarray(X_treino)\n",
        "treino_X.append(X_treino)\n",
        "Y_treino = list(itertools.chain(masks_final_lista[0],masks_final_lista[1],masks_final_lista[2],masks_final_lista[3]))\n",
        "Y_treino = np.asarray(Y_treino)\n",
        "treino_Y.append(Y_treino)\n",
        "\n",
        "Y_teste = masks_final_lista[4]\n",
        "Y_teste = np.asarray(Y_teste)\n",
        "teste_Y.append(Y_teste)\n",
        "\n",
        "X_teste = features_final_lista[4]\n",
        "X_teste = np.asarray(X_teste)\n",
        "teste_X.append(X_teste)\n",
        "#--------------------------\n",
        "X_treino = list(itertools.chain(features_final_lista[1],features_final_lista[2],features_final_lista[3],features_final_lista[4]))\n",
        "X_treino = np.asarray(X_treino)\n",
        "treino_X.append(X_treino)\n",
        "Y_treino = list(itertools.chain(masks_final_lista[1],masks_final_lista[2],masks_final_lista[3],masks_final_lista[4]))\n",
        "Y_treino = np.asarray(Y_treino)\n",
        "treino_Y.append(Y_treino)\n",
        "\n",
        "Y_teste = masks_final_lista[0]\n",
        "Y_teste = np.asarray(Y_teste)\n",
        "teste_Y.append(Y_teste)\n",
        "\n",
        "X_teste = features_final_lista[0]\n",
        "X_teste = np.asarray(X_teste)\n",
        "teste_X.append(X_teste)\n",
        "#---------------------------\n",
        "X_treino = list(itertools.chain(features_final_lista[2],features_final_lista[3],features_final_lista[4],features_final_lista[0]))\n",
        "X_treino = np.asarray(X_treino)\n",
        "treino_X.append(X_treino)\n",
        "Y_treino = list(itertools.chain(masks_final_lista[2],masks_final_lista[3],masks_final_lista[4],masks_final_lista[0]))\n",
        "Y_treino = np.asarray(Y_treino)\n",
        "treino_Y.append(Y_treino)\n",
        "\n",
        "Y_teste = masks_final_lista[1]\n",
        "Y_teste = np.asarray(Y_teste)\n",
        "teste_Y.append(Y_teste)\n",
        "\n",
        "X_teste = features_final_lista[1]\n",
        "X_teste = np.asarray(X_teste)\n",
        "teste_X.append(X_teste)\n",
        "#---------------------------\n",
        "X_treino = list(itertools.chain(features_final_lista[3],features_final_lista[4],features_final_lista[0],features_final_lista[1]))\n",
        "X_treino = np.asarray(X_treino)\n",
        "treino_X.append(X_treino)\n",
        "Y_treino = list(itertools.chain(masks_final_lista[3],masks_final_lista[4],masks_final_lista[0],masks_final_lista[1]))\n",
        "Y_treino = np.asarray(Y_treino)\n",
        "treino_Y.append(Y_treino)\n",
        "\n",
        "Y_teste = masks_final_lista[2]\n",
        "Y_teste = np.asarray(Y_teste)\n",
        "teste_Y.append(Y_teste)\n",
        "\n",
        "X_teste = features_final_lista[2]\n",
        "X_teste = np.asarray(X_teste)\n",
        "teste_X.append(X_teste)\n",
        "#---------------------------\n",
        "X_treino = list(itertools.chain(features_final_lista[4],features_final_lista[0],features_final_lista[1],features_final_lista[2]))\n",
        "X_treino = np.asarray(X_treino)\n",
        "treino_X.append(X_treino)\n",
        "Y_treino = list(itertools.chain(masks_final_lista[4],masks_final_lista[0],masks_final_lista[1],masks_final_lista[2]))\n",
        "Y_treino = np.asarray(Y_treino)\n",
        "treino_Y.append(Y_treino)\n",
        "\n",
        "Y_teste = masks_final_lista[3]\n",
        "Y_teste = np.asarray(Y_teste)\n",
        "teste_Y.append(Y_teste)\n",
        "\n",
        "X_teste = features_final_lista[3]\n",
        "X_teste = np.asarray(X_teste)\n",
        "teste_X.append(X_teste)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "id": "6faa9ed5-ca6b-427a-b6db-2fdb9ba59c04",
      "metadata": {
        "id": "6faa9ed5-ca6b-427a-b6db-2fdb9ba59c04"
      },
      "outputs": [],
      "source": [
        "modelos=[]\n",
        "\n",
        "nome = 'RandomForest'\n",
        "modelo = RandomForestClassifier(random_state=42) #44%\n",
        "grid = {\n",
        "    'modelo__max_depth':[int(x) for x in range(2,4)],\n",
        "    'modelo__n_estimators': [int(x) for x in np.linspace(start=650, stop=1000, num=2)],\n",
        "    'modelo__criterion': ('gini', 'entropy'),\n",
        "    'modelo__class_weight': [{0:1, 1:2},{0:1, 1:4},{0:1, 1:6}],\n",
        "}\n",
        "\n",
        "modelos.append({'nome': nome, 'modelo': modelo, 'grid': grid})\n",
        "\n",
        "nome = 'XGBoost'\n",
        "modelo = XGBClassifier(objective='binary:logistic',device='cuda',subsample=0.5)\n",
        "grid = {\n",
        "    'modelo__max_depth': [int(x) for x in np.linspace(start=2, stop=5,num=3)],\n",
        "    'modelo__n_estimators': [int(x) for x in np.linspace(start=650, stop=1000, num=4)],\n",
        "    'modelo__learning_rate': [int(x) for x in np.linspace(start=0.2, stop=1,num=3)],\n",
        "    'modelo__min_child_weight': [int(x) for x in np.linspace(start=2, stop=5,num=3)],\n",
        "    'modelo__scale_pos_weight': [int(x) for x in np.linspace(start=2, stop=6,num=3)]\n",
        "}\n",
        "\n",
        "modelos.append({'nome': nome, 'modelo': modelo, 'grid': grid})\n",
        "\n",
        "nome = 'LGBMClassifier'\n",
        "modelo = lgb.LGBMClassifier(random_state=42,objective='binary')\n",
        "grid = {\n",
        "#    'modelo__num_leaves': [int(x) for x in np.linspace(start=2, stop=20,num=3)],\n",
        "    'modelo__n_estimators': [int(x) for x in np.linspace(start=600, stop=1000, num=3)],\n",
        "#    'modelo__reg_alpha': [0, 1e-1, 1],\n",
        "#    'modelo__reg_lambda': [0, 1e-1, 1],\n",
        "    'modelo__max_depth': [int(x) for x in np.linspace(start=2, stop=5,num=3)],\n",
        "    'modelo__class_weight': [{0:1, 1:2},{0:1, 1:4},{0:1, 1:6}],\n",
        "    'modelo__learning_rate': [int(x) for x in np.linspace(start=0.2, stop=1,num=3)],\n",
        "}\n",
        "\n",
        "modelos.append({'nome': nome, 'modelo': modelo, 'grid': grid})\n",
        "\n",
        "nome = 'Kmeans'\n",
        "modelo = KMeans(n_clusters=2,random_state=42)\n",
        "grid = {\n",
        "#    'modelo__n_clusters': [int(x) for x in np.linspace(start=1, stop=2, num=2)],\n",
        "    'modelo__max_iter': [int(x) for x in np.linspace(start=50, stop=500, num=10)],\n",
        "    'modelo__algorithm': ['lloyd', 'elkan'],\n",
        "}\n",
        "\n",
        "modelos.append({'nome': nome, 'modelo': modelo, 'grid': grid})\n",
        "\n",
        "nome = 'GaussianNB'\n",
        "modelo = GaussianNB()\n",
        "grid = {\n",
        "#    'modelo__n_features_in_': [int(x) for x in np.linspace(start=1, stop=20, num=3)],\n",
        "#    'modelo__epsilon_': [int(x) for x in np.linspace(start=1, stop=50, num=5)],\n",
        "}\n",
        "\n",
        "modelos.append({'nome': nome, 'modelo': modelo, 'grid': grid})\n",
        "\n",
        "nome = 'LogisticRegression'\n",
        "modelo = LogisticRegression(random_state=42)\n",
        "grid = {\n",
        "#    'modelo__penalty': ['l2', 'l1', 'elasticnet'],\n",
        "#    'modelo__solver': ['lbfgs', 'sag'],\n",
        "}\n",
        "\n",
        "modelos.append({'nome': nome, 'modelo': modelo, 'grid': grid})\n",
        "\n",
        "nome = 'MLPClassifier'\n",
        "modelo = MLPClassifier(random_state=42)\n",
        "grid = {\n",
        "    'modelo__max_iter': [int(x) for x in np.linspace(start=20, stop=100,num=3)],\n",
        "}\n",
        "\n",
        "modelos.append({'nome': nome, 'modelo': modelo, 'grid': grid})"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c21ca5c3-36d8-4afb-8a84-1cf76342bc06",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "collapsed": true,
        "id": "c21ca5c3-36d8-4afb-8a84-1cf76342bc06",
        "outputId": "0fb4fad1-7cd2-41d3-b321-35e32022f777"
      },
      "outputs": [],
      "source": [
        "%%time\n",
        "resultados = [train_test(mat, X_treino, Y_treino) for mat in modelos] #X_treino_pca\n",
        "\n",
        "info_modelo = []\n",
        "info_values = []\n",
        "info_time = []\n",
        "\n",
        "for i, v in enumerate(resultados):\n",
        "\n",
        "    info_modelo.append(resultados[i]['name'])\n",
        "    info_values.append(round(resultados[i]['object'].best_score_,3))\n",
        "    info_time.append(resultados[i]['object'].refit_time_)\n",
        "\n",
        "dados = pd.DataFrame({'Model': info_modelo, 'Score': info_values, 'Time': info_time})\n",
        "fig = ff.create_table(dados, height_constant=20)\n",
        "fig.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "b9822766",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 710
        },
        "id": "b9822766",
        "outputId": "b2259fff-da2e-416b-e654-6c33481049f2"
      },
      "outputs": [],
      "source": [
        "resultado_folds = []\n",
        "pred_folds = []\n",
        "\n",
        "for j in range(0,len(treino_X)):\n",
        "  info_modelo = []\n",
        "#  info_time = []\n",
        "  modelo_final = []\n",
        "  pred_modelo = []\n",
        "\n",
        "  resultado = pd.DataFrame(columns=['modelo','F1 Grid', 'Acurácia','F1 Score','Precisao', 'Recall'])\n",
        "  resultado_roc = pd.DataFrame(columns=['classificadores', 'fpr','tpr','auc'])\n",
        "\n",
        "  for i, v in enumerate(modelos):\n",
        "\n",
        "      info_modelo.append(modelos[i]['nome'])\n",
        "#      info_time.append(resultados[i]['object'].refit_time_)\n",
        "#      info_values = round(resultados[i]['object'].best_score_,3)\n",
        "\n",
        "#      model = resultados[i]['object'].best_estimator_\n",
        "      model = modelos[i]['modelo']\n",
        "      model.fit(treino_X[j], treino_Y[j]) #X_treino_pca\n",
        "\n",
        "      y_pred = model.predict(teste_X[j]) #X_treino_pca\n",
        "\n",
        "      acc = round(accuracy_score(teste_Y[j], y_pred),3)\n",
        "      f1 = round(f1_score(teste_Y[j], y_pred),3)\n",
        "      pc = round(precision_score(teste_Y[j], y_pred),3)\n",
        "      re = round(recall_score(teste_Y[j], y_pred),3)\n",
        "\n",
        "#      resultado = resultado._append({'modelo':resultados[i]['name'],\n",
        "      resultado = resultado._append({'modelo':modelos[i]['nome'],\n",
        "#                              'F1 Grid': info_values,\n",
        "                              'Acurácia':acc,\n",
        "                              'F1 Score':f1,\n",
        "                              'Precisao':pc,\n",
        "                              'Recall': re}, ignore_index=True)\n",
        "\n",
        "#      modelo_final.append({'nome': resultados[i]['name'], 'modelo': model})\n",
        "      modelo_final.append({'nome': modelos[i]['nome'], 'modelo': model})\n",
        "      pred_modelo.append({'nome': modelo_final[i]['nome'], 'pred': y_pred})\n",
        "\n",
        "      fpr, tpr, _ = roc_curve(teste_Y[j],  y_pred)\n",
        "      auc = roc_auc_score(teste_Y[j], y_pred)\n",
        "#      resultado_roc = resultado_roc._append({'classificadores':resultados[i]['name'],\n",
        "      resultado_roc = resultado_roc._append({'classificadores':modelos[i]['nome'],\n",
        "                                          'fpr':fpr,\n",
        "                                          'tpr':tpr,\n",
        "                                          'auc':auc}, ignore_index=True)\n",
        "\n",
        "  resultado_folds.append(resultado)\n",
        "  pred_folds.append(pred_modelo)\n",
        "  resultado_roc.set_index('classificadores', inplace=True)\n",
        "  fig = ff.create_table(resultado, height_constant=20)\n",
        "  fig.show()\n",
        "\n",
        "  fig = plt.figure(figsize=(12,6))\n",
        "  for k in resultado_roc.index:\n",
        "      plt.plot(resultado_roc.loc[k]['fpr'],\n",
        "              resultado_roc.loc[k]['tpr'],\n",
        "              label=\"{}, AUC={:.3f}\".format(k, resultado_roc.loc[k]['auc']))\n",
        "\n",
        "  plt.plot([0,1], [0,1], color='orange', linestyle='--')\n",
        "\n",
        "  plt.xticks(np.arange(0.0, 1.1, step=0.1))\n",
        "  plt.xlabel(\"Taxa de Falso Positivo\", fontsize=15)\n",
        "\n",
        "  plt.yticks(np.arange(0.0, 1.1, step=0.1))\n",
        "  plt.ylabel(\"Taxa de Verdadeiro Positivo\", fontsize=15)\n",
        "\n",
        "  plt.title('Curva ROC', fontweight='bold', fontsize=15)\n",
        "  plt.legend(prop={'size':13}, loc='lower right')\n",
        "\n",
        "  #plt.savefig('roc_AGNS.png', format='png', dpi=100)\n",
        "  plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "869c169b",
      "metadata": {},
      "outputs": [],
      "source": [
        "superpixels_fz = []\n",
        "superpixels_SLIC = []\n",
        "features = []\n",
        "for img in imagens_treino:\n",
        "    superpixel_slic, superpixel_fz, img_segmentada = superpixel_image_AllBands(img,verbose=True)\n",
        "    plt.imshow(mark_boundaries(img, superpixel_fz, mode='subpixel'))\n",
        "    plt.show()\n",
        "    plt.imshow(mark_boundaries(img, superpixel_slic, mode='subpixel'))\n",
        "    plt.show()\n",
        "    superpixels_fz.append(superpixel_fz)\n",
        "    superpixels_SLIC.append(superpixel_slic)\n",
        "    features.append(add_features(img,superpixel_fz,superpixel_slic))\n",
        "for f in features:\n",
        "    print(len(f), len(f[0]))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "id": "36484d13",
      "metadata": {},
      "outputs": [],
      "source": [
        "treino_X = []\n",
        "treino_Y = []\n",
        "teste_X = []\n",
        "teste_Y = []\n",
        "\n",
        "X_treino = list(itertools.chain(features[1],features[2],features[3],features[4]))\n",
        "X_treino = np.asarray(X_treino)\n",
        "treino_X.append(X_treino)\n",
        "Y_treino = list(itertools.chain(gt_lista[1],gt_lista[2],gt_lista[3],gt_lista[4]))\n",
        "Y_treino = np.asarray(Y_treino)\n",
        "treino_Y.append(Y_treino)\n",
        "\n",
        "Y_teste = gt_lista[0]\n",
        "Y_teste = np.asarray(Y_teste)\n",
        "teste_Y.append(Y_teste)\n",
        "\n",
        "X_teste = features[0]\n",
        "X_teste = np.asarray(X_teste)\n",
        "teste_X.append(X_teste)\n",
        "#--------------------------\n",
        "X_treino = list(itertools.chain(features[0],features[2],features[3],features[4]))\n",
        "X_treino = np.asarray(X_treino)\n",
        "treino_X.append(X_treino)\n",
        "Y_treino = list(itertools.chain(gt_lista[0],gt_lista[2],gt_lista[3],gt_lista[4]))\n",
        "Y_treino = np.asarray(Y_treino)\n",
        "treino_Y.append(Y_treino)\n",
        "\n",
        "Y_teste = gt_lista[1]\n",
        "Y_teste = np.asarray(Y_teste)\n",
        "teste_Y.append(Y_teste)\n",
        "\n",
        "X_teste = features[1]\n",
        "X_teste = np.asarray(X_teste)\n",
        "teste_X.append(X_teste)\n",
        "#---------------------------\n",
        "X_treino = list(itertools.chain(features[1],features[3],features[4],features[0]))\n",
        "X_treino = np.asarray(X_treino)\n",
        "treino_X.append(X_treino)\n",
        "Y_treino = list(itertools.chain(gt_lista[1],gt_lista[3],gt_lista[4],gt_lista[0]))\n",
        "Y_treino = np.asarray(Y_treino)\n",
        "treino_Y.append(Y_treino)\n",
        "\n",
        "Y_teste = gt_lista[2]\n",
        "Y_teste = np.asarray(Y_teste)\n",
        "teste_Y.append(Y_teste)\n",
        "\n",
        "X_teste = features[2]\n",
        "X_teste = np.asarray(X_teste)\n",
        "teste_X.append(X_teste)\n",
        "#---------------------------\n",
        "X_treino = list(itertools.chain(features[2],features[4],features[0],features[1]))\n",
        "X_treino = np.asarray(X_treino)\n",
        "treino_X.append(X_treino)\n",
        "Y_treino = list(itertools.chain(gt_lista[2],gt_lista[4],gt_lista[0],gt_lista[1]))\n",
        "Y_treino = np.asarray(Y_treino)\n",
        "treino_Y.append(Y_treino)\n",
        "\n",
        "Y_teste = gt_lista[3]\n",
        "Y_teste = np.asarray(Y_teste)\n",
        "teste_Y.append(Y_teste)\n",
        "\n",
        "X_teste = features[3]\n",
        "X_teste = np.asarray(X_teste)\n",
        "teste_X.append(X_teste)\n",
        "#---------------------------\n",
        "X_treino = list(itertools.chain(features[3],features[0],features[1],features[2]))\n",
        "X_treino = np.asarray(X_treino)\n",
        "treino_X.append(X_treino)\n",
        "Y_treino = list(itertools.chain(gt_lista[3],gt_lista[0],gt_lista[1],gt_lista[2]))\n",
        "Y_treino = np.asarray(Y_treino)\n",
        "treino_Y.append(Y_treino)\n",
        "\n",
        "Y_teste = gt_lista[4]\n",
        "Y_teste = np.asarray(Y_teste)\n",
        "teste_Y.append(Y_teste)\n",
        "\n",
        "X_teste = features[4]\n",
        "X_teste = np.asarray(X_teste)\n",
        "teste_X.append(X_teste)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "id": "70da2e53",
      "metadata": {},
      "outputs": [],
      "source": [
        "for i in range(0,4):\n",
        "    teste_X[i][np.isnan(teste_X[i])] = 0\n",
        "    treino_X[i][np.isnan(treino_X[i])] = 0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "dcf68433",
      "metadata": {},
      "outputs": [],
      "source": [
        "#Refazer a imagem\n",
        "iou = MeanIoU(num_classes=2)\n",
        "for i in range(0, len(imagens_treino)):\n",
        "    for j in range(0,len(modelos)):\n",
        "        modelo = modelos[j]['modelo']\n",
        "        modelo.fit(treino_X[i].reshape((treino_X[i].shape[0]*treino_X[i].shape[1],treino_X[i].shape[2])),treino_Y[i].reshape((treino_Y[i].shape[0]*treino_Y[i].shape[1],1)).ravel())\n",
        "\n",
        "        y_pred = modelo.predict(teste_X[i].reshape((teste_X[i].shape[0]*teste_X[i].shape[1],teste_X[i].shape[2])))\n",
        "        \n",
        "        fig, ax = plt.subplots(1,2, figsize=(10, 10), sharex=True, sharey=True)\n",
        "\n",
        "        ax[0].imshow(gt_lista[i], cmap=cmap_)\n",
        "        ax[0].set_title(\"Máscara\")\n",
        "        ax[1].imshow(np.reshape(y_pred,(img.shape[0],img.shape[1])), cmap=cmap_)\n",
        "        ax[1].set_title(modelos[j]['nome'])\n",
        "\n",
        "        plt.tight_layout()\n",
        "        plt.show()\n",
        "\n",
        "        iou.update_state(gt_lista[i].reshape((500*500)),y_pred)\n",
        "        mIoU = iou.result().numpy()\n",
        "        print(\"IoU\",f\"{modelos[j]['nome']} =\", f\"{mIoU*100:.2f}%\")\n",
        "#Fazer a média dos scores"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [
        "cubL3DjOhOtT",
        "_98l2dQFmhVN"
      ],
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.5"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}