Field_Aggregation_Mean.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.4"
    },
    "colab": {
      "name": "Field_Aggregation_Mean.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "7b430419c8e04c91acafc35a53118cf3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8466a8ec4bc347b4b69f220d31679012",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2bce2a23961544b1960e268693e33947",
              "IPY_MODEL_89380e275f19406a8614dace6af1f99b",
              "IPY_MODEL_54724ef8ab44408c8fcc616610dd9bdb"
            ]
          }
        },
        "8466a8ec4bc347b4b69f220d31679012": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2bce2a23961544b1960e268693e33947": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1bd7d3e972804c65bed5fdc30011d9f1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b771437f48194bad9dbcc254019b6803"
          }
        },
        "89380e275f19406a8614dace6af1f99b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e8fa90fbc2b741ad988e6b5d13af6d67",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 10,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 10,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f649ba6bc4b14c7ab06d08da5fe3b7de"
          }
        },
        "54724ef8ab44408c8fcc616610dd9bdb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e5be528350c54c34b702a87be4c16b40",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 10/10 [00:11&lt;00:00,  1.58it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_355037c622754c2bba10bd49c8f25754"
          }
        },
        "1bd7d3e972804c65bed5fdc30011d9f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b771437f48194bad9dbcc254019b6803": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e8fa90fbc2b741ad988e6b5d13af6d67": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f649ba6bc4b14c7ab06d08da5fe3b7de": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e5be528350c54c34b702a87be4c16b40": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "355037c622754c2bba10bd49c8f25754": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dfc9917c7f834a50a4243f9c2b0c7acf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_44459dfef2754dfe8c5c5a4ff37dcf4a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d0fcacb5d7624f04a939f93dc6ce2263",
              "IPY_MODEL_02bd460511784a6daf3dbe9d1ef63443",
              "IPY_MODEL_2f5def119b2745018845e600fd6478bf"
            ]
          }
        },
        "44459dfef2754dfe8c5c5a4ff37dcf4a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d0fcacb5d7624f04a939f93dc6ce2263": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ce26a50eec88416384396ea39d569c61",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0f9efdd1b2214514be75d26fcdb18385"
          }
        },
        "02bd460511784a6daf3dbe9d1ef63443": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_cfcd541aac3f4c4d8b848d93599feaaf",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 10,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 10,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_87cf6567215b4ba8b02ae7fe4e680e6d"
          }
        },
        "2f5def119b2745018845e600fd6478bf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d66ad955b2c34d15bd37b2bde9d0439b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 10/10 [00:00&lt;00:00, 89.67it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ccfe7c215f9942c99ce74be1931161c8"
          }
        },
        "ce26a50eec88416384396ea39d569c61": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0f9efdd1b2214514be75d26fcdb18385": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cfcd541aac3f4c4d8b848d93599feaaf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "87cf6567215b4ba8b02ae7fe4e680e6d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d66ad955b2c34d15bd37b2bde9d0439b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ccfe7c215f9942c99ce74be1931161c8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c05b441e442f4c2396c9ba920423e05a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_80ac4e5eaa574d1d9796dafb61c968b1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_dbdc3f534915460d8c094fe3fbf6d3e5",
              "IPY_MODEL_a930671d0de84ca093dcb8b4e3cb1d0d",
              "IPY_MODEL_62b1340ef5b146a69f5426ddd31a132e"
            ]
          }
        },
        "80ac4e5eaa574d1d9796dafb61c968b1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dbdc3f534915460d8c094fe3fbf6d3e5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b760c18a75cb4a3785b0aa429ac21660",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c02d098387c44ac899bcc1a5ef97aeaa"
          }
        },
        "a930671d0de84ca093dcb8b4e3cb1d0d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_81305ee731aa4b958c71a180aedcc7a2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 5,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 5,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f37f5d60c87b4652b46b6175882c9caa"
          }
        },
        "62b1340ef5b146a69f5426ddd31a132e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4843dbc1f2dd4701a136f87dfa1b940e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 5/5 [00:01&lt;00:00,  2.52it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e7f81c8eee9e460faa0c2a8eba58f3c9"
          }
        },
        "b760c18a75cb4a3785b0aa429ac21660": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c02d098387c44ac899bcc1a5ef97aeaa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "81305ee731aa4b958c71a180aedcc7a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f37f5d60c87b4652b46b6175882c9caa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4843dbc1f2dd4701a136f87dfa1b940e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e7f81c8eee9e460faa0c2a8eba58f3c9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dc7d7680505d44999e165717debcd1a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_63cf9d3439294863a5728040904c8ad7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2bfaac11391a4adfb91433e0f3368683",
              "IPY_MODEL_788a1eafa3814b9399816accd124f6eb",
              "IPY_MODEL_d984a74eb7ad432a82fde659554da5b9"
            ]
          }
        },
        "63cf9d3439294863a5728040904c8ad7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2bfaac11391a4adfb91433e0f3368683": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_50696b849220459495b9b9a091ce05bb",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_72efa9c92d01410dbca8b357bc581e56"
          }
        },
        "788a1eafa3814b9399816accd124f6eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5be0c6d165d748fcbb017817fd29c708",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 5,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 5,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f3591ac91f184068827b659c302bbe99"
          }
        },
        "d984a74eb7ad432a82fde659554da5b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9bca2e1c229b41d7add2792d72015b3a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 5/5 [00:00&lt;00:00, 86.87it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_03f86a1edb104dd2b55a587599a8339e"
          }
        },
        "50696b849220459495b9b9a091ce05bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "72efa9c92d01410dbca8b357bc581e56": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5be0c6d165d748fcbb017817fd29c708": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f3591ac91f184068827b659c302bbe99": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9bca2e1c229b41d7add2792d72015b3a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "03f86a1edb104dd2b55a587599a8339e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/DariusTheGeek/Radiant-Earth-Spot-the-Crop-XL-Challenge/blob/main/Field_Aggregation_Mean.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SmtqOeQpmANk"
      },
      "source": [
        "#### Use this notebook to calculate mean of pixels per field for \n",
        "  - Sentinel one data\n",
        "  - Sentinel 2 data with 25 periods/dates\n",
        "  - Sentinel 2 data with month start month end dates\n",
        "\n",
        "**All you have to change is the path to the files**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mr9kNjYMfssv",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8b873aec-cba7-47df-82b2-ab1170e1062f"
      },
      "source": [
        "!pip -qq install pyspark findspark\n",
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
        "!wget -q https://www-us.apache.org/dist/spark/spark-3.0.1/spark-3.0.1-bin-hadoop2.7.tgz"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[K     |████████████████████████████████| 212.4 MB 48 kB/s \n",
            "\u001b[K     |████████████████████████████████| 198 kB 50.4 MB/s \n",
            "\u001b[?25h  Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_0YW6UeGfiZo"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import os\n",
        "import glob\n",
        "from tqdm.notebook import tqdm_notebook\n",
        "import pyspark\n",
        "from pyspark import SparkContext\n",
        "from pyspark.sql import SparkSession\n",
        "import pyspark.sql.functions as F\n",
        "from pyspark.sql import Row, DataFrame, Column\n",
        "from pyspark.sql.functions import lit\n",
        "\n",
        "from functools import reduce"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VeJYJfIefiZr"
      },
      "source": [
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6Pr1lA3HEA7P"
      },
      "source": [
        "spark = SparkSession.builder \\\n",
        "    .master('local[*]') \\\n",
        "    .config(\"spark.driver.memory\", \"15g\") \\\n",
        "    .appName('my-cool-app') \\\n",
        "    .getOrCreate()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 214
        },
        "id": "p-lXDoJ7smXI",
        "outputId": "00dfad31-12f5-4b54-ae9b-c379af98eb47"
      },
      "source": [
        "spark"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "            <div>\n",
              "                <p><b>SparkSession - in-memory</b></p>\n",
              "                \n",
              "        <div>\n",
              "            <p><b>SparkContext</b></p>\n",
              "\n",
              "            <p><a href=\"http://8a3235ada4bd:4040\">Spark UI</a></p>\n",
              "\n",
              "            <dl>\n",
              "              <dt>Version</dt>\n",
              "                <dd><code>v3.1.2</code></dd>\n",
              "              <dt>Master</dt>\n",
              "                <dd><code>local[*]</code></dd>\n",
              "              <dt>AppName</dt>\n",
              "                <dd><code>my-cool-app</code></dd>\n",
              "            </dl>\n",
              "        </div>\n",
              "        \n",
              "            </div>\n",
              "        "
            ],
            "text/plain": [
              "<pyspark.sql.session.SparkSession at 0x7fe974a423d0>"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zPAm0Gykyo6M",
        "outputId": "b1ddfce2-eeb8-4936-8586-3eb6e85ab041"
      },
      "source": [
        "train = pd.read_csv('/content/drive/MyDrive/CompeData/Radiant/train_file.csv')\n",
        "field_ids = train.field.unique()\n",
        "len(field_ids)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "87114"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CpH-dnlrB1HD",
        "outputId": "65b14ec0-9fcf-4837-c5c2-1d0dc1a264b2"
      },
      "source": [
        "train.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(87114, 106)"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FzO9EQksO5sb"
      },
      "source": [
        "## SAR"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 82,
          "referenced_widgets": [
            "7b430419c8e04c91acafc35a53118cf3",
            "8466a8ec4bc347b4b69f220d31679012",
            "2bce2a23961544b1960e268693e33947",
            "89380e275f19406a8614dace6af1f99b",
            "54724ef8ab44408c8fcc616610dd9bdb",
            "1bd7d3e972804c65bed5fdc30011d9f1",
            "b771437f48194bad9dbcc254019b6803",
            "e8fa90fbc2b741ad988e6b5d13af6d67",
            "f649ba6bc4b14c7ab06d08da5fe3b7de",
            "e5be528350c54c34b702a87be4c16b40",
            "355037c622754c2bba10bd49c8f25754"
          ]
        },
        "id": "-0HJ-fPHO4lC",
        "outputId": "7034e8c0-d17c-4ef0-fcd6-2a0945299e24"
      },
      "source": [
        "%%time\n",
        "dfs = []\n",
        "for i in tqdm_notebook(glob.glob('/content/drive/MyDrive/CompeData/Radiant/s1_all_data/s1_train*.csv')):\n",
        "  dfs.append(spark.read.csv(path = i, sep =',', encoding = 'UTF-8', comment = None, header = True))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7b430419c8e04c91acafc35a53118cf3",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/10 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 165 ms, sys: 19.9 ms, total: 184 ms\n",
            "Wall time: 11.1 s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "B8WCJBwJO4Z1",
        "outputId": "a342fc55-4f13-4675-8c50-1e7f89580080"
      },
      "source": [
        "dfs[0].show(5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+---+------+\n",
            "|04_0_VV|04_0_VH|04_1_VV|04_1_VH|04_2_VV|04_2_VH|04_3_VV|04_3_VH|04_4_VV|04_4_VH|04_5_VV|04_5_VH|05_0_VV|05_0_VH|05_1_VV|05_1_VH|05_2_VV|05_2_VH|05_3_VV|05_3_VH|05_4_VV|05_4_VH|06_0_VV|06_0_VH|06_1_VV|06_1_VH|06_2_VV|06_2_VH|06_3_VV|06_3_VH|06_4_VV|06_4_VH|07_0_VV|07_0_VH|07_1_VV|07_1_VH|07_2_VV|07_2_VH|07_3_VV|07_3_VH|07_4_VV|07_4_VH|08_0_VV|08_0_VH|08_1_VV|08_1_VH|08_2_VV|08_2_VH|08_3_VV|08_3_VH|08_4_VV|08_4_VH|09_0_VV|09_0_VH|09_1_VV|09_1_VH|09_2_VV|09_2_VH|09_3_VV|09_3_VH|09_4_VV|09_4_VH|10_0_VV|10_0_VH|10_1_VV|10_1_VH|10_2_VV|10_2_VH|10_3_VV|10_3_VH|10_4_VV|10_4_VH|11_0_VV|11_0_VH|11_1_VV|11_1_VH|11_2_VV|11_2_VH|11_3_VV|11_3_VH|11_4_VV|11_4_VH|  y|fields|\n",
            "+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+---+------+\n",
            "|    7.0|    3.0|    7.0|    3.0|   16.0|    4.0|    9.0|    2.0|    8.0|    1.0|   24.0|   13.0|    4.0|    6.0|    4.0|    2.0|    6.0|    2.0|   42.0|    9.0|   15.0|    2.0|   11.0|    3.0|   16.0|    4.0|   35.0|    4.0|   12.0|    5.0|   26.0|    5.0|    6.0|    3.0|    9.0|    3.0|   10.0|    1.0|   13.0|    4.0|   13.0|    1.0|    7.0|    3.0|   17.0|    2.0|   24.0|    3.0|   24.0|    4.0|   14.0|    7.0|   14.0|    3.0|   20.0|    3.0|    3.0|    2.0|   12.0|    7.0|    8.0|    4.0|   35.0|    7.0|   19.0|   10.0|   71.0|    5.0|    8.0|    7.0|   11.0|    4.0|   23.0|    7.0|    4.0|    4.0|    5.0|    5.0|   12.0|    7.0|   18.0|    1.0|  0|     0|\n",
            "|   11.0|    2.0|    6.0|    1.0|    9.0|    1.0|    7.0|    3.0|    8.0|    0.0|   19.0|    4.0|    6.0|    0.0|    5.0|    1.0|    9.0|    1.0|   44.0|    2.0|    6.0|    1.0|    9.0|    1.0|   23.0|    6.0|   24.0|    1.0|   13.0|    3.0|   30.0|    2.0|    8.0|    3.0|    3.0|    3.0|   19.0|    1.0|   16.0|    4.0|   14.0|    3.0|    7.0|    3.0|   36.0|    1.0|   36.0|    4.0|   24.0|    5.0|   19.0|    2.0|   14.0|    5.0|   24.0|   10.0|    3.0|    9.0|    8.0|    4.0|   18.0|   15.0|   18.0|    6.0|   20.0|    7.0|   73.0|    5.0|    7.0|    4.0|   12.0|    4.0|   11.0|    1.0|    8.0|    2.0|    5.0|    0.0|   12.0|    3.0|   14.0|    2.0|  0|     0|\n",
            "|   12.0|    3.0|    6.0|    2.0|   13.0|    1.0|    6.0|    3.0|    5.0|    0.0|   19.0|    7.0|    6.0|    1.0|    5.0|    1.0|   11.0|    2.0|   32.0|    1.0|    3.0|    1.0|    7.0|    1.0|   21.0|    2.0|    6.0|    1.0|    6.0|    0.0|   19.0|    2.0|   10.0|    1.0|    3.0|    5.0|    3.0|    0.0|   13.0|    2.0|    7.0|    5.0|    5.0|    2.0|   44.0|    2.0|   31.0|    1.0|   16.0|    4.0|   11.0|    3.0|    9.0|    2.0|   13.0|    2.0|    8.0|    6.0|   10.0|    6.0|   19.0|   12.0|   10.0|    2.0|    9.0|    3.0|   35.0|    8.0|   10.0|    2.0|    9.0|    1.0|   31.0|    3.0|   12.0|    1.0|    8.0|    0.0|   11.0|    2.0|   14.0|    1.0|  0|     0|\n",
            "|   12.0|    3.0|    3.0|    2.0|   13.0|    0.0|    4.0|    2.0|   13.0|    0.0|   27.0|    6.0|    6.0|    3.0|    6.0|    1.0|   12.0|    3.0|   20.0|    1.0|   12.0|    1.0|    7.0|    0.0|   16.0|    2.0|    8.0|    1.0|    7.0|    1.0|    9.0|    1.0|    1.0|    0.0|    8.0|    5.0|    2.0|    0.0|    2.0|    0.0|    4.0|    3.0|    7.0|    1.0|   37.0|    3.0|   19.0|    5.0|   18.0|    5.0|   10.0|    4.0|    6.0|    1.0|    4.0|    1.0|    6.0|    3.0|   10.0|    3.0|   10.0|    2.0|   11.0|    4.0|   11.0|    1.0|   29.0|    5.0|   17.0|    1.0|    8.0|    2.0|   18.0|    3.0|    7.0|    1.0|    4.0|    1.0|    8.0|    1.0|    7.0|    2.0|  0|     0|\n",
            "|   17.0|    3.0|    2.0|    1.0|    6.0|    0.0|    3.0|    0.0|   10.0|    2.0|   72.0|    5.0|    6.0|    2.0|    3.0|    1.0|   16.0|    0.0|   14.0|    1.0|   10.0|    1.0|   15.0|    0.0|   36.0|    2.0|   17.0|    1.0|    9.0|    2.0|    8.0|    2.0|    6.0|    0.0|   16.0|    1.0|    7.0|    0.0|    2.0|    1.0|    9.0|    2.0|    9.0|    2.0|   32.0|    4.0|   23.0|    6.0|    6.0|    4.0|    4.0|    0.0|   11.0|    2.0|    8.0|    1.0|   11.0|    0.0|    5.0|    2.0|   10.0|    3.0|   12.0|    3.0|   15.0|    2.0|   38.0|    7.0|   21.0|    1.0|    9.0|    1.0|   25.0|    1.0|    5.0|    1.0|    4.0|    1.0|    5.0|    1.0|    3.0|    0.0|  0|     0|\n",
            "+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+---+------+\n",
            "only showing top 5 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 82,
          "referenced_widgets": [
            "dfc9917c7f834a50a4243f9c2b0c7acf",
            "44459dfef2754dfe8c5c5a4ff37dcf4a",
            "d0fcacb5d7624f04a939f93dc6ce2263",
            "02bd460511784a6daf3dbe9d1ef63443",
            "2f5def119b2745018845e600fd6478bf",
            "ce26a50eec88416384396ea39d569c61",
            "0f9efdd1b2214514be75d26fcdb18385",
            "cfcd541aac3f4c4d8b848d93599feaaf",
            "87cf6567215b4ba8b02ae7fe4e680e6d",
            "d66ad955b2c34d15bd37b2bde9d0439b",
            "ccfe7c215f9942c99ce74be1931161c8"
          ]
        },
        "id": "LOruCmAUPhk5",
        "outputId": "5ffc21f2-944d-4793-8ade-671117d7bf30"
      },
      "source": [
        "%%time\n",
        "cols = set()\n",
        "for j in tqdm_notebook(dfs):\n",
        "    for x in j.columns:\n",
        "        cols.add(x)\n",
        "cols = sorted(cols)\n",
        "\n",
        "# Create a dictionary with all the dataframes\n",
        "all_data = {}\n",
        "for i, d in enumerate(dfs):\n",
        "    new_name = 'new_df' + str(i)  # New name for the key, the dataframe is the value\n",
        "    all_data[new_name] = d\n",
        "    # Loop through all column names. Add the missing columns to the dataframe (with value 0)\n",
        "    for x in cols:\n",
        "        if x not in d.columns:\n",
        "            all_data[new_name] = all_data[new_name].withColumn(x, F.lit(0))\n",
        "    all_data[new_name] = all_data[new_name].select(cols)  # Use 'select' to get the columns sorted"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "dfc9917c7f834a50a4243f9c2b0c7acf",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/10 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 418 ms, sys: 97.2 ms, total: 515 ms\n",
            "Wall time: 2.52 s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9hPrNtFRPzhA",
        "outputId": "118716ce-43a3-4fc0-9444-ed1f85aa5094"
      },
      "source": [
        "len(dfs)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7RndGX9vP4mk",
        "outputId": "124c2062-54f0-4965-be24-fbd067525a4e"
      },
      "source": [
        "%%time\n",
        "all_data_dfs = [all_data['new_df'+str(i)] for i in range(10)]\n",
        "final_data = reduce(DataFrame.unionAll, all_data_dfs)\n",
        "final_data.show(5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+------+---+\n",
            "|04_0_VH|04_0_VV|04_1_VH|04_1_VV|04_2_VH|04_2_VV|04_3_VH|04_3_VV|04_4_VH|04_4_VV|04_5_VH|04_5_VV|05_0_VH|05_0_VV|05_1_VH|05_1_VV|05_2_VH|05_2_VV|05_3_VH|05_3_VV|05_4_VH|05_4_VV|06_0_VH|06_0_VV|06_1_VH|06_1_VV|06_2_VH|06_2_VV|06_3_VH|06_3_VV|06_4_VH|06_4_VV|07_0_VH|07_0_VV|07_1_VH|07_1_VV|07_2_VH|07_2_VV|07_3_VH|07_3_VV|07_4_VH|07_4_VV|08_0_VH|08_0_VV|08_1_VH|08_1_VV|08_2_VH|08_2_VV|08_3_VH|08_3_VV|08_4_VH|08_4_VV|09_0_VH|09_0_VV|09_1_VH|09_1_VV|09_2_VH|09_2_VV|09_3_VH|09_3_VV|09_4_VH|09_4_VV|10_0_VH|10_0_VV|10_1_VH|10_1_VV|10_2_VH|10_2_VV|10_3_VH|10_3_VV|10_4_VH|10_4_VV|11_0_VH|11_0_VV|11_1_VH|11_1_VV|11_2_VH|11_2_VV|11_3_VH|11_3_VV|11_4_VH|11_4_VV|fields|  y|\n",
            "+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+------+---+\n",
            "|    3.0|    7.0|    3.0|    7.0|    4.0|   16.0|    2.0|    9.0|    1.0|    8.0|   13.0|   24.0|    6.0|    4.0|    2.0|    4.0|    2.0|    6.0|    9.0|   42.0|    2.0|   15.0|    3.0|   11.0|    4.0|   16.0|    4.0|   35.0|    5.0|   12.0|    5.0|   26.0|    3.0|    6.0|    3.0|    9.0|    1.0|   10.0|    4.0|   13.0|    1.0|   13.0|    3.0|    7.0|    2.0|   17.0|    3.0|   24.0|    4.0|   24.0|    7.0|   14.0|    3.0|   14.0|    3.0|   20.0|    2.0|    3.0|    7.0|   12.0|    4.0|    8.0|    7.0|   35.0|   10.0|   19.0|    5.0|   71.0|    7.0|    8.0|    4.0|   11.0|    7.0|   23.0|    4.0|    4.0|    5.0|    5.0|    7.0|   12.0|    1.0|   18.0|     0|  0|\n",
            "|    2.0|   11.0|    1.0|    6.0|    1.0|    9.0|    3.0|    7.0|    0.0|    8.0|    4.0|   19.0|    0.0|    6.0|    1.0|    5.0|    1.0|    9.0|    2.0|   44.0|    1.0|    6.0|    1.0|    9.0|    6.0|   23.0|    1.0|   24.0|    3.0|   13.0|    2.0|   30.0|    3.0|    8.0|    3.0|    3.0|    1.0|   19.0|    4.0|   16.0|    3.0|   14.0|    3.0|    7.0|    1.0|   36.0|    4.0|   36.0|    5.0|   24.0|    2.0|   19.0|    5.0|   14.0|   10.0|   24.0|    9.0|    3.0|    4.0|    8.0|   15.0|   18.0|    6.0|   18.0|    7.0|   20.0|    5.0|   73.0|    4.0|    7.0|    4.0|   12.0|    1.0|   11.0|    2.0|    8.0|    0.0|    5.0|    3.0|   12.0|    2.0|   14.0|     0|  0|\n",
            "|    3.0|   12.0|    2.0|    6.0|    1.0|   13.0|    3.0|    6.0|    0.0|    5.0|    7.0|   19.0|    1.0|    6.0|    1.0|    5.0|    2.0|   11.0|    1.0|   32.0|    1.0|    3.0|    1.0|    7.0|    2.0|   21.0|    1.0|    6.0|    0.0|    6.0|    2.0|   19.0|    1.0|   10.0|    5.0|    3.0|    0.0|    3.0|    2.0|   13.0|    5.0|    7.0|    2.0|    5.0|    2.0|   44.0|    1.0|   31.0|    4.0|   16.0|    3.0|   11.0|    2.0|    9.0|    2.0|   13.0|    6.0|    8.0|    6.0|   10.0|   12.0|   19.0|    2.0|   10.0|    3.0|    9.0|    8.0|   35.0|    2.0|   10.0|    1.0|    9.0|    3.0|   31.0|    1.0|   12.0|    0.0|    8.0|    2.0|   11.0|    1.0|   14.0|     0|  0|\n",
            "|    3.0|   12.0|    2.0|    3.0|    0.0|   13.0|    2.0|    4.0|    0.0|   13.0|    6.0|   27.0|    3.0|    6.0|    1.0|    6.0|    3.0|   12.0|    1.0|   20.0|    1.0|   12.0|    0.0|    7.0|    2.0|   16.0|    1.0|    8.0|    1.0|    7.0|    1.0|    9.0|    0.0|    1.0|    5.0|    8.0|    0.0|    2.0|    0.0|    2.0|    3.0|    4.0|    1.0|    7.0|    3.0|   37.0|    5.0|   19.0|    5.0|   18.0|    4.0|   10.0|    1.0|    6.0|    1.0|    4.0|    3.0|    6.0|    3.0|   10.0|    2.0|   10.0|    4.0|   11.0|    1.0|   11.0|    5.0|   29.0|    1.0|   17.0|    2.0|    8.0|    3.0|   18.0|    1.0|    7.0|    1.0|    4.0|    1.0|    8.0|    2.0|    7.0|     0|  0|\n",
            "|    3.0|   17.0|    1.0|    2.0|    0.0|    6.0|    0.0|    3.0|    2.0|   10.0|    5.0|   72.0|    2.0|    6.0|    1.0|    3.0|    0.0|   16.0|    1.0|   14.0|    1.0|   10.0|    0.0|   15.0|    2.0|   36.0|    1.0|   17.0|    2.0|    9.0|    2.0|    8.0|    0.0|    6.0|    1.0|   16.0|    0.0|    7.0|    1.0|    2.0|    2.0|    9.0|    2.0|    9.0|    4.0|   32.0|    6.0|   23.0|    4.0|    6.0|    0.0|    4.0|    2.0|   11.0|    1.0|    8.0|    0.0|   11.0|    2.0|    5.0|    3.0|   10.0|    3.0|   12.0|    2.0|   15.0|    7.0|   38.0|    1.0|   21.0|    1.0|    9.0|    1.0|   25.0|    1.0|    5.0|    1.0|    4.0|    1.0|    5.0|    0.0|    3.0|     0|  0|\n",
            "+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+------+---+\n",
            "only showing top 5 rows\n",
            "\n",
            "CPU times: user 12.7 ms, sys: 122 µs, total: 12.8 ms\n",
            "Wall time: 1.75 s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lVtaJ0btQEzO"
      },
      "source": [
        "final_data = final_data.select(*(F.col(c).cast(\"float\").alias(c) for c in final_data.columns))\n",
        "final_data.printSchema()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6082CzASQ6qA",
        "outputId": "0aae82aa-0ad6-4909-f3ba-b6a54705a877"
      },
      "source": [
        "%%time\n",
        "final_data_mean = final_data.groupBy('fields').mean().toPandas()\n",
        "final_data_mean.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 22.7 s, sys: 2.51 s, total: 25.2 s\n",
            "Wall time: 59min 1s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 252
        },
        "id": "4xTXELIwQ6kr",
        "outputId": "83494c8c-882b-45e9-dde8-38e57ec15c7d"
      },
      "source": [
        "final_data_mean.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>fields</th>\n",
              "      <th>avg(04_0_VH)</th>\n",
              "      <th>avg(04_0_VV)</th>\n",
              "      <th>avg(04_1_VH)</th>\n",
              "      <th>avg(04_1_VV)</th>\n",
              "      <th>avg(04_2_VH)</th>\n",
              "      <th>avg(04_2_VV)</th>\n",
              "      <th>avg(04_3_VH)</th>\n",
              "      <th>avg(04_3_VV)</th>\n",
              "      <th>avg(04_4_VH)</th>\n",
              "      <th>avg(04_4_VV)</th>\n",
              "      <th>avg(04_5_VH)</th>\n",
              "      <th>avg(04_5_VV)</th>\n",
              "      <th>avg(05_0_VH)</th>\n",
              "      <th>avg(05_0_VV)</th>\n",
              "      <th>avg(05_1_VH)</th>\n",
              "      <th>avg(05_1_VV)</th>\n",
              "      <th>avg(05_2_VH)</th>\n",
              "      <th>avg(05_2_VV)</th>\n",
              "      <th>avg(05_3_VH)</th>\n",
              "      <th>avg(05_3_VV)</th>\n",
              "      <th>avg(05_4_VH)</th>\n",
              "      <th>avg(05_4_VV)</th>\n",
              "      <th>avg(06_0_VH)</th>\n",
              "      <th>avg(06_0_VV)</th>\n",
              "      <th>avg(06_1_VH)</th>\n",
              "      <th>avg(06_1_VV)</th>\n",
              "      <th>avg(06_2_VH)</th>\n",
              "      <th>avg(06_2_VV)</th>\n",
              "      <th>avg(06_3_VH)</th>\n",
              "      <th>avg(06_3_VV)</th>\n",
              "      <th>avg(06_4_VH)</th>\n",
              "      <th>avg(06_4_VV)</th>\n",
              "      <th>avg(07_0_VH)</th>\n",
              "      <th>avg(07_0_VV)</th>\n",
              "      <th>avg(07_1_VH)</th>\n",
              "      <th>avg(07_1_VV)</th>\n",
              "      <th>avg(07_2_VH)</th>\n",
              "      <th>avg(07_2_VV)</th>\n",
              "      <th>avg(07_3_VH)</th>\n",
              "      <th>...</th>\n",
              "      <th>avg(08_1_VH)</th>\n",
              "      <th>avg(08_1_VV)</th>\n",
              "      <th>avg(08_2_VH)</th>\n",
              "      <th>avg(08_2_VV)</th>\n",
              "      <th>avg(08_3_VH)</th>\n",
              "      <th>avg(08_3_VV)</th>\n",
              "      <th>avg(08_4_VH)</th>\n",
              "      <th>avg(08_4_VV)</th>\n",
              "      <th>avg(09_0_VH)</th>\n",
              "      <th>avg(09_0_VV)</th>\n",
              "      <th>avg(09_1_VH)</th>\n",
              "      <th>avg(09_1_VV)</th>\n",
              "      <th>avg(09_2_VH)</th>\n",
              "      <th>avg(09_2_VV)</th>\n",
              "      <th>avg(09_3_VH)</th>\n",
              "      <th>avg(09_3_VV)</th>\n",
              "      <th>avg(09_4_VH)</th>\n",
              "      <th>avg(09_4_VV)</th>\n",
              "      <th>avg(10_0_VH)</th>\n",
              "      <th>avg(10_0_VV)</th>\n",
              "      <th>avg(10_1_VH)</th>\n",
              "      <th>avg(10_1_VV)</th>\n",
              "      <th>avg(10_2_VH)</th>\n",
              "      <th>avg(10_2_VV)</th>\n",
              "      <th>avg(10_3_VH)</th>\n",
              "      <th>avg(10_3_VV)</th>\n",
              "      <th>avg(10_4_VH)</th>\n",
              "      <th>avg(10_4_VV)</th>\n",
              "      <th>avg(11_0_VH)</th>\n",
              "      <th>avg(11_0_VV)</th>\n",
              "      <th>avg(11_1_VH)</th>\n",
              "      <th>avg(11_1_VV)</th>\n",
              "      <th>avg(11_2_VH)</th>\n",
              "      <th>avg(11_2_VV)</th>\n",
              "      <th>avg(11_3_VH)</th>\n",
              "      <th>avg(11_3_VV)</th>\n",
              "      <th>avg(11_4_VH)</th>\n",
              "      <th>avg(11_4_VV)</th>\n",
              "      <th>avg(fields)</th>\n",
              "      <th>avg(y)</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>87958.0</td>\n",
              "      <td>2.676827</td>\n",
              "      <td>19.940902</td>\n",
              "      <td>2.954588</td>\n",
              "      <td>16.914774</td>\n",
              "      <td>1.517263</td>\n",
              "      <td>17.317885</td>\n",
              "      <td>2.625816</td>\n",
              "      <td>15.063453</td>\n",
              "      <td>2.490202</td>\n",
              "      <td>19.443235</td>\n",
              "      <td>2.078072</td>\n",
              "      <td>13.496112</td>\n",
              "      <td>3.026439</td>\n",
              "      <td>24.278383</td>\n",
              "      <td>3.165474</td>\n",
              "      <td>20.325661</td>\n",
              "      <td>4.285537</td>\n",
              "      <td>29.982271</td>\n",
              "      <td>3.807465</td>\n",
              "      <td>24.830171</td>\n",
              "      <td>3.390358</td>\n",
              "      <td>28.850078</td>\n",
              "      <td>2.199689</td>\n",
              "      <td>17.498600</td>\n",
              "      <td>4.494868</td>\n",
              "      <td>53.866874</td>\n",
              "      <td>3.990980</td>\n",
              "      <td>29.867496</td>\n",
              "      <td>3.549611</td>\n",
              "      <td>41.758009</td>\n",
              "      <td>3.655988</td>\n",
              "      <td>21.973250</td>\n",
              "      <td>2.746812</td>\n",
              "      <td>22.492379</td>\n",
              "      <td>3.851633</td>\n",
              "      <td>20.743390</td>\n",
              "      <td>4.149611</td>\n",
              "      <td>31.140280</td>\n",
              "      <td>4.115708</td>\n",
              "      <td>...</td>\n",
              "      <td>6.016485</td>\n",
              "      <td>23.416174</td>\n",
              "      <td>5.159876</td>\n",
              "      <td>17.355521</td>\n",
              "      <td>5.144323</td>\n",
              "      <td>19.175117</td>\n",
              "      <td>3.609331</td>\n",
              "      <td>9.614308</td>\n",
              "      <td>4.097667</td>\n",
              "      <td>11.322240</td>\n",
              "      <td>4.806843</td>\n",
              "      <td>11.568896</td>\n",
              "      <td>5.043857</td>\n",
              "      <td>14.158942</td>\n",
              "      <td>5.234526</td>\n",
              "      <td>13.975117</td>\n",
              "      <td>5.299222</td>\n",
              "      <td>16.238258</td>\n",
              "      <td>5.269362</td>\n",
              "      <td>14.159876</td>\n",
              "      <td>4.789114</td>\n",
              "      <td>17.764852</td>\n",
              "      <td>5.262830</td>\n",
              "      <td>19.453810</td>\n",
              "      <td>3.973872</td>\n",
              "      <td>16.525972</td>\n",
              "      <td>4.184137</td>\n",
              "      <td>13.902022</td>\n",
              "      <td>2.562364</td>\n",
              "      <td>15.148056</td>\n",
              "      <td>3.333126</td>\n",
              "      <td>14.901400</td>\n",
              "      <td>2.629238</td>\n",
              "      <td>17.477138</td>\n",
              "      <td>3.062519</td>\n",
              "      <td>14.792224</td>\n",
              "      <td>2.813997</td>\n",
              "      <td>18.101711</td>\n",
              "      <td>87958.0</td>\n",
              "      <td>6.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>26338.0</td>\n",
              "      <td>5.603060</td>\n",
              "      <td>26.801530</td>\n",
              "      <td>2.649399</td>\n",
              "      <td>27.394098</td>\n",
              "      <td>4.034973</td>\n",
              "      <td>20.696393</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>3.618798</td>\n",
              "      <td>21.302077</td>\n",
              "      <td>4.794973</td>\n",
              "      <td>25.068852</td>\n",
              "      <td>5.045683</td>\n",
              "      <td>27.548634</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>3.813552</td>\n",
              "      <td>42.742732</td>\n",
              "      <td>3.390164</td>\n",
              "      <td>31.265792</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>3.405246</td>\n",
              "      <td>17.995410</td>\n",
              "      <td>4.087869</td>\n",
              "      <td>25.903825</td>\n",
              "      <td>3.808743</td>\n",
              "      <td>16.022295</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>4.260765</td>\n",
              "      <td>15.931585</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>4.268415</td>\n",
              "      <td>11.620328</td>\n",
              "      <td>4.164372</td>\n",
              "      <td>11.830383</td>\n",
              "      <td>5.489836</td>\n",
              "      <td>15.177049</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>5.150601</td>\n",
              "      <td>15.444809</td>\n",
              "      <td>3.140765</td>\n",
              "      <td>15.039781</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2.434973</td>\n",
              "      <td>15.283279</td>\n",
              "      <td>2.280000</td>\n",
              "      <td>15.930710</td>\n",
              "      <td>3.110820</td>\n",
              "      <td>17.633443</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>26338.0</td>\n",
              "      <td>7.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>55436.0</td>\n",
              "      <td>1.374696</td>\n",
              "      <td>12.523684</td>\n",
              "      <td>1.226113</td>\n",
              "      <td>12.234818</td>\n",
              "      <td>1.432591</td>\n",
              "      <td>12.479352</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>4.423279</td>\n",
              "      <td>26.419231</td>\n",
              "      <td>5.006680</td>\n",
              "      <td>28.885628</td>\n",
              "      <td>4.181377</td>\n",
              "      <td>26.215992</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>4.687045</td>\n",
              "      <td>35.269636</td>\n",
              "      <td>4.108704</td>\n",
              "      <td>28.842713</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>4.374089</td>\n",
              "      <td>24.329960</td>\n",
              "      <td>8.350405</td>\n",
              "      <td>42.986437</td>\n",
              "      <td>9.071457</td>\n",
              "      <td>36.728340</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>8.344939</td>\n",
              "      <td>25.227530</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>9.095344</td>\n",
              "      <td>20.195142</td>\n",
              "      <td>14.563158</td>\n",
              "      <td>32.720040</td>\n",
              "      <td>17.946559</td>\n",
              "      <td>38.577733</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>12.281984</td>\n",
              "      <td>28.092105</td>\n",
              "      <td>7.105263</td>\n",
              "      <td>18.811741</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>3.455466</td>\n",
              "      <td>13.469231</td>\n",
              "      <td>2.971053</td>\n",
              "      <td>15.141296</td>\n",
              "      <td>2.600810</td>\n",
              "      <td>14.885628</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>55436.0</td>\n",
              "      <td>8.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>101335.0</td>\n",
              "      <td>1.514648</td>\n",
              "      <td>9.358670</td>\n",
              "      <td>1.258116</td>\n",
              "      <td>7.973872</td>\n",
              "      <td>1.628662</td>\n",
              "      <td>8.589865</td>\n",
              "      <td>1.377672</td>\n",
              "      <td>8.021378</td>\n",
              "      <td>1.794141</td>\n",
              "      <td>10.216944</td>\n",
              "      <td>7.686461</td>\n",
              "      <td>44.382423</td>\n",
              "      <td>1.743468</td>\n",
              "      <td>11.716548</td>\n",
              "      <td>1.385590</td>\n",
              "      <td>9.500396</td>\n",
              "      <td>1.771971</td>\n",
              "      <td>12.045131</td>\n",
              "      <td>4.846397</td>\n",
              "      <td>29.173397</td>\n",
              "      <td>1.908947</td>\n",
              "      <td>13.020586</td>\n",
              "      <td>1.387173</td>\n",
              "      <td>10.524149</td>\n",
              "      <td>3.228029</td>\n",
              "      <td>29.858274</td>\n",
              "      <td>3.025337</td>\n",
              "      <td>21.114014</td>\n",
              "      <td>3.159145</td>\n",
              "      <td>20.205067</td>\n",
              "      <td>3.876485</td>\n",
              "      <td>20.482185</td>\n",
              "      <td>3.369755</td>\n",
              "      <td>14.817102</td>\n",
              "      <td>4.915281</td>\n",
              "      <td>22.543151</td>\n",
              "      <td>5.932700</td>\n",
              "      <td>25.900238</td>\n",
              "      <td>5.742676</td>\n",
              "      <td>...</td>\n",
              "      <td>6.458432</td>\n",
              "      <td>21.057007</td>\n",
              "      <td>5.045922</td>\n",
              "      <td>17.707838</td>\n",
              "      <td>5.969121</td>\n",
              "      <td>18.872526</td>\n",
              "      <td>3.909739</td>\n",
              "      <td>13.253365</td>\n",
              "      <td>4.026128</td>\n",
              "      <td>10.783848</td>\n",
              "      <td>4.828979</td>\n",
              "      <td>15.520190</td>\n",
              "      <td>3.944576</td>\n",
              "      <td>9.995249</td>\n",
              "      <td>4.441805</td>\n",
              "      <td>14.840063</td>\n",
              "      <td>5.030087</td>\n",
              "      <td>13.472684</td>\n",
              "      <td>8.271576</td>\n",
              "      <td>21.724466</td>\n",
              "      <td>5.296912</td>\n",
              "      <td>16.022169</td>\n",
              "      <td>6.555819</td>\n",
              "      <td>22.524149</td>\n",
              "      <td>3.804434</td>\n",
              "      <td>13.372130</td>\n",
              "      <td>3.058591</td>\n",
              "      <td>11.595408</td>\n",
              "      <td>4.068884</td>\n",
              "      <td>23.426762</td>\n",
              "      <td>1.793349</td>\n",
              "      <td>9.307205</td>\n",
              "      <td>1.619161</td>\n",
              "      <td>9.403009</td>\n",
              "      <td>1.338876</td>\n",
              "      <td>8.879652</td>\n",
              "      <td>1.653998</td>\n",
              "      <td>10.286619</td>\n",
              "      <td>101335.0</td>\n",
              "      <td>7.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>13268.0</td>\n",
              "      <td>3.000000</td>\n",
              "      <td>19.500000</td>\n",
              "      <td>5.166667</td>\n",
              "      <td>12.833333</td>\n",
              "      <td>3.666667</td>\n",
              "      <td>20.666667</td>\n",
              "      <td>2.000000</td>\n",
              "      <td>10.833333</td>\n",
              "      <td>3.666667</td>\n",
              "      <td>18.666667</td>\n",
              "      <td>2.333333</td>\n",
              "      <td>14.333333</td>\n",
              "      <td>2.833333</td>\n",
              "      <td>20.666667</td>\n",
              "      <td>1.666667</td>\n",
              "      <td>13.500000</td>\n",
              "      <td>2.333333</td>\n",
              "      <td>25.166667</td>\n",
              "      <td>4.166667</td>\n",
              "      <td>17.500000</td>\n",
              "      <td>2.000000</td>\n",
              "      <td>18.166667</td>\n",
              "      <td>3.333333</td>\n",
              "      <td>20.500000</td>\n",
              "      <td>6.000000</td>\n",
              "      <td>58.000000</td>\n",
              "      <td>3.833333</td>\n",
              "      <td>33.333333</td>\n",
              "      <td>2.166667</td>\n",
              "      <td>35.833333</td>\n",
              "      <td>2.333333</td>\n",
              "      <td>21.166667</td>\n",
              "      <td>4.333333</td>\n",
              "      <td>29.000000</td>\n",
              "      <td>3.833333</td>\n",
              "      <td>28.833333</td>\n",
              "      <td>5.833333</td>\n",
              "      <td>17.833333</td>\n",
              "      <td>4.833333</td>\n",
              "      <td>...</td>\n",
              "      <td>3.333333</td>\n",
              "      <td>32.500000</td>\n",
              "      <td>7.500000</td>\n",
              "      <td>22.333333</td>\n",
              "      <td>4.500000</td>\n",
              "      <td>19.666667</td>\n",
              "      <td>4.333333</td>\n",
              "      <td>9.166667</td>\n",
              "      <td>2.166667</td>\n",
              "      <td>13.166667</td>\n",
              "      <td>8.500000</td>\n",
              "      <td>16.666667</td>\n",
              "      <td>3.833333</td>\n",
              "      <td>22.166667</td>\n",
              "      <td>2.833333</td>\n",
              "      <td>21.000000</td>\n",
              "      <td>1.333333</td>\n",
              "      <td>28.833333</td>\n",
              "      <td>2.500000</td>\n",
              "      <td>27.166667</td>\n",
              "      <td>2.500000</td>\n",
              "      <td>27.166667</td>\n",
              "      <td>3.333333</td>\n",
              "      <td>28.666667</td>\n",
              "      <td>2.166667</td>\n",
              "      <td>20.333333</td>\n",
              "      <td>2.166667</td>\n",
              "      <td>11.000000</td>\n",
              "      <td>2.333333</td>\n",
              "      <td>27.333333</td>\n",
              "      <td>3.000000</td>\n",
              "      <td>15.500000</td>\n",
              "      <td>5.333333</td>\n",
              "      <td>24.000000</td>\n",
              "      <td>9.166667</td>\n",
              "      <td>42.333333</td>\n",
              "      <td>5.000000</td>\n",
              "      <td>23.166667</td>\n",
              "      <td>13268.0</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 85 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "     fields  avg(04_0_VH)  avg(04_0_VV)  ...  avg(11_4_VV)  avg(fields)  avg(y)\n",
              "0   87958.0      2.676827     19.940902  ...     18.101711      87958.0     6.0\n",
              "1   26338.0      5.603060     26.801530  ...           NaN      26338.0     7.0\n",
              "2   55436.0      1.374696     12.523684  ...           NaN      55436.0     8.0\n",
              "3  101335.0      1.514648      9.358670  ...     10.286619     101335.0     7.0\n",
              "4   13268.0      3.000000     19.500000  ...     23.166667      13268.0     2.0\n",
              "\n",
              "[5 rows x 85 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 252
        },
        "id": "cTIe-_-jfNK_",
        "outputId": "75d3f1b7-42f7-4828-dc25-52fbb1aa97a3"
      },
      "source": [
        "sar_train = final_data_mean.iloc[:, 1:]\n",
        "sar_train.columns = final_data.columns\n",
        "sar_train.y = round(sar_train.y).astype(int)\n",
        "sar_train = sar_train[sar_train.y != 0].reset_index(drop = True)\n",
        "sar_train.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>04_0_VH</th>\n",
              "      <th>04_0_VV</th>\n",
              "      <th>04_1_VH</th>\n",
              "      <th>04_1_VV</th>\n",
              "      <th>04_2_VH</th>\n",
              "      <th>04_2_VV</th>\n",
              "      <th>04_3_VH</th>\n",
              "      <th>04_3_VV</th>\n",
              "      <th>04_4_VH</th>\n",
              "      <th>04_4_VV</th>\n",
              "      <th>04_5_VH</th>\n",
              "      <th>04_5_VV</th>\n",
              "      <th>05_0_VH</th>\n",
              "      <th>05_0_VV</th>\n",
              "      <th>05_1_VH</th>\n",
              "      <th>05_1_VV</th>\n",
              "      <th>05_2_VH</th>\n",
              "      <th>05_2_VV</th>\n",
              "      <th>05_3_VH</th>\n",
              "      <th>05_3_VV</th>\n",
              "      <th>05_4_VH</th>\n",
              "      <th>05_4_VV</th>\n",
              "      <th>06_0_VH</th>\n",
              "      <th>06_0_VV</th>\n",
              "      <th>06_1_VH</th>\n",
              "      <th>06_1_VV</th>\n",
              "      <th>06_2_VH</th>\n",
              "      <th>06_2_VV</th>\n",
              "      <th>06_3_VH</th>\n",
              "      <th>06_3_VV</th>\n",
              "      <th>06_4_VH</th>\n",
              "      <th>06_4_VV</th>\n",
              "      <th>07_0_VH</th>\n",
              "      <th>07_0_VV</th>\n",
              "      <th>07_1_VH</th>\n",
              "      <th>07_1_VV</th>\n",
              "      <th>07_2_VH</th>\n",
              "      <th>07_2_VV</th>\n",
              "      <th>07_3_VH</th>\n",
              "      <th>07_3_VV</th>\n",
              "      <th>...</th>\n",
              "      <th>08_1_VH</th>\n",
              "      <th>08_1_VV</th>\n",
              "      <th>08_2_VH</th>\n",
              "      <th>08_2_VV</th>\n",
              "      <th>08_3_VH</th>\n",
              "      <th>08_3_VV</th>\n",
              "      <th>08_4_VH</th>\n",
              "      <th>08_4_VV</th>\n",
              "      <th>09_0_VH</th>\n",
              "      <th>09_0_VV</th>\n",
              "      <th>09_1_VH</th>\n",
              "      <th>09_1_VV</th>\n",
              "      <th>09_2_VH</th>\n",
              "      <th>09_2_VV</th>\n",
              "      <th>09_3_VH</th>\n",
              "      <th>09_3_VV</th>\n",
              "      <th>09_4_VH</th>\n",
              "      <th>09_4_VV</th>\n",
              "      <th>10_0_VH</th>\n",
              "      <th>10_0_VV</th>\n",
              "      <th>10_1_VH</th>\n",
              "      <th>10_1_VV</th>\n",
              "      <th>10_2_VH</th>\n",
              "      <th>10_2_VV</th>\n",
              "      <th>10_3_VH</th>\n",
              "      <th>10_3_VV</th>\n",
              "      <th>10_4_VH</th>\n",
              "      <th>10_4_VV</th>\n",
              "      <th>11_0_VH</th>\n",
              "      <th>11_0_VV</th>\n",
              "      <th>11_1_VH</th>\n",
              "      <th>11_1_VV</th>\n",
              "      <th>11_2_VH</th>\n",
              "      <th>11_2_VV</th>\n",
              "      <th>11_3_VH</th>\n",
              "      <th>11_3_VV</th>\n",
              "      <th>11_4_VH</th>\n",
              "      <th>11_4_VV</th>\n",
              "      <th>fields</th>\n",
              "      <th>y</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2.676827</td>\n",
              "      <td>19.940902</td>\n",
              "      <td>2.954588</td>\n",
              "      <td>16.914774</td>\n",
              "      <td>1.517263</td>\n",
              "      <td>17.317885</td>\n",
              "      <td>2.625816</td>\n",
              "      <td>15.063453</td>\n",
              "      <td>2.490202</td>\n",
              "      <td>19.443235</td>\n",
              "      <td>2.078072</td>\n",
              "      <td>13.496112</td>\n",
              "      <td>3.026439</td>\n",
              "      <td>24.278383</td>\n",
              "      <td>3.165474</td>\n",
              "      <td>20.325661</td>\n",
              "      <td>4.285537</td>\n",
              "      <td>29.982271</td>\n",
              "      <td>3.807465</td>\n",
              "      <td>24.830171</td>\n",
              "      <td>3.390358</td>\n",
              "      <td>28.850078</td>\n",
              "      <td>2.199689</td>\n",
              "      <td>17.498600</td>\n",
              "      <td>4.494868</td>\n",
              "      <td>53.866874</td>\n",
              "      <td>3.990980</td>\n",
              "      <td>29.867496</td>\n",
              "      <td>3.549611</td>\n",
              "      <td>41.758009</td>\n",
              "      <td>3.655988</td>\n",
              "      <td>21.973250</td>\n",
              "      <td>2.746812</td>\n",
              "      <td>22.492379</td>\n",
              "      <td>3.851633</td>\n",
              "      <td>20.743390</td>\n",
              "      <td>4.149611</td>\n",
              "      <td>31.140280</td>\n",
              "      <td>4.115708</td>\n",
              "      <td>15.497356</td>\n",
              "      <td>...</td>\n",
              "      <td>6.016485</td>\n",
              "      <td>23.416174</td>\n",
              "      <td>5.159876</td>\n",
              "      <td>17.355521</td>\n",
              "      <td>5.144323</td>\n",
              "      <td>19.175117</td>\n",
              "      <td>3.609331</td>\n",
              "      <td>9.614308</td>\n",
              "      <td>4.097667</td>\n",
              "      <td>11.322240</td>\n",
              "      <td>4.806843</td>\n",
              "      <td>11.568896</td>\n",
              "      <td>5.043857</td>\n",
              "      <td>14.158942</td>\n",
              "      <td>5.234526</td>\n",
              "      <td>13.975117</td>\n",
              "      <td>5.299222</td>\n",
              "      <td>16.238258</td>\n",
              "      <td>5.269362</td>\n",
              "      <td>14.159876</td>\n",
              "      <td>4.789114</td>\n",
              "      <td>17.764852</td>\n",
              "      <td>5.262830</td>\n",
              "      <td>19.453810</td>\n",
              "      <td>3.973872</td>\n",
              "      <td>16.525972</td>\n",
              "      <td>4.184137</td>\n",
              "      <td>13.902022</td>\n",
              "      <td>2.562364</td>\n",
              "      <td>15.148056</td>\n",
              "      <td>3.333126</td>\n",
              "      <td>14.901400</td>\n",
              "      <td>2.629238</td>\n",
              "      <td>17.477138</td>\n",
              "      <td>3.062519</td>\n",
              "      <td>14.792224</td>\n",
              "      <td>2.813997</td>\n",
              "      <td>18.101711</td>\n",
              "      <td>87958.0</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>5.603060</td>\n",
              "      <td>26.801530</td>\n",
              "      <td>2.649399</td>\n",
              "      <td>27.394098</td>\n",
              "      <td>4.034973</td>\n",
              "      <td>20.696393</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>3.618798</td>\n",
              "      <td>21.302077</td>\n",
              "      <td>4.794973</td>\n",
              "      <td>25.068852</td>\n",
              "      <td>5.045683</td>\n",
              "      <td>27.548634</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>3.813552</td>\n",
              "      <td>42.742732</td>\n",
              "      <td>3.390164</td>\n",
              "      <td>31.265792</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>3.405246</td>\n",
              "      <td>17.995410</td>\n",
              "      <td>4.087869</td>\n",
              "      <td>25.903825</td>\n",
              "      <td>3.808743</td>\n",
              "      <td>16.022295</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>4.260765</td>\n",
              "      <td>15.931585</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>4.268415</td>\n",
              "      <td>11.620328</td>\n",
              "      <td>4.164372</td>\n",
              "      <td>11.830383</td>\n",
              "      <td>5.489836</td>\n",
              "      <td>15.177049</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>5.150601</td>\n",
              "      <td>15.444809</td>\n",
              "      <td>3.140765</td>\n",
              "      <td>15.039781</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2.434973</td>\n",
              "      <td>15.283279</td>\n",
              "      <td>2.280000</td>\n",
              "      <td>15.930710</td>\n",
              "      <td>3.110820</td>\n",
              "      <td>17.633443</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>26338.0</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.374696</td>\n",
              "      <td>12.523684</td>\n",
              "      <td>1.226113</td>\n",
              "      <td>12.234818</td>\n",
              "      <td>1.432591</td>\n",
              "      <td>12.479352</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>4.423279</td>\n",
              "      <td>26.419231</td>\n",
              "      <td>5.006680</td>\n",
              "      <td>28.885628</td>\n",
              "      <td>4.181377</td>\n",
              "      <td>26.215992</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>4.687045</td>\n",
              "      <td>35.269636</td>\n",
              "      <td>4.108704</td>\n",
              "      <td>28.842713</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>4.374089</td>\n",
              "      <td>24.329960</td>\n",
              "      <td>8.350405</td>\n",
              "      <td>42.986437</td>\n",
              "      <td>9.071457</td>\n",
              "      <td>36.728340</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>8.344939</td>\n",
              "      <td>25.227530</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>9.095344</td>\n",
              "      <td>20.195142</td>\n",
              "      <td>14.563158</td>\n",
              "      <td>32.720040</td>\n",
              "      <td>17.946559</td>\n",
              "      <td>38.577733</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>12.281984</td>\n",
              "      <td>28.092105</td>\n",
              "      <td>7.105263</td>\n",
              "      <td>18.811741</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>3.455466</td>\n",
              "      <td>13.469231</td>\n",
              "      <td>2.971053</td>\n",
              "      <td>15.141296</td>\n",
              "      <td>2.600810</td>\n",
              "      <td>14.885628</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>55436.0</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1.514648</td>\n",
              "      <td>9.358670</td>\n",
              "      <td>1.258116</td>\n",
              "      <td>7.973872</td>\n",
              "      <td>1.628662</td>\n",
              "      <td>8.589865</td>\n",
              "      <td>1.377672</td>\n",
              "      <td>8.021378</td>\n",
              "      <td>1.794141</td>\n",
              "      <td>10.216944</td>\n",
              "      <td>7.686461</td>\n",
              "      <td>44.382423</td>\n",
              "      <td>1.743468</td>\n",
              "      <td>11.716548</td>\n",
              "      <td>1.385590</td>\n",
              "      <td>9.500396</td>\n",
              "      <td>1.771971</td>\n",
              "      <td>12.045131</td>\n",
              "      <td>4.846397</td>\n",
              "      <td>29.173397</td>\n",
              "      <td>1.908947</td>\n",
              "      <td>13.020586</td>\n",
              "      <td>1.387173</td>\n",
              "      <td>10.524149</td>\n",
              "      <td>3.228029</td>\n",
              "      <td>29.858274</td>\n",
              "      <td>3.025337</td>\n",
              "      <td>21.114014</td>\n",
              "      <td>3.159145</td>\n",
              "      <td>20.205067</td>\n",
              "      <td>3.876485</td>\n",
              "      <td>20.482185</td>\n",
              "      <td>3.369755</td>\n",
              "      <td>14.817102</td>\n",
              "      <td>4.915281</td>\n",
              "      <td>22.543151</td>\n",
              "      <td>5.932700</td>\n",
              "      <td>25.900238</td>\n",
              "      <td>5.742676</td>\n",
              "      <td>20.196358</td>\n",
              "      <td>...</td>\n",
              "      <td>6.458432</td>\n",
              "      <td>21.057007</td>\n",
              "      <td>5.045922</td>\n",
              "      <td>17.707838</td>\n",
              "      <td>5.969121</td>\n",
              "      <td>18.872526</td>\n",
              "      <td>3.909739</td>\n",
              "      <td>13.253365</td>\n",
              "      <td>4.026128</td>\n",
              "      <td>10.783848</td>\n",
              "      <td>4.828979</td>\n",
              "      <td>15.520190</td>\n",
              "      <td>3.944576</td>\n",
              "      <td>9.995249</td>\n",
              "      <td>4.441805</td>\n",
              "      <td>14.840063</td>\n",
              "      <td>5.030087</td>\n",
              "      <td>13.472684</td>\n",
              "      <td>8.271576</td>\n",
              "      <td>21.724466</td>\n",
              "      <td>5.296912</td>\n",
              "      <td>16.022169</td>\n",
              "      <td>6.555819</td>\n",
              "      <td>22.524149</td>\n",
              "      <td>3.804434</td>\n",
              "      <td>13.372130</td>\n",
              "      <td>3.058591</td>\n",
              "      <td>11.595408</td>\n",
              "      <td>4.068884</td>\n",
              "      <td>23.426762</td>\n",
              "      <td>1.793349</td>\n",
              "      <td>9.307205</td>\n",
              "      <td>1.619161</td>\n",
              "      <td>9.403009</td>\n",
              "      <td>1.338876</td>\n",
              "      <td>8.879652</td>\n",
              "      <td>1.653998</td>\n",
              "      <td>10.286619</td>\n",
              "      <td>101335.0</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3.000000</td>\n",
              "      <td>19.500000</td>\n",
              "      <td>5.166667</td>\n",
              "      <td>12.833333</td>\n",
              "      <td>3.666667</td>\n",
              "      <td>20.666667</td>\n",
              "      <td>2.000000</td>\n",
              "      <td>10.833333</td>\n",
              "      <td>3.666667</td>\n",
              "      <td>18.666667</td>\n",
              "      <td>2.333333</td>\n",
              "      <td>14.333333</td>\n",
              "      <td>2.833333</td>\n",
              "      <td>20.666667</td>\n",
              "      <td>1.666667</td>\n",
              "      <td>13.500000</td>\n",
              "      <td>2.333333</td>\n",
              "      <td>25.166667</td>\n",
              "      <td>4.166667</td>\n",
              "      <td>17.500000</td>\n",
              "      <td>2.000000</td>\n",
              "      <td>18.166667</td>\n",
              "      <td>3.333333</td>\n",
              "      <td>20.500000</td>\n",
              "      <td>6.000000</td>\n",
              "      <td>58.000000</td>\n",
              "      <td>3.833333</td>\n",
              "      <td>33.333333</td>\n",
              "      <td>2.166667</td>\n",
              "      <td>35.833333</td>\n",
              "      <td>2.333333</td>\n",
              "      <td>21.166667</td>\n",
              "      <td>4.333333</td>\n",
              "      <td>29.000000</td>\n",
              "      <td>3.833333</td>\n",
              "      <td>28.833333</td>\n",
              "      <td>5.833333</td>\n",
              "      <td>17.833333</td>\n",
              "      <td>4.833333</td>\n",
              "      <td>20.333333</td>\n",
              "      <td>...</td>\n",
              "      <td>3.333333</td>\n",
              "      <td>32.500000</td>\n",
              "      <td>7.500000</td>\n",
              "      <td>22.333333</td>\n",
              "      <td>4.500000</td>\n",
              "      <td>19.666667</td>\n",
              "      <td>4.333333</td>\n",
              "      <td>9.166667</td>\n",
              "      <td>2.166667</td>\n",
              "      <td>13.166667</td>\n",
              "      <td>8.500000</td>\n",
              "      <td>16.666667</td>\n",
              "      <td>3.833333</td>\n",
              "      <td>22.166667</td>\n",
              "      <td>2.833333</td>\n",
              "      <td>21.000000</td>\n",
              "      <td>1.333333</td>\n",
              "      <td>28.833333</td>\n",
              "      <td>2.500000</td>\n",
              "      <td>27.166667</td>\n",
              "      <td>2.500000</td>\n",
              "      <td>27.166667</td>\n",
              "      <td>3.333333</td>\n",
              "      <td>28.666667</td>\n",
              "      <td>2.166667</td>\n",
              "      <td>20.333333</td>\n",
              "      <td>2.166667</td>\n",
              "      <td>11.000000</td>\n",
              "      <td>2.333333</td>\n",
              "      <td>27.333333</td>\n",
              "      <td>3.000000</td>\n",
              "      <td>15.500000</td>\n",
              "      <td>5.333333</td>\n",
              "      <td>24.000000</td>\n",
              "      <td>9.166667</td>\n",
              "      <td>42.333333</td>\n",
              "      <td>5.000000</td>\n",
              "      <td>23.166667</td>\n",
              "      <td>13268.0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 84 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "    04_0_VH    04_0_VV   04_1_VH    04_1_VV  ...   11_4_VH    11_4_VV    fields  y\n",
              "0  2.676827  19.940902  2.954588  16.914774  ...  2.813997  18.101711   87958.0  6\n",
              "1  5.603060  26.801530  2.649399  27.394098  ...       NaN        NaN   26338.0  7\n",
              "2  1.374696  12.523684  1.226113  12.234818  ...       NaN        NaN   55436.0  8\n",
              "3  1.514648   9.358670  1.258116   7.973872  ...  1.653998  10.286619  101335.0  7\n",
              "4  3.000000  19.500000  5.166667  12.833333  ...  5.000000  23.166667   13268.0  2\n",
              "\n",
              "[5 rows x 84 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ox20DSAEfyyG",
        "outputId": "64bdf921-d532-42f8-c280-3df1b8061138"
      },
      "source": [
        "sar_train.y.unique()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([6, 7, 8, 2, 5, 4, 1, 3, 9])"
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N5PwFjR0hKZY"
      },
      "source": [
        "sar_train.to_csv('/content/drive/MyDrive/CompeData/Radiant/s1_all_data/sar_train.csv', index = False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 82,
          "referenced_widgets": [
            "c05b441e442f4c2396c9ba920423e05a",
            "80ac4e5eaa574d1d9796dafb61c968b1",
            "dbdc3f534915460d8c094fe3fbf6d3e5",
            "a930671d0de84ca093dcb8b4e3cb1d0d",
            "62b1340ef5b146a69f5426ddd31a132e",
            "b760c18a75cb4a3785b0aa429ac21660",
            "c02d098387c44ac899bcc1a5ef97aeaa",
            "81305ee731aa4b958c71a180aedcc7a2",
            "f37f5d60c87b4652b46b6175882c9caa",
            "4843dbc1f2dd4701a136f87dfa1b940e",
            "e7f81c8eee9e460faa0c2a8eba58f3c9"
          ]
        },
        "id": "lfwLN-mbhYhb",
        "outputId": "2fa25d79-cfe8-4a5f-af3d-35ff314c90fb"
      },
      "source": [
        "%%time\n",
        "dfs = []\n",
        "for i in tqdm_notebook(glob.glob('/content/drive/MyDrive/CompeData/Radiant/s1_all_data/s1_test*.csv')):\n",
        "  dfs.append(spark.read.csv(path = i, sep =',', encoding = 'UTF-8', comment = None, header = True))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c05b441e442f4c2396c9ba920423e05a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/5 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 89.9 ms, sys: 3.57 ms, total: 93.4 ms\n",
            "Wall time: 1.99 s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 82,
          "referenced_widgets": [
            "dc7d7680505d44999e165717debcd1a4",
            "63cf9d3439294863a5728040904c8ad7",
            "2bfaac11391a4adfb91433e0f3368683",
            "788a1eafa3814b9399816accd124f6eb",
            "d984a74eb7ad432a82fde659554da5b9",
            "50696b849220459495b9b9a091ce05bb",
            "72efa9c92d01410dbca8b357bc581e56",
            "5be0c6d165d748fcbb017817fd29c708",
            "f3591ac91f184068827b659c302bbe99",
            "9bca2e1c229b41d7add2792d72015b3a",
            "03f86a1edb104dd2b55a587599a8339e"
          ]
        },
        "id": "-YNI_CPkhgHX",
        "outputId": "f2f498b1-8514-4e31-c151-ffe28bc77a7a"
      },
      "source": [
        "%%time\n",
        "cols = set()\n",
        "for j in tqdm_notebook(dfs):\n",
        "    for x in j.columns:\n",
        "        cols.add(x)\n",
        "cols = sorted(cols)\n",
        "\n",
        "# Create a dictionary with all the dataframes\n",
        "all_data = {}\n",
        "for i, d in enumerate(dfs):\n",
        "    new_name = 'new_df' + str(i)  # New name for the key, the dataframe is the value\n",
        "    all_data[new_name] = d\n",
        "    # Loop through all column names. Add the missing columns to the dataframe (with value 0)\n",
        "    for x in cols:\n",
        "        if x not in d.columns:\n",
        "            all_data[new_name] = all_data[new_name].withColumn(x, F.lit(0))\n",
        "    all_data[new_name] = all_data[new_name].select(cols)  # Use 'select' to get the columns sorted"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "dc7d7680505d44999e165717debcd1a4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/5 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 199 ms, sys: 28.2 ms, total: 227 ms\n",
            "Wall time: 1.02 s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "83S0Xj9whgDh",
        "outputId": "b28edf77-03f0-453d-b8e8-6c8ac87316f4"
      },
      "source": [
        "len(dfs)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "5"
            ]
          },
          "metadata": {},
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7ggUilAahf-4",
        "outputId": "56a82920-dc5f-4e4c-bed5-7929f73e4454"
      },
      "source": [
        "%%time\n",
        "all_data_dfs = [all_data['new_df'+str(i)] for i in range(5)]\n",
        "final_data = reduce(DataFrame.unionAll, all_data_dfs)\n",
        "final_data.show(5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+------+\n",
            "|04_0_VH|04_0_VV|04_1_VH|04_1_VV|04_2_VH|04_2_VV|04_3_VH|04_3_VV|04_4_VH|04_4_VV|04_5_VH|04_5_VV|05_0_VH|05_0_VV|05_1_VH|05_1_VV|05_2_VH|05_2_VV|05_3_VH|05_3_VV|05_4_VH|05_4_VV|06_0_VH|06_0_VV|06_1_VH|06_1_VV|06_2_VH|06_2_VV|06_3_VH|06_3_VV|06_4_VH|06_4_VV|07_0_VH|07_0_VV|07_1_VH|07_1_VV|07_2_VH|07_2_VV|07_3_VH|07_3_VV|07_4_VH|07_4_VV|08_0_VH|08_0_VV|08_1_VH|08_1_VV|08_2_VH|08_2_VV|08_3_VH|08_3_VV|08_4_VH|08_4_VV|09_0_VH|09_0_VV|09_1_VH|09_1_VV|09_2_VH|09_2_VV|09_3_VH|09_3_VV|09_4_VH|09_4_VV|10_0_VH|10_0_VV|10_1_VH|10_1_VV|10_2_VH|10_2_VV|10_3_VH|10_3_VV|10_4_VH|10_4_VV|11_0_VH|11_0_VV|11_1_VH|11_1_VV|11_2_VH|11_2_VV|11_3_VH|11_3_VV|11_4_VH|11_4_VV|fields|\n",
            "+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+------+\n",
            "|    6.0|   14.0|    5.0|   35.0|    5.0|   15.0|    7.0|   16.0|    9.0|   23.0|    7.0|   14.0|    7.0|   31.0|   10.0|   32.0|    2.0|   26.0|    6.0|   28.0|    6.0|   37.0|    6.0|   28.0|   14.0|   21.0|   13.0|   20.0|    6.0|   16.0|    4.0|   12.0|    3.0|   37.0|    5.0|   14.0|    1.0|   27.0|    4.0|   17.0|    4.0|   23.0|    4.0|   15.0|   14.0|   21.0|   11.0|   21.0|    5.0|   29.0|    5.0|   10.0|    1.0|   16.0|    2.0|   19.0|    1.0|   27.0|    9.0|   26.0|    3.0|   27.0|    9.0|   27.0|   11.0|   33.0|   12.0|   36.0|    5.0|    9.0|   11.0|   20.0|   15.0|   21.0|    5.0|   23.0|    4.0|    9.0|    7.0|   28.0|    3.0|   20.0|     0|\n",
            "|    4.0|   19.0|    8.0|   30.0|    3.0|   14.0|    4.0|   12.0|    6.0|   28.0|   10.0|   39.0|   10.0|   35.0|    9.0|   13.0|    1.0|   29.0|    9.0|   19.0|    5.0|   45.0|    3.0|   33.0|   11.0|   42.0|    8.0|   27.0|   10.0|   31.0|    3.0|   11.0|    3.0|   48.0|    7.0|   17.0|    2.0|   24.0|    3.0|   11.0|    8.0|   34.0|    4.0|   16.0|    3.0|   23.0|   13.0|   43.0|   10.0|   36.0|    6.0|   10.0|    9.0|   12.0|    6.0|   24.0|    1.0|   34.0|    4.0|   13.0|    3.0|   29.0|    3.0|   29.0|    8.0|   23.0|   10.0|   15.0|    6.0|    4.0|    9.0|   17.0|    4.0|   27.0|    5.0|   12.0|    3.0|   12.0|    8.0|   16.0|    7.0|    8.0|     0|\n",
            "|    1.0|   12.0|   10.0|   10.0|    6.0|   19.0|    2.0|    8.0|    5.0|   30.0|   10.0|   22.0|    4.0|   18.0|    5.0|    7.0|    1.0|   25.0|    8.0|    7.0|    3.0|   28.0|    4.0|   42.0|    6.0|   36.0|    8.0|   27.0|    7.0|   27.0|    2.0|   21.0|    5.0|   34.0|    4.0|   14.0|    3.0|   15.0|    3.0|    9.0|    9.0|   22.0|    2.0|   23.0|    3.0|   23.0|    9.0|   37.0|    6.0|   33.0|    4.0|   12.0|   12.0|    8.0|    3.0|   20.0|    4.0|   14.0|    4.0|   10.0|    4.0|   16.0|   10.0|   29.0|    5.0|   28.0|    8.0|   14.0|    9.0|   10.0|    8.0|   16.0|    5.0|   20.0|    6.0|   10.0|    1.0|    7.0|    7.0|   17.0|   10.0|   16.0|     0|\n",
            "|    5.0|    6.0|    4.0|   13.0|   10.0|   15.0|    4.0|   12.0|    7.0|   21.0|   17.0|   22.0|    5.0|    6.0|    6.0|   12.0|    2.0|   14.0|    9.0|   16.0|    5.0|    9.0|    6.0|   30.0|    3.0|   19.0|    5.0|   17.0|    7.0|   18.0|    8.0|   29.0|    4.0|   27.0|    4.0|   23.0|    6.0|   16.0|    7.0|    5.0|    7.0|   10.0|    3.0|    8.0|    3.0|   19.0|    5.0|   30.0|    9.0|   19.0|    2.0|   12.0|    8.0|   17.0|    1.0|   18.0|    9.0|    9.0|    4.0|   10.0|    5.0|   10.0|   15.0|   24.0|    5.0|   19.0|    3.0|   15.0|    9.0|   20.0|    6.0|    9.0|    5.0|   26.0|    6.0|    6.0|    6.0|    6.0|    8.0|   15.0|    8.0|   10.0|     0|\n",
            "|   10.0|    9.0|    5.0|   21.0|   10.0|   18.0|    7.0|   13.0|    4.0|   18.0|    9.0|   24.0|    6.0|   10.0|   10.0|   15.0|    6.0|   10.0|    9.0|   16.0|   11.0|   18.0|   10.0|   21.0|    2.0|   17.0|    8.0|   15.0|    6.0|   23.0|   11.0|   19.0|    7.0|   29.0|    8.0|   17.0|    8.0|   18.0|    9.0|    6.0|    8.0|    9.0|    5.0|   11.0|    4.0|   27.0|    6.0|   25.0|    7.0|   15.0|    8.0|   16.0|    4.0|   23.0|    2.0|   13.0|   12.0|    5.0|    3.0|    6.0|    2.0|    6.0|   19.0|   14.0|    4.0|    9.0|    2.0|   15.0|    8.0|   17.0|    5.0|   12.0|    3.0|   35.0|    5.0|   31.0|    8.0|   17.0|   16.0|   16.0|    9.0|   13.0|     0|\n",
            "+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+-------+------+\n",
            "only showing top 5 rows\n",
            "\n",
            "CPU times: user 5.53 ms, sys: 2.15 ms, total: 7.68 ms\n",
            "Wall time: 781 ms\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D6fzj2Rshf6s"
      },
      "source": [
        "final_data = final_data.select(*(F.col(c).cast(\"float\").alias(c) for c in final_data.columns))\n",
        "final_data.printSchema()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4Z_rdapMhqdo",
        "outputId": "cc794914-81ff-484e-fefc-7559d23e07f2"
      },
      "source": [
        "%%time\n",
        "final_data_mean = final_data.groupBy('fields').mean().toPandas()\n",
        "final_data_mean.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 10.1 s, sys: 1.04 s, total: 11.1 s\n",
            "Wall time: 25min 29s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 252
        },
        "id": "VSQCX0ZshqaK",
        "outputId": "1f78a9bc-c87b-40a8-d39b-ffeecd5e1ece"
      },
      "source": [
        "sar_test = final_data_mean.iloc[:, 1:]\n",
        "sar_test.columns = final_data.columns\n",
        "sar_test.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>04_0_VH</th>\n",
              "      <th>04_0_VV</th>\n",
              "      <th>04_1_VH</th>\n",
              "      <th>04_1_VV</th>\n",
              "      <th>04_2_VH</th>\n",
              "      <th>04_2_VV</th>\n",
              "      <th>04_3_VH</th>\n",
              "      <th>04_3_VV</th>\n",
              "      <th>04_4_VH</th>\n",
              "      <th>04_4_VV</th>\n",
              "      <th>04_5_VH</th>\n",
              "      <th>04_5_VV</th>\n",
              "      <th>05_0_VH</th>\n",
              "      <th>05_0_VV</th>\n",
              "      <th>05_1_VH</th>\n",
              "      <th>05_1_VV</th>\n",
              "      <th>05_2_VH</th>\n",
              "      <th>05_2_VV</th>\n",
              "      <th>05_3_VH</th>\n",
              "      <th>05_3_VV</th>\n",
              "      <th>05_4_VH</th>\n",
              "      <th>05_4_VV</th>\n",
              "      <th>06_0_VH</th>\n",
              "      <th>06_0_VV</th>\n",
              "      <th>06_1_VH</th>\n",
              "      <th>06_1_VV</th>\n",
              "      <th>06_2_VH</th>\n",
              "      <th>06_2_VV</th>\n",
              "      <th>06_3_VH</th>\n",
              "      <th>06_3_VV</th>\n",
              "      <th>06_4_VH</th>\n",
              "      <th>06_4_VV</th>\n",
              "      <th>07_0_VH</th>\n",
              "      <th>07_0_VV</th>\n",
              "      <th>07_1_VH</th>\n",
              "      <th>07_1_VV</th>\n",
              "      <th>07_2_VH</th>\n",
              "      <th>07_2_VV</th>\n",
              "      <th>07_3_VH</th>\n",
              "      <th>07_3_VV</th>\n",
              "      <th>...</th>\n",
              "      <th>08_0_VV</th>\n",
              "      <th>08_1_VH</th>\n",
              "      <th>08_1_VV</th>\n",
              "      <th>08_2_VH</th>\n",
              "      <th>08_2_VV</th>\n",
              "      <th>08_3_VH</th>\n",
              "      <th>08_3_VV</th>\n",
              "      <th>08_4_VH</th>\n",
              "      <th>08_4_VV</th>\n",
              "      <th>09_0_VH</th>\n",
              "      <th>09_0_VV</th>\n",
              "      <th>09_1_VH</th>\n",
              "      <th>09_1_VV</th>\n",
              "      <th>09_2_VH</th>\n",
              "      <th>09_2_VV</th>\n",
              "      <th>09_3_VH</th>\n",
              "      <th>09_3_VV</th>\n",
              "      <th>09_4_VH</th>\n",
              "      <th>09_4_VV</th>\n",
              "      <th>10_0_VH</th>\n",
              "      <th>10_0_VV</th>\n",
              "      <th>10_1_VH</th>\n",
              "      <th>10_1_VV</th>\n",
              "      <th>10_2_VH</th>\n",
              "      <th>10_2_VV</th>\n",
              "      <th>10_3_VH</th>\n",
              "      <th>10_3_VV</th>\n",
              "      <th>10_4_VH</th>\n",
              "      <th>10_4_VV</th>\n",
              "      <th>11_0_VH</th>\n",
              "      <th>11_0_VV</th>\n",
              "      <th>11_1_VH</th>\n",
              "      <th>11_1_VV</th>\n",
              "      <th>11_2_VH</th>\n",
              "      <th>11_2_VV</th>\n",
              "      <th>11_3_VH</th>\n",
              "      <th>11_3_VV</th>\n",
              "      <th>11_4_VH</th>\n",
              "      <th>11_4_VV</th>\n",
              "      <th>fields</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.816514</td>\n",
              "      <td>5.449541</td>\n",
              "      <td>0.651376</td>\n",
              "      <td>4.724771</td>\n",
              "      <td>0.917431</td>\n",
              "      <td>4.431193</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.715596</td>\n",
              "      <td>3.256881</td>\n",
              "      <td>0.761468</td>\n",
              "      <td>4.458716</td>\n",
              "      <td>0.853211</td>\n",
              "      <td>4.458716</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.825688</td>\n",
              "      <td>14.165138</td>\n",
              "      <td>0.954128</td>\n",
              "      <td>16.211009</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1.284404</td>\n",
              "      <td>5.422018</td>\n",
              "      <td>0.880734</td>\n",
              "      <td>11.623853</td>\n",
              "      <td>0.853211</td>\n",
              "      <td>6.009174</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>23.899083</td>\n",
              "      <td>1.807339</td>\n",
              "      <td>16.972477</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1.724771</td>\n",
              "      <td>9.348624</td>\n",
              "      <td>2.321101</td>\n",
              "      <td>7.706422</td>\n",
              "      <td>1.834862</td>\n",
              "      <td>8.174312</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1.522936</td>\n",
              "      <td>6.715596</td>\n",
              "      <td>1.899083</td>\n",
              "      <td>10.045872</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1.862385</td>\n",
              "      <td>7.807339</td>\n",
              "      <td>1.972477</td>\n",
              "      <td>8.275229</td>\n",
              "      <td>1.110092</td>\n",
              "      <td>7.440367</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>714.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.954545</td>\n",
              "      <td>22.136364</td>\n",
              "      <td>1.636364</td>\n",
              "      <td>11.272727</td>\n",
              "      <td>2.863636</td>\n",
              "      <td>9.181818</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1.545455</td>\n",
              "      <td>9.318182</td>\n",
              "      <td>2.363636</td>\n",
              "      <td>8.272727</td>\n",
              "      <td>1.818182</td>\n",
              "      <td>10.772727</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1.590909</td>\n",
              "      <td>15.363636</td>\n",
              "      <td>2.000000</td>\n",
              "      <td>8.545455</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2.363636</td>\n",
              "      <td>12.863636</td>\n",
              "      <td>3.181818</td>\n",
              "      <td>11.136364</td>\n",
              "      <td>2.590909</td>\n",
              "      <td>6.818182</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>7.454545</td>\n",
              "      <td>1.545455</td>\n",
              "      <td>10.590909</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1.227273</td>\n",
              "      <td>11.136364</td>\n",
              "      <td>3.227273</td>\n",
              "      <td>9.772727</td>\n",
              "      <td>1.772727</td>\n",
              "      <td>12.136364</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2.818182</td>\n",
              "      <td>11.409091</td>\n",
              "      <td>2.181818</td>\n",
              "      <td>9.954545</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>9.136364</td>\n",
              "      <td>2.136364</td>\n",
              "      <td>9.318182</td>\n",
              "      <td>1.954545</td>\n",
              "      <td>9.590909</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>118859.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.137255</td>\n",
              "      <td>7.758170</td>\n",
              "      <td>0.705882</td>\n",
              "      <td>4.725490</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>7.254902</td>\n",
              "      <td>0.882353</td>\n",
              "      <td>5.594771</td>\n",
              "      <td>1.346405</td>\n",
              "      <td>7.431373</td>\n",
              "      <td>2.836601</td>\n",
              "      <td>16.104575</td>\n",
              "      <td>1.104575</td>\n",
              "      <td>6.882353</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>5.496732</td>\n",
              "      <td>1.418301</td>\n",
              "      <td>7.549020</td>\n",
              "      <td>0.856209</td>\n",
              "      <td>6.248366</td>\n",
              "      <td>1.287582</td>\n",
              "      <td>7.006536</td>\n",
              "      <td>0.856209</td>\n",
              "      <td>6.241830</td>\n",
              "      <td>1.993464</td>\n",
              "      <td>23.379085</td>\n",
              "      <td>2.111111</td>\n",
              "      <td>16.274510</td>\n",
              "      <td>2.895425</td>\n",
              "      <td>19.019608</td>\n",
              "      <td>2.647059</td>\n",
              "      <td>14.287582</td>\n",
              "      <td>4.130719</td>\n",
              "      <td>17.235294</td>\n",
              "      <td>3.980392</td>\n",
              "      <td>14.522876</td>\n",
              "      <td>3.823529</td>\n",
              "      <td>22.444444</td>\n",
              "      <td>4.241830</td>\n",
              "      <td>17.372549</td>\n",
              "      <td>...</td>\n",
              "      <td>14.751634</td>\n",
              "      <td>4.640523</td>\n",
              "      <td>20.758170</td>\n",
              "      <td>4.431373</td>\n",
              "      <td>21.209150</td>\n",
              "      <td>5.947712</td>\n",
              "      <td>19.209150</td>\n",
              "      <td>5.875817</td>\n",
              "      <td>21.725490</td>\n",
              "      <td>4.901961</td>\n",
              "      <td>15.790850</td>\n",
              "      <td>5.248366</td>\n",
              "      <td>16.254902</td>\n",
              "      <td>3.254902</td>\n",
              "      <td>10.856209</td>\n",
              "      <td>4.143791</td>\n",
              "      <td>11.830065</td>\n",
              "      <td>2.732026</td>\n",
              "      <td>7.869281</td>\n",
              "      <td>5.267974</td>\n",
              "      <td>12.862745</td>\n",
              "      <td>2.732026</td>\n",
              "      <td>7.803922</td>\n",
              "      <td>2.150327</td>\n",
              "      <td>10.111111</td>\n",
              "      <td>2.294118</td>\n",
              "      <td>8.830065</td>\n",
              "      <td>2.359477</td>\n",
              "      <td>6.869281</td>\n",
              "      <td>2.235294</td>\n",
              "      <td>15.287582</td>\n",
              "      <td>2.339869</td>\n",
              "      <td>7.542484</td>\n",
              "      <td>1.777778</td>\n",
              "      <td>8.679739</td>\n",
              "      <td>1.627451</td>\n",
              "      <td>7.026144</td>\n",
              "      <td>2.424837</td>\n",
              "      <td>9.052288</td>\n",
              "      <td>97477.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2.223183</td>\n",
              "      <td>7.326990</td>\n",
              "      <td>1.904844</td>\n",
              "      <td>6.553633</td>\n",
              "      <td>1.930796</td>\n",
              "      <td>9.442907</td>\n",
              "      <td>1.871972</td>\n",
              "      <td>6.891003</td>\n",
              "      <td>2.129758</td>\n",
              "      <td>7.645329</td>\n",
              "      <td>3.726644</td>\n",
              "      <td>15.249135</td>\n",
              "      <td>1.762976</td>\n",
              "      <td>6.420415</td>\n",
              "      <td>1.610727</td>\n",
              "      <td>7.237024</td>\n",
              "      <td>1.946367</td>\n",
              "      <td>8.410035</td>\n",
              "      <td>2.178201</td>\n",
              "      <td>9.287197</td>\n",
              "      <td>2.411765</td>\n",
              "      <td>9.077855</td>\n",
              "      <td>1.896194</td>\n",
              "      <td>9.363322</td>\n",
              "      <td>2.603806</td>\n",
              "      <td>12.671280</td>\n",
              "      <td>2.429066</td>\n",
              "      <td>13.475779</td>\n",
              "      <td>2.859862</td>\n",
              "      <td>11.269896</td>\n",
              "      <td>2.169550</td>\n",
              "      <td>10.060554</td>\n",
              "      <td>2.768166</td>\n",
              "      <td>9.553633</td>\n",
              "      <td>2.131488</td>\n",
              "      <td>10.072664</td>\n",
              "      <td>2.695502</td>\n",
              "      <td>11.344291</td>\n",
              "      <td>2.266436</td>\n",
              "      <td>9.012111</td>\n",
              "      <td>...</td>\n",
              "      <td>9.608997</td>\n",
              "      <td>3.200692</td>\n",
              "      <td>17.339100</td>\n",
              "      <td>3.304498</td>\n",
              "      <td>12.577855</td>\n",
              "      <td>3.567474</td>\n",
              "      <td>13.972318</td>\n",
              "      <td>3.399654</td>\n",
              "      <td>11.550173</td>\n",
              "      <td>3.608997</td>\n",
              "      <td>10.935986</td>\n",
              "      <td>3.008651</td>\n",
              "      <td>9.434256</td>\n",
              "      <td>3.266436</td>\n",
              "      <td>10.474048</td>\n",
              "      <td>2.955017</td>\n",
              "      <td>10.346021</td>\n",
              "      <td>2.764706</td>\n",
              "      <td>11.133218</td>\n",
              "      <td>2.752595</td>\n",
              "      <td>9.804498</td>\n",
              "      <td>2.750865</td>\n",
              "      <td>10.935986</td>\n",
              "      <td>2.484429</td>\n",
              "      <td>11.025952</td>\n",
              "      <td>2.804498</td>\n",
              "      <td>10.377163</td>\n",
              "      <td>2.519031</td>\n",
              "      <td>9.332180</td>\n",
              "      <td>2.468858</td>\n",
              "      <td>13.237024</td>\n",
              "      <td>2.385813</td>\n",
              "      <td>9.250865</td>\n",
              "      <td>2.960208</td>\n",
              "      <td>9.309689</td>\n",
              "      <td>2.318339</td>\n",
              "      <td>8.731834</td>\n",
              "      <td>3.013841</td>\n",
              "      <td>10.859862</td>\n",
              "      <td>65754.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2.090909</td>\n",
              "      <td>14.045455</td>\n",
              "      <td>2.462121</td>\n",
              "      <td>11.409091</td>\n",
              "      <td>3.969697</td>\n",
              "      <td>19.333333</td>\n",
              "      <td>2.590909</td>\n",
              "      <td>12.227273</td>\n",
              "      <td>2.742424</td>\n",
              "      <td>12.318182</td>\n",
              "      <td>3.454545</td>\n",
              "      <td>12.780303</td>\n",
              "      <td>2.325758</td>\n",
              "      <td>16.598485</td>\n",
              "      <td>3.757576</td>\n",
              "      <td>22.780303</td>\n",
              "      <td>2.954545</td>\n",
              "      <td>21.439394</td>\n",
              "      <td>2.848485</td>\n",
              "      <td>21.765152</td>\n",
              "      <td>3.090909</td>\n",
              "      <td>23.909091</td>\n",
              "      <td>3.530303</td>\n",
              "      <td>39.219697</td>\n",
              "      <td>7.393939</td>\n",
              "      <td>92.030303</td>\n",
              "      <td>6.325758</td>\n",
              "      <td>65.325758</td>\n",
              "      <td>4.878788</td>\n",
              "      <td>64.098485</td>\n",
              "      <td>7.045455</td>\n",
              "      <td>52.803030</td>\n",
              "      <td>3.257576</td>\n",
              "      <td>39.045455</td>\n",
              "      <td>5.962121</td>\n",
              "      <td>51.613636</td>\n",
              "      <td>4.257576</td>\n",
              "      <td>51.265152</td>\n",
              "      <td>4.886364</td>\n",
              "      <td>34.128788</td>\n",
              "      <td>...</td>\n",
              "      <td>29.333333</td>\n",
              "      <td>6.909091</td>\n",
              "      <td>72.530303</td>\n",
              "      <td>8.939394</td>\n",
              "      <td>51.833333</td>\n",
              "      <td>9.431818</td>\n",
              "      <td>64.310606</td>\n",
              "      <td>3.659091</td>\n",
              "      <td>20.946970</td>\n",
              "      <td>3.484848</td>\n",
              "      <td>22.159091</td>\n",
              "      <td>5.333333</td>\n",
              "      <td>20.787879</td>\n",
              "      <td>3.469697</td>\n",
              "      <td>16.810606</td>\n",
              "      <td>3.984848</td>\n",
              "      <td>12.227273</td>\n",
              "      <td>4.583333</td>\n",
              "      <td>15.893939</td>\n",
              "      <td>4.325758</td>\n",
              "      <td>10.204545</td>\n",
              "      <td>2.856061</td>\n",
              "      <td>9.325758</td>\n",
              "      <td>6.515152</td>\n",
              "      <td>10.166667</td>\n",
              "      <td>3.840909</td>\n",
              "      <td>8.363636</td>\n",
              "      <td>4.325758</td>\n",
              "      <td>7.265152</td>\n",
              "      <td>9.537879</td>\n",
              "      <td>25.234848</td>\n",
              "      <td>4.272727</td>\n",
              "      <td>10.431818</td>\n",
              "      <td>3.462121</td>\n",
              "      <td>16.500000</td>\n",
              "      <td>3.075758</td>\n",
              "      <td>13.515152</td>\n",
              "      <td>3.606061</td>\n",
              "      <td>18.939394</td>\n",
              "      <td>120372.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 83 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "    04_0_VH    04_0_VV   04_1_VH  ...   11_4_VH    11_4_VV    fields\n",
              "0  0.816514   5.449541  0.651376  ...       NaN        NaN     714.0\n",
              "1  0.954545  22.136364  1.636364  ...       NaN        NaN  118859.0\n",
              "2  1.137255   7.758170  0.705882  ...  2.424837   9.052288   97477.0\n",
              "3  2.223183   7.326990  1.904844  ...  3.013841  10.859862   65754.0\n",
              "4  2.090909  14.045455  2.462121  ...  3.606061  18.939394  120372.0\n",
              "\n",
              "[5 rows x 83 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q_fhmm9WhqWH"
      },
      "source": [
        "sar_test.to_csv('/content/drive/MyDrive/CompeData/Radiant/s1_all_data/sar_test.csv', index = False)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}