Setup.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "## Set Up Azure environment to train CV models"
      ],
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import azureml.core\n",
        "from azureml.core import Experiment, Environment, Workspace, Dataset, Datastore, ScriptRunConfig\n",
        "import os\n",
        "import shutil\n",
        "\n",
        "# check core SDK version number\n",
        "\n",
        "print(\"Azure ML SDK Version: \", azureml.core.VERSION)"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": "Azure ML SDK Version:  1.28.0\n"
        }
      ],
      "execution_count": 1,
      "metadata": {
        "gather": {
          "logged": 1647463733094
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# load workspace configuration from the config.json file in the current folder.\n",
        "ws = Workspace.from_config()\n",
        "print(ws.name, ws.location, ws.resource_group, sep='\\t')\n",
        "ws.datastores"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": "wetlands\teastus\tcic_ai\n"
        },
        {
          "output_type": "execute_result",
          "execution_count": 2,
          "data": {
            "text/plain": "{'workspaceartifactstore': {\n   \"name\": \"workspaceartifactstore\",\n   \"container_name\": \"azureml\",\n   \"account_name\": \"wetlands2489098639\",\n   \"protocol\": \"https\",\n   \"endpoint\": \"core.windows.net\"\n },\n 'wetlanddatastore': {\n   \"name\": \"wetlanddatastore\",\n   \"container_name\": \"data\",\n   \"account_name\": \"wetlands2489098639\",\n   \"protocol\": \"https\",\n   \"endpoint\": \"core.windows.net\"\n },\n 'workspacefilestore': {\n   \"name\": \"workspacefilestore\",\n   \"container_name\": \"azureml-filestore-2640f222-8801-40ec-b413-83a7ec003a55\",\n   \"account_name\": \"wetlands2489098639\",\n   \"protocol\": \"https\",\n   \"endpoint\": \"core.windows.net\"\n },\n 'workspaceblobstore': {\n   \"name\": \"workspaceblobstore\",\n   \"container_name\": \"azureml-blobstore-2640f222-8801-40ec-b413-83a7ec003a55\",\n   \"account_name\": \"wetlands2489098639\",\n   \"protocol\": \"https\",\n   \"endpoint\": \"core.windows.net\"\n }}"
          },
          "metadata": {}
        }
      ],
      "execution_count": 2,
      "metadata": {
        "collapsed": true,
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1647463738632
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "datastore = Datastore.get(workspace = ws, datastore_name = 'wetlanddatastore')\n",
        "datastore"
      ],
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 3,
          "data": {
            "text/plain": "{\n  \"name\": \"wetlanddatastore\",\n  \"container_name\": \"data\",\n  \"account_name\": \"wetlands2489098639\",\n  \"protocol\": \"https\",\n  \"endpoint\": \"core.windows.net\"\n}"
          },
          "metadata": {}
        }
      ],
      "execution_count": 3,
      "metadata": {
        "collapsed": true,
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1647463756094
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Register Blob/Fileshare as a workspace Datastore"
      ],
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# register our data share containing wetland image data in this workspace\n",
        "Datastore.register_azure_blob_container(\n",
        "    datastore_name = 'wetlanddatastore',# you make this name up here, it is how the datastore will be referenced in MLStudio from now on\n",
        "    workspace = ws,\n",
        "    container_name = 'data',\n",
        "    account_name = 'wetlands2489098639',\n",
        "    account_key = 'sPNjo0eyBMjErCv+NfwUgZ/4xWb7U5QbmLkIII4N/RiFE3n4QS0SDXIz4zuJQKPiCOesvEgllvBlBxyReqA22w=='\n",
        ")"
      ],
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 4,
          "data": {
            "text/plain": "{\n  \"name\": \"wetlanddatastore\",\n  \"container_name\": \"data\",\n  \"account_name\": \"wetlands2489098639\",\n  \"protocol\": \"https\",\n  \"endpoint\": \"core.windows.net\"\n}"
          },
          "metadata": {}
        }
      ],
      "execution_count": 4,
      "metadata": {
        "collapsed": true,
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1635824093244
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# create a file dataset that can be used in training\n",
        "# datastore = Datastore.get(workspace = ws, datastore_name = 'wetlanddatastore')\n",
        "# datastore_paths = [(datastore, 'GEE')]\n",
        "# gee_dataset = Dataset.File.from_files(path = datastore_paths)\n",
        "# gee_dataset = gee_dataset.register(\n",
        "#     workspace=ws,\n",
        "#     name='gee-acd-data',\n",
        "#     description='training and test TFRecords exported from GEE',\n",
        "#     create_new_version=True)"
      ],
      "outputs": [],
      "execution_count": 5,
      "metadata": {
        "collapsed": true,
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1635824097934
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Create Virtual Environment"
      ],
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# create an environment for the first time\n",
        "\n",
        "envs = Environment.list(workspace = ws)\n",
        "# well start with a pre-built tensorflow environment\n",
        "env = envs.get('AzureML-TensorFlow-2.3-GPU')\n",
        "env\n",
        "\n",
        "# define packages to be installed using CondaDependencies\n",
        "# get the packages that are already part of the pre-built environment\n",
        "conda_dep = env.python.conda_dependencies\n",
        "# list packages to install\n",
        "pip_packages = ['matplotlib', 'rasterio', 'tensorboard']\n",
        "\n",
        "# add each package to the existing conda dependencies\n",
        "for package in pip_packages:\n",
        "    conda_dep.add_pip_package(package)\n",
        "\n",
        "# double check all the packages are there\n",
        "conda_dep.serialize_to_string()\n",
        "# conda_dep = CondaDependencies.create(\n",
        "#     pip_packages=pip_packages)\n",
        "\n",
        "# Now update the conda dependencies of the python environment\n",
        "env.python.conda_dependencies=conda_dep\n",
        "\n",
        "# # Register environment to re-use later\n",
        "env.name = 'wetland-training'\n",
        "env.register(workspace = ws)"
      ],
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 6,
          "data": {
            "text/plain": "{\n    \"databricks\": {\n        \"eggLibraries\": [],\n        \"jarLibraries\": [],\n        \"mavenLibraries\": [],\n        \"pypiLibraries\": [],\n        \"rcranLibraries\": []\n    },\n    \"docker\": {\n        \"arguments\": [],\n        \"baseDockerfile\": null,\n        \"baseImage\": \"mcr.microsoft.com/azureml/openmpi3.1.2-cuda10.1-cudnn7-ubuntu18.04:20210513.v1\",\n        \"baseImageRegistry\": {\n            \"address\": null,\n            \"password\": null,\n            \"registryIdentity\": null,\n            \"username\": null\n        },\n        \"enabled\": false,\n        \"platform\": {\n            \"architecture\": \"amd64\",\n            \"os\": \"Linux\"\n        },\n        \"sharedVolumes\": true,\n        \"shmSize\": null\n    },\n    \"environmentVariables\": {\n        \"EXAMPLE_ENV_VAR\": \"EXAMPLE_VALUE\"\n    },\n    \"inferencingStackVersion\": null,\n    \"name\": \"wetland-training\",\n    \"python\": {\n        \"baseCondaEnvironment\": null,\n        \"condaDependencies\": {\n            \"channels\": [\n                \"conda-forge\"\n            ],\n            \"dependencies\": [\n                \"python=3.6.2\",\n                \"pip=20.2.4\",\n                {\n                    \"pip\": [\n                        \"azureml-core==1.28.0\",\n                        \"azureml-defaults==1.28.0\",\n                        \"azureml-telemetry==1.28.0\",\n                        \"azureml-train-restclients-hyperdrive==1.28.0\",\n                        \"azureml-train-core==1.28.0\",\n                        \"tensorflow-gpu==2.3.0\",\n                        \"cmake==3.18.2\",\n                        \"horovod==0.21.3\",\n                        \"matplotlib\",\n                        \"rasterio\",\n                        \"tensorboard\"\n                    ]\n                }\n            ],\n            \"name\": \"azureml_a3220d0544d334f4cb3ce3bd7b6ca9ec\"\n        },\n        \"condaDependenciesFile\": null,\n        \"interpreterPath\": \"python\",\n        \"userManagedDependencies\": false\n    },\n    \"r\": null,\n    \"spark\": {\n        \"packages\": [],\n        \"precachePackages\": true,\n        \"repositories\": []\n    },\n    \"version\": \"1\"\n}"
          },
          "metadata": {}
        }
      ],
      "execution_count": 6,
      "metadata": {
        "collapsed": true,
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1622044446859
        }
      }
    }
  ],
  "metadata": {
    "kernelspec": {
      "name": "python3-azureml",
      "language": "python",
      "display_name": "Python 3.6 - AzureML"
    },
    "language_info": {
      "name": "python",
      "version": "3.6.9",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "kernel_info": {
      "name": "python3-azureml"
    },
    "nteract": {
      "version": "nteract-front-end@1.0.0"
    },
    "microsoft": {
      "host": {
        "AzureML": {
          "notebookHasBeenCompleted": true
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 2
}