frictionless-data-packages.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "NOTE: no error handling implemented"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Befeore running install netCDF4 module (https://unidata.github.io/netcdf4-python/netCDF4/index.html) \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Modify the following section settings your local save path and the id of the folder to elaborate from https://github.com/Swiss-Polar-Institute/frictionless-data-packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "### VARIABLE TO  MODIFY ###\n",
    "id=\"10.5281_zenodo.3250980\"\n",
    "basesavepath=\"c:\\\\SPI\\\\\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datapackage import Package\n",
    "from tableschema import Table\n",
    "import datetime\n",
    "import os\n",
    "import urllib.request\n",
    "from netCDF4 import Dataset\n",
    "import re\n",
    "from datetime import datetime, timedelta\n",
    "from netCDF4 import num2date, date2num, stringtochar\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def CreateNetCDF(savepath_NetCDF,package,resourceData):\n",
    "    #create netcdf\n",
    "    print(\"create netcdf\")\n",
    "    netcdf = Dataset(savepath_NetCDF, \"w\", format=\"NETCDF4\")\n",
    "\n",
    "    #create dimension\n",
    "    print(\"create dimension\")\n",
    "    row = netcdf.createDimension(\"row\", None) ##unlimited dimension\n",
    "\n",
    "    #create global attribute\n",
    "    print(\"create global attribute\")\n",
    "    netcdf.cdm_data_type=\"Other\"\n",
    "    netcdf.Conventions=\"COARDS, CF-1.6, ACDD-1.3\"\n",
    "    netcdf.creator_type = \"institution\"  \n",
    "    #for key in package.descriptor:\n",
    "    #    netcdf.netcdf(key,package.descriptor[key])\n",
    "\n",
    "    if 'x_spi_platform_code' in package.descriptor:\n",
    "        netcdf.platform_code=package.descriptor['x_spi_platform_code']\n",
    "    if 'x_spi_platform_name' in package.descriptor:\n",
    "        netcdf.platform_name=package.descriptor['x_spi_platform_name']\n",
    "    if 'x_spi_type' in package.descriptor:\n",
    "        netcdf.type=package.descriptor['x_spi_type']\n",
    "    if 'x_spi_citation' in package.descriptor:\n",
    "        netcdf.citation=package.descriptor['x_spi_citation']\n",
    "\n",
    "\n",
    "    if 'keywords' in package.descriptor:\n",
    "        netcdf.keywords=','.join(package.descriptor['keywords'])\n",
    "    if 'contributors' in package.descriptor:\n",
    "        if 'title' in package.descriptor['contributors'][0]:\n",
    "            netcdf.creator_name = package.descriptor['contributors'][0]['title']\n",
    "        if 'path' in package.descriptor['contributors'][0]:\n",
    "            netcdf.creator_url = package.descriptor['contributors'][0]['path']\n",
    "        if 'organisation' in package.descriptor['contributors'][0]:\n",
    "            netcdf.institution = package.descriptor['contributors'][0]['organisation']\n",
    "    if 'homepage' in package.descriptor:\n",
    "        netcdf.DOI=package.descriptor['homepage']\n",
    "        netcdf.infoUrl = package.descriptor['homepage']\n",
    "    if 'licenses' in package.descriptor:    \n",
    "        if 'name' in package.descriptor['licenses'][0]:    \n",
    "            netcdf.license = package.descriptor['licenses'][0]['name']\n",
    "    if 'description' in package.descriptor:\n",
    "        netcdf.summary = package.descriptor['description']\n",
    "    if 'contributors' in package.descriptor:\n",
    "        netcdf.contributors=str(package.descriptor['contributors'])\n",
    "\n",
    "    #create variables\n",
    "    print(\"create variables\")\n",
    "    netcdf_variables={}\n",
    "    time = netcdf.createVariable(\"time\",\"f8\",(\"row\",))\n",
    "    time._CoordinateAxisType = \"Time\";\n",
    "    time.axis = \"T\";\n",
    "    time.long_name = \"Time\";\n",
    "    timestandard_name = \"time\";\n",
    "    time.time_origin = \"01-JAN-1970 00:00:00\";\n",
    "    time.units = \"seconds since 1970-01-01T00:00:00Z\"; \n",
    "    for field in resourceData.schema.fields:\n",
    "        fieldName=field.name.replace(\" \",\"\")\n",
    "        fieldName=re.sub(r'\\[.*\\]','',fieldName)\n",
    "        fieldtype=\"\"\n",
    "        fieldsize=\"\"\n",
    "        if field.type==\"string\":\n",
    "            fieldtype=\"S1\"        \n",
    "            maxFieldSize=0\n",
    "            for row in resourceData.iter(keyed = True):\n",
    "                if len(row[field.name])>maxFieldSize:\n",
    "                    maxFieldSize=len(row[field.name]) \n",
    "            fieldsize=str(maxFieldSize)\n",
    "        elif field.type==\"integer\":\n",
    "            fieldtype=\"i4\"\n",
    "        elif field.type==\"time\":\n",
    "            fieldtype=\"S1\"\n",
    "            fieldsize=\"4\"\n",
    "        elif field.type==\"datetime\":  ##is the time variable\n",
    "            fieldtype=\"\"\n",
    "            netcdf_variables['time']=field.name\n",
    "        elif field.type==\"number\":\n",
    "            fieldtype=\"f8\"\n",
    "\n",
    "        if fieldtype==\"\":\n",
    "            continue\n",
    "        if fieldsize==\"\":\n",
    "            netcdf_variables[field.name]=fieldName\n",
    "            netcdf_variable=netcdf.createVariable(fieldName,fieldtype,(\"row\",))\n",
    "        else:\n",
    "            netcdf.createDimension(fieldName + \"_strlen\", int(fieldsize))\n",
    "            netcdf_variables[field.name]=fieldName\n",
    "            netcdf_variable=netcdf.createVariable(fieldName,fieldtype,(\"row\",fieldName + \"_strlen\",))\n",
    "\n",
    "        netcdf_variable.description=field.descriptor['description']\n",
    "        if 'unit' in field.descriptor:\n",
    "            netcdf_variable.unit=field.descriptor['unit']\n",
    "        if 'x_spi_cf_standard_name' in field.descriptor:\n",
    "            netcdf_variable.standard_name=field.descriptor['x_spi_cf_standard_name']\n",
    "        if 'x_spi_cf_unit' in field.descriptor:\n",
    "            netcdf_variable.unit=field.descriptor['x_spi_cf_unit']\n",
    "        if 'x_spi_cf_attribute' in field.descriptor:\n",
    "            netcdf_variable.attribute=field.descriptor['x_spi_cf_attribute']\n",
    "        netcdf_variable.spi_original_name=field.name\n",
    "        #netcdf_variables[field.name]=newField\n",
    "\n",
    "    #netcdf.close()\n",
    "\n",
    "    print(\"fill values\")    \n",
    "    #netcdf = Dataset(savepath_NetCDF, \"a\", format=\"NETCDF4\")\n",
    "    values={}\n",
    "    for row in resourceData.iter(keyed = True):\n",
    "        datetime_value=row[netcdf_variables['time']]\n",
    "        datetime_value_numeric=date2num([datetime_value],units=\"seconds since 1970-01-01T00:00:00Z\",calendar=\"gregorian\")[0]\n",
    "        if not bool(values) :\n",
    "            values['time']=[datetime_value_numeric]\n",
    "        else:\n",
    "            values['time']=np.append(values['time'],datetime_value_numeric)\n",
    "        #netcdf.variables['time'][:]=values    \n",
    "        for field in resourceData.schema.fields:\n",
    "            if netcdf_variables['time'] != field.name:\n",
    "                if netcdf.variables[netcdf_variables[field.name]].dtype == \"S1\":\n",
    "                    #size=str(netcdf.dimensions[netcdf_variables[field.name] +'_strlen'].size)\n",
    "                    #sizeFormat=\"{:<\" + size + \"}\"\n",
    "                    #value=sizeFormat.format(row[field.name])\n",
    "                    value=row[field.name]\n",
    "                    if netcdf_variables[field.name] in values:\n",
    "                        values[netcdf_variables[field.name]]=np.append(values[netcdf_variables[field.name]],value)\n",
    "                    else:\n",
    "                        values[netcdf_variables[field.name]]=[value]                                    \n",
    "                else:\n",
    "                    if netcdf_variables[field.name] in values:\n",
    "                        values[netcdf_variables[field.name]]=np.append(values[netcdf_variables[field.name]],row[field.name])\n",
    "                    else:\n",
    "                        values[netcdf_variables[field.name]]=[row[field.name]]\n",
    "\n",
    "    print(\"populate netcdf variables\")\n",
    "    for field in values:         \n",
    "        if field=='time':\n",
    "            netcdf.variables['time'][:]=values[field]\n",
    "        elif netcdf.variables[field].dtype == \"S1\":\n",
    "            size=str(netcdf.dimensions[field +'_strlen'].size)\n",
    "            datain=np.array(values[field],dtype='S' + size)\n",
    "            netcdf.variables[field][:] = stringtochar(datain) \n",
    "        else:\n",
    "            netcdf.variables[field][:]=values[field]\n",
    "    netcdf.close() \n",
    "    print(\"END\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "datapackage_link=\"https://github.com/raw/Swiss-Polar-Institute/frictionless-data-packages/dev/\" + id + \"/datapackage.json\"\n",
    "tableschema_link=\"https://github.com/raw/Swiss-Polar-Institute/frictionless-data-packages/dev/\" + id + \"/tableschema.json\"\n",
    "netcdf_name=id +\".nc\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "savepath=os.path.join(basesavepath, id)\n",
    "savepath_datapackage=os.path.join(savepath, \"datapackage.json\")\n",
    "savepath_tableschema=os.path.join(savepath, \"tableschema.json\")\n",
    "savepath_NetCDF=os.path.join(savepath, netcdf_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists(savepath):\n",
    "    os.mkdir(savepath)\n",
    "if os.path.exists(savepath_datapackage):\n",
    "    os.remove(savepath_datapackage)\n",
    "urllib.request.urlretrieve(datapackage_link, savepath_datapackage)\n",
    "if os.path.exists(savepath_tableschema):\n",
    "    os.remove(savepath_tableschema)\n",
    "urllib.request.urlretrieve(tableschema_link, savepath_tableschema)\n",
    "if os.path.exists(savepath_NetCDF):\n",
    "    os.remove(savepath_NetCDF)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ace_seawater_chromium_concentration_isotope_composition\n"
     ]
    }
   ],
   "source": [
    "package = Package(savepath_datapackage)\n",
    "print (package.descriptor['name'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Resource: ace_chromium_isotope_concentration \n",
      "Tableschema: ace_chromium_isotope_concentration is valid!\n",
      "create netcdf\n",
      "create dimension\n",
      "create global attribute\n",
      "create variables\n",
      "fill values\n",
      "populate netcdf variables\n",
      "END\n",
      "Resource: data_file_header \n",
      "Resource: readme \n",
      "Resource: schema \n"
     ]
    }
   ],
   "source": [
    "dataIsValid=False\n",
    "for resource in package.resources:\n",
    "    print(f'Resource: {resource.name} ')\n",
    "    if resource.tabular is False:               \n",
    "        continue\n",
    "    if resource.valid is False:\n",
    "        print(f'Tableschema: {resource.name} is NOT valid!')\n",
    "        print(resource.errors)\n",
    "        continue\n",
    "    resource.read()\n",
    "    dataIsValid=True\n",
    "    print(f'Tableschema: {resource.name} is valid!')\n",
    "    resourceData=package.get_resource(resource.name) \n",
    "    print(\"create netcdf for \" + ace_chromium_isotope_concentration)\n",
    "    CreateNetCDF(savepath_NetCDF,package,resourceData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}