notebooks-guides/Zenodo_API_Notebook.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Zenodo API Notebook.ipynb",
      "version": "0.3.2",
      "views": {},
      "default_view": {},
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    }
  },
  "cells": [
    {
      "metadata": {
        "id": "d6Q1F4tKTu96",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Zenodo API Example\n",
        "\n",
        "Demonstration of how to query the Zenodo REST API. First of all, we create a couple of Python helper functions and classes to make it easier to query Zenodo. Don't worry about the details:"
      ]
    },
    {
      "metadata": {
        "id": "F9WF-j-ETu97",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "import requests\n",
        "\n",
        "def raise_on_error(res):\n",
        "    \"\"\"Helper to check response for errors.\"\"\"\n",
        "    if res.status_code != 200:\n",
        "        data = res.json()\n",
        "        raise Exception('[{status}] {message}'.format(**data))\n",
        "\n",
        "        \n",
        "class ResultWrapper(object):\n",
        "    \"\"\"Helper to work with search results\"\"\"\n",
        "    def __init__(self, session, response):\n",
        "        raise_on_error(response)\n",
        "        self._session = session\n",
        "        self.response = response\n",
        "        \n",
        "    def has_json(self):\n",
        "        \"\"\"Check if content type is JSON.\"\"\"\n",
        "        return self.response.headers['Content-Type'] == 'application/json'\n",
        "            \n",
        "    @property\n",
        "    def data(self):\n",
        "        \"\"\"Get JSON data.\"\"\"\n",
        "        return self.response.json() if self.has_json() else {}\n",
        "    \n",
        "    @property\n",
        "    def aggregations(self):\n",
        "        \"\"\"Get aggregations.\"\"\"\n",
        "        return self.response.json()['aggregations']  if self.has_json() else {}\n",
        "    \n",
        "    @property\n",
        "    def total(self):\n",
        "        \"\"\"Get total number of hits.\"\"\"\n",
        "        return self.response.json()['hits']['total'] if self.has_json() else None\n",
        "        \n",
        "    @property\n",
        "    def pages(self):\n",
        "        \"\"\"Helper to fetch all result pages.\"\"\"\n",
        "        yield self.response\n",
        "        next_url = self.response.links.get('next', {}).get('url')\n",
        "        while next_url:\n",
        "            res_page = self._session.get(next_url)\n",
        "            raise_on_error(res_page)\n",
        "            yield res_page\n",
        "            next_url = res_page.links.get('next', {}).get('url')\n",
        "    \n",
        "    @property\n",
        "    def hits(self):\n",
        "        \"\"\"Helper to iterate over each hit.\"\"\"\n",
        "        if not self.has_json():\n",
        "            return None\n",
        "        for res in self.pages:\n",
        "            for h in res.json()['hits']['hits']:\n",
        "                yield h\n",
        "                \n",
        "class ZenodoClient(object):\n",
        "    \"\"\"Simple Zenodo API Client\"\"\"\n",
        "\n",
        "    def __init__(self, accept=None, token=None):\n",
        "        self._accept = accept\n",
        "        self._token = token\n",
        "        self._session = None\n",
        "        self._endpoint = 'https://www.zenodo.org/api/'\n",
        "        self._endpoint_search = '{}records/'.format(self._endpoint)\n",
        "        self._endpoint_styles = '{}csl/styles'.format(self._endpoint)\n",
        "        \n",
        "    @property\n",
        "    def session(self):\n",
        "        \"\"\"Create a session for making HTTP requests to the API.\"\"\"\n",
        "        if self._session is None:\n",
        "            self._session = requests.Session()    \n",
        "            # Construct headers\n",
        "            headers = {\n",
        "                'Accept': self._accept or 'application/vnd.zenodo.v1+json',\n",
        "                'Accept-Charset': 'utf-8',\n",
        "            }\n",
        "            if self._token:\n",
        "                headers['Authorization'] = 'Bearer {}'.format(self._token)            \n",
        "            self._session.headers.update(headers)\n",
        "        return self._session\n",
        "\n",
        "    def search(self, query=None, size=None, sort=None, **filters):\n",
        "        \"\"\"Search Zenodo\"\"\"\n",
        "        params = {'q': query or '', 'sort': sort or 'bestmatch', 'size': size or 100}\n",
        "        if filters:\n",
        "            for f, vals in filters.items():\n",
        "                params[f] = vals\n",
        "        return ResultWrapper(self.session, self.session.get(self._endpoint_search, params=params))\n",
        "    \n",
        "    def record(self, record_id, accept=None, **params):\n",
        "        \"\"\"Retrieve a single record.\"\"\"\n",
        "        url = '{base}{recid}'.format(base=self._endpoint_search, recid=record_id)\n",
        "        params = params or {}\n",
        "        headers = self.session.headers\n",
        "        if accept:\n",
        "            headers['Accept'] = accept\n",
        "            \n",
        "        res = self.session.get(url, headers=headers, params=params)\n",
        "        raise_on_error(res)\n",
        "        return res\n",
        "    \n",
        "    def csl_styles(self):\n",
        "        res = self.session.get(self._endpoint_styles)\n",
        "        raise_on_error(res)\n",
        "        return res\n",
        "        "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "6yH4IYPITu9-",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Initialize the API client and make a query\n",
        "Using our just created helper classes we can now create a Zenodo API client:"
      ]
    },
    {
      "metadata": {
        "id": "CMW19B6BTu9_",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "api = ZenodoClient()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "HRa5xm75Tu-B",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Simple search"
      ]
    },
    {
      "metadata": {
        "id": "ncZbMTDfTu-C",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Using the API client we can now execute queries against Zenodo search API:"
      ]
    },
    {
      "metadata": {
        "id": "XtFlMyjvTu-C",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "90e1cff2-666e-4276-e97c-a95acb72fca4",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1516698631661,
          "user_tz": -60,
          "elapsed": 3013,
          "user": {
            "displayName": "Christopher Erdmann",
            "photoUrl": "//lh4.googleusercontent.com/-6MICru_0gt0/AAAAAAAAAAI/AAAAAAAAAAo/DbCa8uLldiw/s50-c-k-no/photo.jpg",
            "userId": "110635524421337976220"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "result = api.search('North Carolina State University')\n",
        "result.total"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "98741"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "metadata": {
        "id": "JcAkv5fgTu-G",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Phrase search\n",
        "\n",
        "The query string is exactly like you would type in the Zenodo search box. The underlying search engine on Zenodo is Elasticsearch, which has a powerful query syntax. Above, we are searching for 4 terms, let's instead make it into a phrase search:"
      ]
    },
    {
      "metadata": {
        "id": "Tfbyo-yDTu-J",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "70b0fb3e-9b90-4746-947a-2cd1094122eb",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1516698640964,
          "user_tz": -60,
          "elapsed": 1430,
          "user": {
            "displayName": "Christopher Erdmann",
            "photoUrl": "//lh4.googleusercontent.com/-6MICru_0gt0/AAAAAAAAAAI/AAAAAAAAAAo/DbCa8uLldiw/s50-c-k-no/photo.jpg",
            "userId": "110635524421337976220"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "result = api.search('\"North Carolina State University\"')\n",
        "result.total"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1496"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "metadata": {
        "id": "_7eebfjvTu-M",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Field search"
      ]
    },
    {
      "metadata": {
        "id": "VxG9HRNBTu-N",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "fc3b757b-aba7-44fe-db24-9b71b0eff134",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1516698645484,
          "user_tz": -60,
          "elapsed": 1274,
          "user": {
            "displayName": "Christopher Erdmann",
            "photoUrl": "//lh4.googleusercontent.com/-6MICru_0gt0/AAAAAAAAAAI/AAAAAAAAAAo/DbCa8uLldiw/s50-c-k-no/photo.jpg",
            "userId": "110635524421337976220"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "ncsu_query = '+creators.affiliation:(\"NCSU\" \"NC State University\" \"North Carolina State University\") +doi:10.5281*'\n",
        "result = api.search(ncsu_query)\n",
        "result.total"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "59"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "metadata": {
        "id": "a1b_R4MaTu-R",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Notice the query string. Here is a couple of points:\n",
        "\n",
        "* **Field search**: We are now searching on two specific fields - ``creators.affiliation`` and ``doi``.\n",
        "* **Required fields**: The small ``+`` in front the field indicate the field must be present (i.e. both fields are required sincen they have a plus in the front).\n",
        "* **Field multi-term**: On ``creators.affiliation`` we search for ``NCSU`` or ``NC State University`` or ``North Carolina State University``.\n",
        "* **Wildcard**: On the ``doi`` field we search for any string starting with ``10.5281`` (the Zenodo DOI prefix).\n",
        "\n",
        "The full query syntax is documented [here](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax)."
      ]
    },
    {
      "metadata": {
        "id": "mQq-WADtTu-S",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Data inspection\n",
        "Let's know inspect the author affiliations. First let's get all the records and create a list of all the author affiliations:"
      ]
    },
    {
      "metadata": {
        "id": "O7H2OZGfTu-S",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "# Retrieve all records (be careful with large result sets!)\n",
        "records = list(result.hits)\n",
        "\n",
        "# Build a list of author affiliations\n",
        "affiliations = []\n",
        "for record in records:\n",
        "    for creator in record['metadata']['creators']:\n",
        "            affiliations.append(creator.get('affiliation'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "doGKCzYBTu-V",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Next, let's all the affiliations:"
      ]
    },
    {
      "metadata": {
        "id": "XyLN8QDXTu-W",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "# Create a Pandas data frame\n",
        "import pandas as pd\n",
        "df = pd.DataFrame(data=affiliations, columns=['affiliation'])\n",
        "df.groupby('affiliation').size().reset_index(name='count').sort_values(by=['count'], ascending=False).head()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "v6jV6tAzTu-Z",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Export a bibliograph\n",
        "Zenodo supports several other output metadata formats other than JSON. For search results Zenodo can produce the following metadata formats:\n",
        "\n",
        "* BibTex: ``application/x-bibtex``\n",
        "* DublinCore: ``application/x-dc+xml``\n",
        "* DataCite: ``application/x-datacite+xml``\n",
        "* MARC21: ``application/marcxml+xml``\n",
        "\n",
        "For individual records the following additional formats can be produced:\n",
        "\n",
        "* JSON-LD: ``application/ld+json``\n",
        "* Citation Style Language (CSL): ``application/vnd.citationstyles.csl+json``\n",
        "* Citation text: ``text/x-bibliography``\n",
        "\n",
        "For instance we can export a BibTeX bibliography of all records in above search query and write it to a file:"
      ]
    },
    {
      "metadata": {
        "id": "Fb7UMjh9Tu-Z",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "# Create a API client (specifying the desired output format)\n",
        "api = ZenodoClient(accept='application/x-bibtex')\n",
        "\n",
        "# Execute query and write result to \"bibliography.bib\"\n",
        "result = api.search(ncsu_query)\n",
        "with open('bibliography.bib', 'wb') as fp:\n",
        "    # Iterate over all the results and write them to the file.\n",
        "    for page in result.pages:\n",
        "        fp.write(page.content)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "-ZPDVxsDTu-b",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Aggregations\n",
        "Want to know about types, open/closed access, file types and keywords?"
      ]
    },
    {
      "metadata": {
        "id": "5PyDTqkzTu-b",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        },
        "outputId": "0cde7801-bcf7-4d08-9947-39f81cc9d592"
      },
      "cell_type": "code",
      "source": [
        "api = ZenodoClient()\n",
        "result = api.search(ncsu_query)\n",
        "for b in result.aggregations['type']['buckets']:\n",
        "    print('{key}: {doc_count}'.format(**b))\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "dataset: 24\n",
            "publication: 20\n",
            "software: 14\n",
            "presentation: 1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "TLE9sHsBTu-f",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        },
        "outputId": "3feb9360-3dcd-4b02-a712-e55fbdb06234"
      },
      "cell_type": "code",
      "source": [
        "for b in result.aggregations['access_right']['buckets']:\n",
        "    print('{key}: {doc_count}'.format(**b))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "open: 58\n",
            "restricted: 1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "67lTV8F3Tu-j",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        },
        "outputId": "9ee8f3ed-7673-46a1-869d-f9167fa5b089"
      },
      "cell_type": "code",
      "source": [
        "for b in result.aggregations['keywords']['buckets']:\n",
        "    print('{key}: {doc_count}'.format(**b))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "center: 13\n",
            "synthesis: 13\n",
            "evolutionary: 12\n",
            "science: 12\n",
            "annual: 9\n",
            "report: 9\n",
            "carbon: 5\n",
            "biomass: 4\n",
            "dada2: 3\n",
            "defect: 3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "7SwUZyX3Tu-n",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        },
        "outputId": "0ef0035c-7cca-4856-a494-13804e164e22"
      },
      "cell_type": "code",
      "source": [
        "for b in result.aggregations['file_type']['buckets']:\n",
        "    print('{key}: {doc_count}'.format(**b))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "zip: 23\n",
            "pdf: 17\n",
            "csv: 5\n",
            "gz: 4\n",
            "dbf: 3\n",
            "shp: 3\n",
            "shx: 3\n",
            "txt: 3\n",
            "prj: 2\n",
            "tif: 2\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}