index.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Resolve PIDS\n",
    "\n",
    "Ian Thomas\n",
    "Research Capability Unit, \n",
    "RMIT University\n",
    "\n",
    "\n",
    "## Introduction\n",
    "\n",
    "The script takes a list of library persistent identifiers (PIDS), such as ISBN and LCCN and return the corresponding\n",
    "data from the [openlibary.org](http://openlibrary.org).  It creates a CSV file with selected fields, with blank lines for any missing data.\n",
    "\n",
    "## Instructions\n",
    "\n",
    "Replace the example list of PIDS in the next cell with one PID per line with the correct prefix."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "pids = \"\"\"\n",
    "    LCCN:93005405\n",
    "    ISBN:123141244124\n",
    "    ISBN:9780980200447\n",
    "    OL25319523M\n",
    "    OL15203199W\n",
    "    \"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When you are ready, select the above cell and press the run \">\" button above to advance execution through this cell. \n",
    "\n",
    "Then press the run button multiple times to execute the next cell and then wait for output.\n",
    "\n",
    "When the program completes you will see a new file called `final.csv` in left hand side panel. Double click that CSV file to review here, and  and then right click the filename to slect and  download the file to your desktop.\n",
    "\n",
    "This notebook deployment under [mybinder.org](http://mybinder.org) is temporary and will disappear quickly if unused or left idle, so it is important that you download the CSV as soon as your are satisfied with the result.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import sys\n",
    "import json\n",
    "from jsonpath_rw import jsonpath, parse\n",
    "from time import sleep\n",
    "import requests\n",
    "import random\n",
    "from itertools import zip_longest\n",
    "import logging\n",
    "from pprint import pformat\n",
    "from IPython.display import display\n",
    "\n",
    "logger = logging.getLogger(__name__)\n",
    "\n",
    "#logging.basicConfig(level=logging.DEBUG)\n",
    "logging.basicConfig(level=logging.INFO)\n",
    "\n",
    "# The query for openlibrary {} is replaced with the pids\n",
    "query_template = \"https://openlibrary.org/api/books?bibkeys={}&jscmd=data&format=json\"\n",
    "\n",
    "# The set of patterns for selecting which fields in the response to put into the csv\n",
    "# Uses the jsonpath schema: http://goessner.net/articles/JsonPath/\n",
    "paths = [\n",
    "    (\"$.title\", \"Title\"),\n",
    "    (\"$.authors[*].name\", \"Author(s) Name\"),\n",
    "    (\"$.publish_date\", \"Publish Date\"),\n",
    "    (\"$.url\",\"URL\")\n",
    "    ]\n",
    "\n",
    "\n",
    "def grouper(iterable, n, fillvalue=None):\n",
    "    args = [iter(iterable)] * n\n",
    "    return zip_longest(*args, fillvalue=fillvalue)\n",
    "\n",
    "output_file = \"final.csv\"\n",
    "chunk_size = 10\n",
    "pid_column_name = \"pid\"\n",
    "\n",
    "# Try to find parse errors earlier\n",
    "for path,pname in paths:\n",
    "    try:\n",
    "        jsonpath_expr = parse(path)\n",
    "        value = [match.value for match in jsonpath_expr.find([])]\n",
    "    except Exception as e:\n",
    "        logger.error(f\"Possible path error in {path} gave error: {repr(e)}\")\n",
    "        raise e\n",
    "\n",
    "pids_list =  pids.split()\n",
    "logger.info(f\"Pids to Check: {', '.join(pids_list)}\")\n",
    "logger.info(\"checking...\")\n",
    "# TODO: Check api encoding \n",
    "results = {}\n",
    "for i, pds in enumerate(grouper(pids_list, chunk_size)):\n",
    "    for p in pds:\n",
    "        results[p] = {}\n",
    "    logger.debug(f\"pds={pds}\")\n",
    "    q_str = ','.join([x.strip() for x in pds if x is not None])\n",
    "    query = query_template.format(q_str)\n",
    "    logger.info(f\"query = {query}\")\n",
    "    # TODO: add retries for this request\n",
    "    res = requests.get(query)\n",
    "    if res:\n",
    "        results.update(res.json())\n",
    "    sleep(5+i) # sleep to avoid flooding api\n",
    "\n",
    "logger.debug(f\"query results: {pformat(results)}\")\n",
    "\n",
    "with open(output_file, 'w',  encoding=\"utf-8-sig\") as csvfile:   \n",
    "    field_names = [pid_column_name] + [pname for (p, pname) in paths]\n",
    "    logger.debug(f\"field_names= {field_names}\")\n",
    "    writer = csv.DictWriter(csvfile, fieldnames=field_names)\n",
    "    writer.writeheader()\n",
    "    for k,v in results.items():\n",
    "        logger.debug(f\"v={pformat(v)}\")\n",
    "        row = {}\n",
    "        row[pid_column_name] = k\n",
    "        for p,pname in paths:\n",
    "            try:\n",
    "                jsonpath_expr = parse(p)\n",
    "                value = [match.value for match in jsonpath_expr.find(v)]\n",
    "            except Exception as e:\n",
    "                logger.warning(f\"Parse error in entry {pid} gave error: {e}\")\n",
    "                value = []\n",
    "            row[pname] = ', '.join(value)\n",
    "        writer.writerow(row)\n",
    "        \n",
    "logger.info(\"Done\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Don't forget to download the final.csv result from the left hand panel."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}