all-journals-filter.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Starting with all pubmed records since 2014, parse dates and filter for records that have publication, acceptance, and retreival dates."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "library(readr)\n",
    "library(dplyr, warn=F)\n",
    "library(lubridate)\n",
    "library(magrittr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "|================================================================================| 100%  263 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning message:\n",
      ": 1 problems parsing 'data/pubmed-since-2014.tsv.gz'. See problems(...) for more details."
     ]
    },
    {
     "data": {
      "text/html": [
       "1572538"
      ],
      "text/latex": [
       "1572538"
      ],
      "text/markdown": [
       "1572538"
      ],
      "text/plain": [
       "[1] 1572538"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "col_types = list(\n",
    "  date_accepted = readr::col_datetime(\"%Y/%m/%d %H:%M\"),\n",
    "  date_epublish = readr::col_datetime(\"%Y/%m/%d %H:%M\"),\n",
    "  date_medline  = readr::col_datetime(\"%Y/%m/%d %H:%M\"),\n",
    "  date_pubmed   = readr::col_datetime(\"%Y/%m/%d %H:%M\"),\n",
    "  date_received = readr::col_datetime(\"%Y/%m/%d %H:%M\"),\n",
    "  epub_date     = readr::col_datetime(\"%Y %b %d\")\n",
    ")\n",
    "\n",
    "# read in tsv of pubmed records\n",
    "pubmed_df = readr::read_tsv('data/pubmed-since-2014.tsv.gz', col_types = col_types)\n",
    "nrow(pubmed_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th></th><th scope=col>journal_abbrev</th><th scope=col>total_articles</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><th scope=row>1</th><td></td><td>2389</td></tr>\n",
       "\t<tr><th scope=row>2</th><td>20 Century Br Hist</td><td>7</td></tr>\n",
       "\t<tr><th scope=row>3</th><td>A A Case Rep</td><td>183</td></tr>\n",
       "\t<tr><th scope=row>4</th><td>AACN Adv Crit Care</td><td>72</td></tr>\n",
       "\t<tr><th scope=row>5</th><td>AANA J</td><td>96</td></tr>\n",
       "\t<tr><th scope=row>6</th><td>AAPS J</td><td>242</td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ll}\n",
       "  & journal_abbrev & total_articles\\\\\n",
       "\\hline\n",
       "\t1 &  & 2389\\\\\n",
       "\t2 & 20 Century Br Hist & 7\\\\\n",
       "\t3 & A A Case Rep & 183\\\\\n",
       "\t4 & AACN Adv Crit Care & 72\\\\\n",
       "\t5 & AANA J & 96\\\\\n",
       "\t6 & AAPS J & 242\\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "Source: local data frame [6 x 2]\n",
       "\n",
       "      journal_abbrev total_articles\n",
       "1                              2389\n",
       "2 20 Century Br Hist              7\n",
       "3       A A Case Rep            183\n",
       "4 AACN Adv Crit Care             72\n",
       "5             AANA J             96\n",
       "6             AAPS J            242"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate total articles per journal before filtering\n",
    "total_df = pubmed_df %>%\n",
    "  dplyr::group_by(journal_abbrev) %>%\n",
    "  dplyr::summarize(total_articles = n())\n",
    "head(total_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "667773"
      ],
      "text/latex": [
       "667773"
      ],
      "text/markdown": [
       "667773"
      ],
      "text/plain": [
       "[1] 667773"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pubmed_df %<>%\n",
    "  dplyr::filter(journal_abbrev != '') %>%\n",
    "  dplyr::filter(! is.na(date_received)) %>%\n",
    "  dplyr::filter(! is.na(date_accepted)) %>%\n",
    "  dplyr::filter(! is.na(epub_date)) %>%\n",
    "  dplyr::mutate(acceptance_days = (date_accepted - date_received ) / lubridate::ddays(1)) %>%\n",
    "  dplyr::mutate(publication_days = (epub_date - date_accepted) / lubridate::ddays(1)) %>%\n",
    "  dplyr::select(pubmed_id, doi, pubtype, journal, journal_abbrev, epub_date, acceptance_days, publication_days)\n",
    "\n",
    "nrow(pubmed_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "657828"
      ],
      "text/latex": [
       "657828"
      ],
      "text/markdown": [
       "657828"
      ],
      "text/plain": [
       "[1] 657828"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# filter to remove likely errors\n",
    "pubmed_df %<>%\n",
    "  dplyr::filter(publication_days > 0) %>%\n",
    "  dplyr::filter(publication_days < 750) %>%\n",
    "  dplyr::filter(acceptance_days > 0) %>%\n",
    "  dplyr::filter(acceptance_days < 1000)\n",
    "\n",
    "nrow(pubmed_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th></th><th scope=col>pubmed_id</th><th scope=col>doi</th><th scope=col>pubtype</th><th scope=col>journal</th><th scope=col>journal_abbrev</th><th scope=col>epub_date</th><th scope=col>acceptance_days</th><th scope=col>publication_days</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><th scope=row>1</th><td>26111384</td><td>10.7554/eLife.07072</td><td>Journal Article</td><td>eLife</td><td>Elife</td><td>2015-06-25</td><td>111</td><td>17</td></tr>\n",
       "\t<tr><th scope=row>2</th><td>26111374</td><td>10.1371/journal.pcbi.1004310</td><td>Journal Article</td><td>PLoS computational biology</td><td>PLoS Comput Biol</td><td>2015-06-25</td><td>163</td><td>56</td></tr>\n",
       "\t<tr><th scope=row>3</th><td>26111373</td><td>10.1111/wrr.12333</td><td>Journal Article</td><td>Wound repair and regeneration : official publication of the Wound Healing Society [and] the European Tissue Repair Society</td><td>Wound Repair Regen</td><td>2015-06-24</td><td>97</td><td>7</td></tr>\n",
       "\t<tr><th scope=row>4</th><td>26111363</td><td>10.1002/ppul.23226</td><td>Journal Article</td><td>Pediatric pulmonology</td><td>Pediatr Pulmonol</td><td>2015-06-25</td><td>159</td><td>32</td></tr>\n",
       "\t<tr><th scope=row>5</th><td>26111358</td><td>10.1111/jgh.13026</td><td>Journal Article</td><td>Journal of gastroenterology and hepatology</td><td>J Gastroenterol Hepatol</td><td>2015-06-25</td><td>151</td><td>16</td></tr>\n",
       "\t<tr><th scope=row>6</th><td>26111357</td><td>10.1002/asia.201500332</td><td>Journal Article</td><td>Chemistry, an Asian journal</td><td>Chem Asian J</td><td>2015-06-25</td><td>82</td><td>2</td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllllll}\n",
       "  & pubmed_id & doi & pubtype & journal & journal_abbrev & epub_date & acceptance_days & publication_days\\\\\n",
       "\\hline\n",
       "\t1 & 26111384 & 10.7554/eLife.07072 & Journal Article & eLife & Elife & 2015-06-25 & 111 & 17\\\\\n",
       "\t2 & 26111374 & 10.1371/journal.pcbi.1004310 & Journal Article & PLoS computational biology & PLoS Comput Biol & 2015-06-25 & 163 & 56\\\\\n",
       "\t3 & 26111373 & 10.1111/wrr.12333 & Journal Article & Wound repair and regeneration : official publication of the Wound Healing Society [and] the European Tissue Repair Society & Wound Repair Regen & 2015-06-24 & 97 & 7\\\\\n",
       "\t4 & 26111363 & 10.1002/ppul.23226 & Journal Article & Pediatric pulmonology & Pediatr Pulmonol & 2015-06-25 & 159 & 32\\\\\n",
       "\t5 & 26111358 & 10.1111/jgh.13026 & Journal Article & Journal of gastroenterology and hepatology & J Gastroenterol Hepatol & 2015-06-25 & 151 & 16\\\\\n",
       "\t6 & 26111357 & 10.1002/asia.201500332 & Journal Article & Chemistry, an Asian journal & Chem Asian J & 2015-06-25 & 82 & 2\\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "Source: local data frame [6 x 8]\n",
       "\n",
       "  pubmed_id                          doi         pubtype\n",
       "1  26111384          10.7554/eLife.07072 Journal Article\n",
       "2  26111374 10.1371/journal.pcbi.1004310 Journal Article\n",
       "3  26111373            10.1111/wrr.12333 Journal Article\n",
       "4  26111363           10.1002/ppul.23226 Journal Article\n",
       "5  26111358            10.1111/jgh.13026 Journal Article\n",
       "6  26111357       10.1002/asia.201500332 Journal Article\n",
       "Variables not shown: journal (chr), journal_abbrev (chr), epub_date (time),\n",
       "  acceptance_days (dbl), publication_days (dbl)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "head(pubmed_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# write as tsv\n",
    "pubmed_df %>% \n",
    "  readr::write_tsv('data/pubmed-since-2014-filtered.tsv')\n",
    "system('gzip --force data/pubmed-since-2014-filtered.tsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Joining by: \"journal_abbrev\"\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "3482"
      ],
      "text/latex": [
       "3482"
      ],
      "text/markdown": [
       "3482"
      ],
      "text/plain": [
       "[1] 3482"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summary_df = pubmed_df %>%\n",
    "  dplyr::group_by(journal, journal_abbrev) %>%\n",
    "  dplyr::summarize(\n",
    "    articles = n(),\n",
    "    publication_days_mean = mean(publication_days),\n",
    "    publication_days_median = median(publication_days),\n",
    "    publication_days_max = max(publication_days),\n",
    "    publication_days_min = min(publication_days),\n",
    "    publication_days_geq_100 = mean(publication_days >= 100),\n",
    "    acceptance_days_mean = mean(acceptance_days),\n",
    "    acceptance_days_median = median(acceptance_days),\n",
    "    acceptance_days_max = max(acceptance_days),\n",
    "    acceptance_days_min = min(acceptance_days)\n",
    "  ) %>%\n",
    "  dplyr::left_join(total_df)\n",
    "\n",
    "nrow(summary_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th></th><th scope=col>journal</th><th scope=col>journal_abbrev</th><th scope=col>articles</th><th scope=col>publication_days_mean</th><th scope=col>publication_days_median</th><th scope=col>publication_days_max</th><th scope=col>publication_days_min</th><th scope=col>publication_days_geq_100</th><th scope=col>acceptance_days_mean</th><th scope=col>acceptance_days_median</th><th scope=col>acceptance_days_max</th><th scope=col>acceptance_days_min</th><th scope=col>total_articles</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><th scope=row>1</th><td>AAPS PharmSciTech</td><td>AAPS PharmSciTech</td><td>282</td><td>32.96809</td><td>27</td><td>210</td><td>8</td><td>0.0248227</td><td>116.4043</td><td>108</td><td>447</td><td>1</td><td>287</td></tr>\n",
       "\t<tr><th scope=row>2</th><td>ACS macro letters</td><td>ACS Macro Lett</td><td>17</td><td>6.294118</td><td>5</td><td>22</td><td>1</td><td>0</td><td>54.47059</td><td>44</td><td>105</td><td>8</td><td>23</td></tr>\n",
       "\t<tr><th scope=row>3</th><td>ACS medicinal chemistry letters</td><td>ACS Med Chem Lett</td><td>59</td><td>6.915254</td><td>5</td><td>62</td><td>1</td><td>0</td><td>65.20339</td><td>57</td><td>150</td><td>6</td><td>589</td></tr>\n",
       "\t<tr><th scope=row>4</th><td>AIDS research and therapy</td><td>AIDS Res Ther</td><td>60</td><td>18.41667</td><td>13.5</td><td>65</td><td>3</td><td>0</td><td>139.4333</td><td>136</td><td>336</td><td>23</td><td>61</td></tr>\n",
       "\t<tr><th scope=row>5</th><td>AIDS research and treatment</td><td>AIDS Res Treat</td><td>41</td><td>29.58537</td><td>26</td><td>66</td><td>7</td><td>0</td><td>116.2927</td><td>111</td><td>226</td><td>32</td><td>43</td></tr>\n",
       "\t<tr><th scope=row>6</th><td>AIP advances</td><td>AIP Adv</td><td>5</td><td>43</td><td>9</td><td>176</td><td>8</td><td>0.2</td><td>48.2</td><td>43</td><td>89</td><td>16</td><td>5</td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllllllllllll}\n",
       "  & journal & journal_abbrev & articles & publication_days_mean & publication_days_median & publication_days_max & publication_days_min & publication_days_geq_100 & acceptance_days_mean & acceptance_days_median & acceptance_days_max & acceptance_days_min & total_articles\\\\\n",
       "\\hline\n",
       "\t1 & AAPS PharmSciTech & AAPS PharmSciTech & 282 & 32.96809 & 27 & 210 & 8 & 0.0248227 & 116.4043 & 108 & 447 & 1 & 287\\\\\n",
       "\t2 & ACS macro letters & ACS Macro Lett & 17 & 6.294118 & 5 & 22 & 1 & 0 & 54.47059 & 44 & 105 & 8 & 23\\\\\n",
       "\t3 & ACS medicinal chemistry letters & ACS Med Chem Lett & 59 & 6.915254 & 5 & 62 & 1 & 0 & 65.20339 & 57 & 150 & 6 & 589\\\\\n",
       "\t4 & AIDS research and therapy & AIDS Res Ther & 60 & 18.41667 & 13.5 & 65 & 3 & 0 & 139.4333 & 136 & 336 & 23 & 61\\\\\n",
       "\t5 & AIDS research and treatment & AIDS Res Treat & 41 & 29.58537 & 26 & 66 & 7 & 0 & 116.2927 & 111 & 226 & 32 & 43\\\\\n",
       "\t6 & AIP advances & AIP Adv & 5 & 43 & 9 & 176 & 8 & 0.2 & 48.2 & 43 & 89 & 16 & 5\\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "Source: local data frame [6 x 13]\n",
       "Groups: journal\n",
       "\n",
       "                          journal    journal_abbrev articles\n",
       "1               AAPS PharmSciTech AAPS PharmSciTech      282\n",
       "2               ACS macro letters    ACS Macro Lett       17\n",
       "3 ACS medicinal chemistry letters ACS Med Chem Lett       59\n",
       "4       AIDS research and therapy     AIDS Res Ther       60\n",
       "5     AIDS research and treatment    AIDS Res Treat       41\n",
       "6                    AIP advances           AIP Adv        5\n",
       "Variables not shown: publication_days_mean (dbl), publication_days_median\n",
       "  (dbl), publication_days_max (dbl), publication_days_min (dbl),\n",
       "  publication_days_geq_100 (dbl), acceptance_days_mean (dbl),\n",
       "  acceptance_days_median (dbl), acceptance_days_max (dbl), acceptance_days_min\n",
       "  (dbl), total_articles (int)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "head(summary_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# write as tsv\n",
    "summary_df %>% \n",
    "  readr::write_tsv('data/pubmed-since-2014-summary.tsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "R",
   "language": "R",
   "name": "ir"
  },
  "language_info": {
   "codemirror_mode": "r",
   "file_extension": ".r",
   "mimetype": "text/x-r-source",
   "name": "R",
   "pygments_lexer": "r",
   "version": "3.2.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}