convert_to_text.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports\n",
    "import pdfplumber\n",
    "import glob\n",
    "import pandas as pd\n",
    "import pathlib"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Note:__ The parts below are seperated since the first step takes to much time and to be able to continue when terminated.\n",
    "Simply, for continuos processing of PDFs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracted text is None in file ./data/full\\05b318ce39aa6c45bd74ca124d164f07ecf8eaad\n",
      "Extracted text is None in file ./data/full\\0ceb3374fd286becf3a246f5ee814891cbc8fca5\n",
      "Extracted text is None in file ./data/full\\214cd44e332a5500466dc2e6907d5a6e55ddc4ee\n",
      "Extracted text is None in file ./data/full\\237c4db0e117656bd1465f3b97f086da6df6131d\n",
      "Extracted text is None in file ./data/full\\30cd86cd0faecb1abfc1dde33a3d4af4b7638b3b\n",
      "Extracted text is None in file ./data/full\\330d78742fb540749284ad205dde15ae1ea9c511\n",
      "Extracted text is None in file ./data/full\\3677b239c47aeec54310414c31c6683378198f7d\n",
      "Extracted text is None in file ./data/full\\37e50876249c0cba8bda840efef2252ae6621fcf\n",
      "Extracted text is None in file ./data/full\\38c785996a5bddc35529b18cb4f89370bdc6cd68\n",
      "Extracted text is None in file ./data/full\\41cffad66c64081592b5d0927d9571fd9f45e59f\n",
      "Extracted text is None in file ./data/full\\48ee98dc989cb5c13f3baf60cfb674c8f49611f9\n",
      "Extracted text is None in file ./data/full\\4d988b924c8e63f10dcd8d80e32963ae47ec4d2e\n",
      "Extracted text is None in file ./data/full\\53865150b4f5976d6060dd50991d552d4f1e0741\n",
      "Extracted text is None in file ./data/full\\53a642c4ceffeaeca229eee651644fa9aab93823\n",
      "Extracted text is None in file ./data/full\\5a1f8d7eb0e4d5aba9b487af67822c18a6d1f691\n",
      "Extracted text is None in file ./data/full\\5a1f8d7eb0e4d5aba9b487af67822c18a6d1f691\n",
      "Extracted text is None in file ./data/full\\60c9102fd886fe682c1905d17fbfff10bc753152\n",
      "Extracted text is None in file ./data/full\\68d74ea21cf0a4ffa3ccf4b59a94f375aa84f938\n",
      "Extracted text is None in file ./data/full\\6fce81cf97ab857b62df62906ba0830655e0b6c7\n",
      "Extracted text is None in file ./data/full\\755e2a00aca04f7be48cf65432d153b517a900c7\n",
      "Extracted text is None in file ./data/full\\7ad5502ea2ef172a8dd4ecec045457f1c42cd3ca\n",
      "Extracted text is None in file ./data/full\\7e61e666ff1d771a3672779d7e9c14943d250d59\n",
      "Extracted text is None in file ./data/full\\841fb86dcc9693908d47ca6009269ac412117044\n",
      "Extracted text is None in file ./data/full\\929ad104634a9fc4220516df0e9c313d05fd3598\n",
      "Extracted text is None in file ./data/full\\929ad104634a9fc4220516df0e9c313d05fd3598\n",
      "Extracted text is None in file ./data/full\\95b75ffbd01ef911e231a705d76a183abc6e2c3f\n",
      "Extracted text is None in file ./data/full\\97e3dc704f40edd84b294e4f94bd6c0612cb49d2\n",
      "Extracted text is None in file ./data/full\\9970869dba5c2ad526043f0ff4f39d4c8ae6778a\n",
      "Extracted text is None in file ./data/full\\a1104bbaffa97859f08b9ef46d2b3aa40d876512\n",
      "Extracted text is None in file ./data/full\\a6d70cff1d549d6c8264a1d61c298419c74f3b70\n",
      "Extracted text is None in file ./data/full\\ac1205a756148b032c551befadc420db36ea9215\n",
      "Extracted text is None in file ./data/full\\ac5e28be66bcc365bab818a5b86184f0c1a363eb\n",
      "Extracted text is None in file ./data/full\\ba46e3aceeac7337b1501508d7e69c6b140620aa\n",
      "Extracted text is None in file ./data/full\\bdf4666b5c2fdf8ddb2c782cbed9078533b54b68\n",
      "Extracted text is None in file ./data/full\\cc4fbd3ce5c69c4fe2952213565119ba516989a5\n",
      "Extracted text is None in file ./data/full\\cef8189dfa0100e917b1292e3b370e5a2428afcd\n",
      "Extracted text is None in file ./data/full\\d30f3b852a29b2cd88b0d4c60f232bd20e5e4939\n",
      "Extracted text is None in file ./data/full\\d3d270b8706b67344d73b8d11294c690e47b269b\n",
      "Extracted text is None in file ./data/full\\dc46cf685e948a777026ddc3b0f500dc91084b11\n",
      "Extracted text is None in file ./data/full\\de8f225d60d4583e04c87c96a29dea822f01246c\n",
      "Extracted text is None in file ./data/full\\dfa5ec8befb76b9fae72f082c313ceb12a30a9fa\n",
      "Extracted text is None in file ./data/full\\e52a11b0e2d37f0e705a3562d014783f37d400b8\n"
     ]
    }
   ],
   "source": [
    "# Converting PDFs and append them to texts array\n",
    "path = \"./data/full/\"\n",
    "texts_path = \"./data/texts/\"\n",
    "\n",
    "pathlib.Path(texts_path).mkdir(parents=True, exist_ok=True) \n",
    "files = glob.glob(path + \"*\")\n",
    "\n",
    "for f in files:\n",
    "    f_name = f.split(\"\\\\\")[-1]\n",
    "    txt_path = texts_path + f_name + \".txt\"\n",
    "    \n",
    "    file_path = pathlib.Path(txt_path)\n",
    "    \n",
    "    if file_path.exists():\n",
    "        continue\n",
    "\n",
    "    pdf = pdfplumber.open(f)\n",
    "    text = u''\n",
    "    for page in pdf.pages:\n",
    "        try: # since pdfplumber tries to convert \"P14\" into decimal for some reason\n",
    "            et = page.extract_text()\n",
    "        except:\n",
    "            pass\n",
    "        \n",
    "        if et is not None: # since pdfplumber returns None when an empty page occurs\n",
    "            text += et\n",
    "        else:\n",
    "            print(\"Extracted text is None in file \" + f)\n",
    "\n",
    "    file = open(txt_path, \"wb\")\n",
    "    file.write(text.encode(\"utf-8\", \"ignore\"))\n",
    "    file.close()\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BATUHAN ERDURCAN \\n21301855 \\nTURK 101-13 \\nÖD...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\\nYalçın Arslan \\n21300458 \\nAslı Uçar \\nTURK...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Sudenur SOYSAL \\nGÜZELLİK Mİ ÇİRKİNLİK Mİ HAVU...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Mars’ta Yaşam mı? \\n \\nHayatım boyunca evrende...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Bir Şehrin İki Yakasında \\n \\n\"İki büyük cihan...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text\n",
       "0  BATUHAN ERDURCAN \\n21301855 \\nTURK 101-13 \\nÖD...\n",
       "1   \\nYalçın Arslan \\n21300458 \\nAslı Uçar \\nTURK...\n",
       "2  Sudenur SOYSAL \\nGÜZELLİK Mİ ÇİRKİNLİK Mİ HAVU...\n",
       "3  Mars’ta Yaşam mı? \\n \\nHayatım boyunca evrende...\n",
       "4  Bir Şehrin İki Yakasında \\n \\n\"İki büyük cihan..."
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# read texts\n",
    "texts_path = \"./data/texts/\"\n",
    "text_files = glob.glob(texts_path + \"*\")\n",
    "texts = []\n",
    "for f in text_files:\n",
    "    with open(f, 'rb') as file:\n",
    "        texts.append(file.read().decode(\"utf-8\"))\n",
    "        \n",
    "# Create pandas dataframe for texts\n",
    "df = pd.DataFrame({ \"text\": pd.Series(texts) })\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6844, 1)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Number of writings\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert dataframe to csv\n",
    "df.to_csv(\"data/texts.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}