Sentiment_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from utilities.text_preprocessing import preprocess_sentiment_df\n",
    "from utilities.text_preprocessing import preprocess_other_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "def read_df(class_):\n",
    "    l = [open(filename).read() for filename in glob.glob(\n",
    "        \"./nonmovies/review_polarity/txt_sentoken/{}/*.txt\".format(class_)\n",
    "    )]\n",
    "    df = pd.Series(l)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos = read_df('pos')\n",
    "neg = read_df('neg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = pd.DataFrame(pd.concat([pos, neg]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts.to_csv(\"./nonmovies/review_polarity/whole.csv\", index=False, header=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 20 news commentary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "def read_df():\n",
    "    l = [open(filename, errors='ignore').read() for filename in glob.glob(\n",
    "        \"./nonmovies/20news-18828/*/*\".format()\n",
    "    )]\n",
    "    df = pd.Series(l)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_ = read_df()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_ = all_.apply(lambda red: ' '.join([x for x in red.replace('\\t', ' ').split('\\n') if x and not (x.startswith('From:') or x.startswith('Subject: ') or x.startswith('>') or x.startswith('|>') or x.startswith('In article'))]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_ = pd.DataFrame(all_)\n",
    "all_.columns = ['text']\n",
    "news = preprocess_sentiment_df(all_)\n",
    "news = preprocess_other_dataset(news, './nonmovies/news_e.npy', 5044)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reuters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup,SoupStrainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def read_df():\n",
    "    l = sum([[\n",
    "            ' '.join(x.text.replace('\\t', ' ').split('\\n')) for x in BeautifulSoup(open(filename, errors='ignore').read(), \"lxml\").findAll('text')\n",
    "        ] for filename in glob.glob(\"./nonmovies/reuters21578/reut2-*.sgm\")], [])\n",
    "    df = pd.Series(l)\n",
    "    return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = read_df()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reuters = pd.DataFrame(df)\n",
    "reuters.columns = ['text']\n",
    "reuters = preprocess_sentiment_df(reuters)\n",
    "reuters = preprocess_other_dataset(reuters, './nonmovies/reuters_e.npy', 5044)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get customer data review"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "def read_df():\n",
    "    files = [open(filename).read().split('[t]') for filename in glob.glob(\n",
    "        \"./nonmovies/customer review data/*.txt\".format()\n",
    "    )]\n",
    "    \n",
    "    return files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "files = read_df()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reviews = pd.Series(np.concatenate([f[1:] for f in files]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reviews = reviews.str.extractall('##(.*?)\\\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reviews[0] = ' ' + reviews[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reviews_processed = reviews.reset_index()[['level_0', 0]].groupby('level_0')[0].sum().reset_index()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reviews_processed.to_csv(\"./nonmovies/customer review data/whole.csv\", header=None, index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Symbols used in the annotated reviews:\n",
    "\n",
    "  [t]: the title of the review: Each [t] tag starts a review.\n",
    "       We did not use the title information in our papers.\n",
    "  xxxx[+|-n]: xxxx is a product feature.\n",
    "      [+n]: Positive opinion, n is the opinion strength: 3 strongest,\n",
    "            and 1 weakest. Note that the strength is quite subjective.\n",
    "            You may want ignore it, but only considering + and -\n",
    "      [-n]: Negative opinion\n",
    "  \\##  : start of each sentence. Each line is a sentence.\n",
    "  [u] : feature not appeared in the sentence.\n",
    "  [p] : feature not appeared in the sentence. Pronoun resolution is needed.\n",
    "  [s] : suggestion or recommendation.\n",
    "  [cc]: comparison with a competing product from a different brand.\n",
    "  [cs]: comparison with a competing product from the same brand.\n",
    "'''\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Further preprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "crd = pd.read_csv(\"./nonmovies/customer review data/whole.csv\", index_col=None, header=None)\n",
    "rp = pd.read_csv(\"./nonmovies/review_polarity/whole.csv\", index_col=None, header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rp.columns = ['text']\n",
    "crd.columns = ['text']\n",
    "crd2 = preprocess_sentiment_df(crd)\n",
    "rp2 = preprocess_sentiment_df(rp)\n",
    "crd3 = preprocess_other_dataset(crd2, './nonmovies/crd_e.npy', 5044)\n",
    "rp3 = preprocess_other_dataset(rp2, './nonmovies/rp_e.npy', 5044)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Main dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "\n",
    "def read_df(set_, class_):\n",
    "    l = [open(filename).read() for filename in tqdm(glob.glob(\"./movies/aclImdb/{}/{}/*.txt\".format(set_, class_)))]\n",
    "    df = pd.Series(l)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_pos = read_df('train', 'pos')\n",
    "test_pos = read_df('test', 'pos')\n",
    "train_neg = read_df('train', 'neg')\n",
    "test_neg = read_df('test', 'neg')\n",
    "texts = pd.DataFrame(pd.concat([train_pos, train_neg, test_pos, test_neg]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts.reset_index(drop=True, inplace=True)\n",
    "texts.columns = ['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_ = np.zeros(50000, dtype=np.int)\n",
    "class_[:12500] = 1\n",
    "class_[25000:37500] = 1\n",
    "texts['class'] = class_\n",
    "texts['set'] = 'train'\n",
    "texts['set'][25000:] = 'test'\n",
    "texts.to_csv('./movies/whole.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pass texts through embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from utilities.text_preprocessing import preprocess_sentiment_df\n",
    "from utilities.text_preprocessing import get_occurences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = pd.read_csv('./movies/whole.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = preprocess_sentiment_df(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for_embedding = get_occurences(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for_embedding.shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}