notebooks/hipe2022-datasets-stats.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a5be4e76",
   "metadata": {},
   "source": [
    "# HIPE 2022 dataset statistics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d3a7b60b",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39ac9869",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "from typing import List, Dict\n",
    "from hipe_commons.helpers.tsv import parse_tsv, ENTITY_TYPES, HipeDocument, HipeEntity\n",
    "from hipe_commons.stats import describe_dataset, compute_entities_stats"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e16b1a3",
   "metadata": {},
   "source": [
    "# Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7f403d6",
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "def collect_entities(documents: List[HipeDocument]) -> Dict[str, List[HipeEntity]]:\n",
    "    \"\"\"Simple function to gather all entities from documents in a dataset, divided by type.\n",
    "\n",
    "    :param documents: Input documents in HIPE format\n",
    "    :type documents: List[HipeDocument]\n",
    "    :return: A list of `HipeEntity` objects\n",
    "    :rtype: Dict[str, List[HipeEntity]]\n",
    "    \"\"\"\n",
    "    all_entities = {}\n",
    "\n",
    "    for doc in ajmc_sample_en_docs:\n",
    "        for e_type in ENTITY_TYPES:\n",
    "            \n",
    "            if e_type in doc.entities:\n",
    "\n",
    "                if e_type not in all_entities:\n",
    "                    all_entities[e_type] = []\n",
    "\n",
    "                all_entities[e_type] += doc.entities[e_type]\n",
    "    return all_entities"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "86eee725",
   "metadata": {},
   "source": [
    "# HIPE 2022 Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f6a13c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "RELEASE_VERSION = \"v2.1\"\n",
    "HIPE2022_data_path = f\"../data/{RELEASE_VERSION}/\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3fc6a3e6",
   "metadata": {},
   "source": [
    "## ajmc\n",
    "\n",
    "See the [README file](./documentation/README-ajmc.md) for detailed information about this dataset."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "23ea36b7",
   "metadata": {},
   "source": [
    "### File paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1d7980b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# EN\n",
    "ajmc_train_en_path = os.path.join(HIPE2022_data_path, f\"ajmc/en/HIPE-2022-{RELEASE_VERSION}-ajmc-train-en.tsv\")\n",
    "ajmc_dev_en_path = os.path.join(HIPE2022_data_path, f\"ajmc/en/HIPE-2022-{RELEASE_VERSION}-ajmc-dev-en.tsv\")\n",
    "ajmc_test_en_path = os.path.join(HIPE2022_data_path, f\"ajmc/en/HIPE-2022-{RELEASE_VERSION}-ajmc-test-en.tsv\")\n",
    "\n",
    "# DE\n",
    "ajmc_train_de_path = os.path.join(HIPE2022_data_path, f\"ajmc/de/HIPE-2022-{RELEASE_VERSION}-ajmc-train-de.tsv\")\n",
    "ajmc_dev_de_path = os.path.join(HIPE2022_data_path, f\"ajmc/de/HIPE-2022-{RELEASE_VERSION}-ajmc-dev-de.tsv\")\n",
    "ajmc_test_de_path = os.path.join(HIPE2022_data_path, f\"ajmc/de/HIPE-2022-{RELEASE_VERSION}-ajmc-test-de.tsv\")\n",
    "\n",
    "# FR\n",
    "ajmc_train_fr_path = os.path.join(HIPE2022_data_path, f\"ajmc/fr/HIPE-2022-{RELEASE_VERSION}-ajmc-train-fr.tsv\")\n",
    "ajmc_dev_fr_path = os.path.join(HIPE2022_data_path, f\"ajmc/fr/HIPE-2022-{RELEASE_VERSION}-ajmc-dev-fr.tsv\")\n",
    "ajmc_test_fr_path = os.path.join(HIPE2022_data_path, f\"ajmc/fr/HIPE-2022-{RELEASE_VERSION}-ajmc-test-fr.tsv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98069843",
   "metadata": {},
   "source": [
    "### ajmc EN"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6fe36fe5",
   "metadata": {},
   "source": [
    "#### Dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d7d48341",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "ajmc_dev_en_docs = parse_tsv(file_path=ajmc_dev_en_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4d2e96e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=ajmc_dev_en_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cae8b1f1",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c59de36",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "ajmc_train_en_docs = parse_tsv(file_path=ajmc_train_en_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "27ed577b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=ajmc_train_en_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ddd117b3",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c60c253",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "ajmc_test_en_docs = parse_tsv(file_path=ajmc_test_en_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9126e02",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=ajmc_test_en_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85a78a1a",
   "metadata": {},
   "source": [
    "### ajmc DE"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2c54bf0",
   "metadata": {},
   "source": [
    "#### Dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e10a9e0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "ajmc_dev_de_docs = parse_tsv(file_path=ajmc_dev_de_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b3fd754",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=ajmc_dev_de_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67c3f97c",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8da40406",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "ajmc_train_de_docs = parse_tsv(file_path=ajmc_train_de_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af8ae6f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=ajmc_train_de_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71fe8a79",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0c87457",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "ajmc_test_de_docs = parse_tsv(file_path=ajmc_test_de_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f42c1222",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=ajmc_test_de_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f498af21",
   "metadata": {},
   "source": [
    "### ajmc FR"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e2eb3ec",
   "metadata": {},
   "source": [
    "#### Dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc683514",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "ajmc_dev_fr_docs = parse_tsv(file_path=ajmc_dev_fr_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0644243e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=ajmc_dev_fr_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2020856",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de15545d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "ajmc_train_fr_docs = parse_tsv(file_path=ajmc_train_fr_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "786a76a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=ajmc_train_fr_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1e143ac9",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e0e0e7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "ajmc_test_fr_docs = parse_tsv(file_path=ajmc_test_fr_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20ecae70",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=ajmc_test_fr_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d2a85c1",
   "metadata": {},
   "source": [
    "## hipe2020\n",
    "\n",
    "See the [README file](./documentation/README-hipe2020.md) for detailed information about this dataset."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dbd8cb25",
   "metadata": {},
   "source": [
    "### File paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f992992b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# EN\n",
    "hipe2020_dev_en_path = os.path.join(HIPE2022_data_path, f\"hipe2020/en/HIPE-2022-{RELEASE_VERSION}-hipe2020-dev-en.tsv\")\n",
    "hipe2020_test_en_path = os.path.join(HIPE2022_data_path, f\"hipe2020/en/HIPE-2022-{RELEASE_VERSION}-hipe2020-test-en.tsv\")\n",
    "\n",
    "# DE\n",
    "hipe2020_train_de_path = os.path.join(HIPE2022_data_path, f\"hipe2020/de/HIPE-2022-{RELEASE_VERSION}-hipe2020-train-de.tsv\")\n",
    "hipe2020_dev_de_path = os.path.join(HIPE2022_data_path, f\"hipe2020/de/HIPE-2022-{RELEASE_VERSION}-hipe2020-dev-de.tsv\")\n",
    "hipe2020_test_de_path = os.path.join(HIPE2022_data_path, f\"hipe2020/de/HIPE-2022-{RELEASE_VERSION}-hipe2020-test-de.tsv\")\n",
    "\n",
    "\n",
    "# FR\n",
    "hipe2020_dev_fr_path = os.path.join(HIPE2022_data_path, f\"hipe2020/fr/HIPE-2022-{RELEASE_VERSION}-hipe2020-dev-fr.tsv\")\n",
    "hipe2020_train_fr_path = os.path.join(HIPE2022_data_path, f\"hipe2020/fr/HIPE-2022-{RELEASE_VERSION}-hipe2020-train-fr.tsv\")\n",
    "hipe2020_test_fr_path = os.path.join(HIPE2022_data_path, f\"hipe2020/fr/HIPE-2022-{RELEASE_VERSION}-hipe2020-test-fr.tsv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d13fb06",
   "metadata": {},
   "source": [
    "### hipe2020 EN"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6037598d",
   "metadata": {},
   "source": [
    "#### Dev\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5980486f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "hipe2020_dev_en_docs = parse_tsv(file_path=hipe2020_dev_en_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1ebf584",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=hipe2020_dev_en_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8f5ffdc",
   "metadata": {},
   "source": [
    "#### Test\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09095db0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "hipe2020_test_en_docs = parse_tsv(file_path=hipe2020_test_en_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5bc2a16",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=hipe2020_test_en_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "374f4ccb",
   "metadata": {},
   "source": [
    "### hipe2020 DE"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5d064d1",
   "metadata": {},
   "source": [
    "#### Dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee918688",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "hipe2020_dev_de_docs = parse_tsv(file_path=hipe2020_dev_de_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca1b34aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=hipe2020_dev_de_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d9066d1a",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3f07e60",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "hipe2020_train_de_docs = parse_tsv(file_path=hipe2020_train_de_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "681ff82c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=hipe2020_train_de_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14039039",
   "metadata": {},
   "source": [
    "#### Test\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb729ef9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "hipe2020_test_de_docs = parse_tsv(file_path=hipe2020_test_de_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc7c8dec",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=hipe2020_test_de_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d98272e",
   "metadata": {},
   "source": [
    "### hipe2020 FR"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "070880bf",
   "metadata": {},
   "source": [
    "#### Dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb508bb1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "hipe2020_dev_fr_docs = parse_tsv(file_path=hipe2020_dev_fr_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1eac079a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=hipe2020_dev_fr_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8dbd691",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e21ac6f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "hipe2020_train_fr_docs = parse_tsv(file_path=hipe2020_train_fr_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5824fc8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=hipe2020_train_fr_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc5bd52d",
   "metadata": {},
   "source": [
    "#### Test\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff3b0737",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "hipe2020_test_fr_docs = parse_tsv(file_path=hipe2020_test_fr_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d7da663f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=hipe2020_test_fr_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0b06c25",
   "metadata": {},
   "source": [
    "## letemps\n",
    "\n",
    "See the [README file](./documentation/README-letemps.md) for detailed information about this dataset."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2dcea5cf",
   "metadata": {},
   "source": [
    "### File paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "61dec1da",
   "metadata": {},
   "outputs": [],
   "source": [
    "letemps_dev_fr_path = os.path.join(HIPE2022_data_path, f\"letemps/fr/HIPE-2022-{RELEASE_VERSION}-letemps-dev-fr.tsv\")\n",
    "letemps_train_fr_path = os.path.join(HIPE2022_data_path, f\"letemps/fr/HIPE-2022-{RELEASE_VERSION}-letemps-train-fr.tsv\")\n",
    "letemps_test_fr_path = os.path.join(HIPE2022_data_path, f\"letemps/fr/HIPE-2022-{RELEASE_VERSION}-letemps-test-fr.tsv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b0537e0",
   "metadata": {},
   "source": [
    "#### Dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ceedad6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "letemps_dev_fr_docs = parse_tsv(file_path=letemps_dev_fr_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cae0932b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=letemps_dev_fr_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64a305b2",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc6d3eee",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "letemps_train_fr_docs = parse_tsv(file_path=letemps_train_fr_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "867d6f04",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=letemps_train_fr_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3b23248f",
   "metadata": {},
   "source": [
    "#### Dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bafec598",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "letemps_test_fr_docs = parse_tsv(file_path=letemps_test_fr_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91bc22b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=letemps_test_fr_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed696674",
   "metadata": {},
   "source": [
    "## topRes19th\n",
    "\n",
    "See the [README file](./documentation/README-topres19th.md) for detailed information about this dataset."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01e062c6",
   "metadata": {},
   "source": [
    "### File paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51fa12eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "topRes19th_dev_en_path = os.path.join(HIPE2022_data_path, f\"topres19th/en/HIPE-2022-{RELEASE_VERSION}-topres19th-dev-en.tsv\")\n",
    "topRes19th_train_en_path = os.path.join(HIPE2022_data_path, f\"topres19th/en/HIPE-2022-{RELEASE_VERSION}-topres19th-train-en.tsv\")\n",
    "topRes19th_test_en_path = os.path.join(HIPE2022_data_path, f\"topres19th/en/HIPE-2022-{RELEASE_VERSION}-topres19th-test-en.tsv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "563e29f2",
   "metadata": {},
   "source": [
    "#### Dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2457dbf9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "topRes19th_dev_en_docs = parse_tsv(file_path=topRes19th_dev_en_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1fa7ece",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=topRes19th_dev_en_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4784aff3",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8094673a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "topRes19th_train_en_docs = parse_tsv(file_path=topRes19th_train_en_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc46b0f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=topRes19th_train_en_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d90cc1a2",
   "metadata": {},
   "source": [
    "#### Test\n",
    "\n",
    "(lines below are to be commented out once the test files for topRes are published)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f46d26bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "#topRes19th_test_en_docs = parse_tsv(file_path=topRes19th_test_en_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00abdc89",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "#print(describe_dataset(documents=topRes19th_dev_en_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9abfd336",
   "metadata": {},
   "source": [
    "## newseye\n",
    "\n",
    "See the [README file](./documentation/README-newseye.md) for detailed information about this dataset."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "86321d90",
   "metadata": {},
   "source": [
    "### File paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24c4cf39",
   "metadata": {},
   "outputs": [],
   "source": [
    "# FR\n",
    "newseye_dev_fr_path = os.path.join(HIPE2022_data_path, f\"newseye/fr/HIPE-2022-{RELEASE_VERSION}-newseye-dev-fr.tsv\")\n",
    "newseye_train_fr_path = os.path.join(HIPE2022_data_path, f\"newseye/fr/HIPE-2022-{RELEASE_VERSION}-newseye-train-fr.tsv\")\n",
    "newseye_test_fr_path = os.path.join(HIPE2022_data_path, f\"newseye/fr/HIPE-2022-{RELEASE_VERSION}-newseye-test-fr.tsv\")\n",
    "\n",
    "# DE\n",
    "newseye_dev_de_path = os.path.join(HIPE2022_data_path, f\"newseye/de/HIPE-2022-{RELEASE_VERSION}-newseye-dev-de.tsv\")\n",
    "newseye_train_de_path = os.path.join(HIPE2022_data_path, f\"newseye/de/HIPE-2022-{RELEASE_VERSION}-newseye-train-de.tsv\")\n",
    "newseye_test_de_path = os.path.join(HIPE2022_data_path, f\"newseye/de/HIPE-2022-{RELEASE_VERSION}-newseye-test-de.tsv\")\n",
    "\n",
    "# FI\n",
    "newseye_dev_fi_path = os.path.join(HIPE2022_data_path, f\"newseye/fi/HIPE-2022-{RELEASE_VERSION}-newseye-dev-fi.tsv\")\n",
    "newseye_train_fi_path = os.path.join(HIPE2022_data_path, f\"newseye/fi/HIPE-2022-{RELEASE_VERSION}-newseye-train-fi.tsv\")\n",
    "newseye_test_fi_path = os.path.join(HIPE2022_data_path, f\"newseye/fi/HIPE-2022-{RELEASE_VERSION}-newseye-test-fi.tsv\")\n",
    "\n",
    "# SV\n",
    "newseye_dev_sv_path = os.path.join(HIPE2022_data_path, f\"newseye/sv/HIPE-2022-{RELEASE_VERSION}-newseye-dev-sv.tsv\")\n",
    "newseye_train_sv_path = os.path.join(HIPE2022_data_path, f\"newseye/sv/HIPE-2022-{RELEASE_VERSION}-newseye-train-sv.tsv\")\n",
    "newseye_test_sv_path = os.path.join(HIPE2022_data_path, f\"newseye/sv/HIPE-2022-{RELEASE_VERSION}-newseye-test-sv.tsv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "289ca720",
   "metadata": {},
   "source": [
    "### newseye FR"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "acd2b525",
   "metadata": {},
   "source": [
    "#### Dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35e7be47",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "newseye_dev_fr_docs = parse_tsv(file_path=newseye_dev_fr_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0093f53",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=newseye_dev_fr_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f527b0d3",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "541b4c65",
   "metadata": {},
   "outputs": [],
   "source": [
    "newseye_train_fr_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b4cd97a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "newseye_train_fr_docs = parse_tsv(file_path=newseye_train_fr_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4cdc73ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=newseye_train_fr_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8e67dd9d",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a41bd5ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "newseye_test_fr_docs = parse_tsv(file_path=newseye_test_fr_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42ab4bd5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=newseye_test_fr_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "47d69e44",
   "metadata": {},
   "source": [
    "### newseye DE"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8398af96",
   "metadata": {},
   "source": [
    "#### Dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b6ef18f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "newseye_dev_de_docs = parse_tsv(file_path=newseye_dev_de_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3da94913",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=newseye_dev_de_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c73fbcb",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0c65881",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "newseye_train_de_docs = parse_tsv(file_path=newseye_train_de_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "43791e9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=newseye_train_de_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b9b6ce3a",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e59d42ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "newseye_test_de_docs = parse_tsv(file_path=newseye_test_de_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49cacfd5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=newseye_test_de_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "872b6a68",
   "metadata": {},
   "source": [
    "### newseye FI"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a10da0f",
   "metadata": {},
   "source": [
    "#### Dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a6550817",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "newseye_dev_fi_docs = parse_tsv(file_path=newseye_dev_fi_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ef3872d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=newseye_dev_fi_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "51e3615d",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d183a069",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "newseye_train_fi_docs = parse_tsv(file_path=newseye_train_fi_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06e61ccb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=newseye_train_fi_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8585309d",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d4e29a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "newseye_test_fi_docs = parse_tsv(file_path=newseye_test_fi_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d37bb572",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=newseye_test_fi_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74083291",
   "metadata": {},
   "source": [
    "### newseye SV"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b60fcdb9",
   "metadata": {},
   "source": [
    "#### Dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "266f8bee",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "newseye_dev_sv_docs = parse_tsv(file_path=newseye_dev_sv_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42c4a49a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=newseye_dev_sv_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "92def368",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc962b4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "newseye_train_sv_docs = parse_tsv(file_path=newseye_train_sv_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6f6f2e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=newseye_train_sv_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89d1ad9a",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40406c49",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "newseye_test_sv_docs = parse_tsv(file_path=newseye_test_sv_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "018bf812",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=newseye_test_sv_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "170e049d",
   "metadata": {},
   "source": [
    "## sonar\n",
    "\n",
    "See the [README file](./documentation/README-sonar.md) for detailed information about this dataset."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a688da4",
   "metadata": {},
   "source": [
    "### File paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "263a7947",
   "metadata": {},
   "outputs": [],
   "source": [
    "sonar_dev_de_path = os.path.join(HIPE2022_data_path, f\"sonar/de/HIPE-2022-{RELEASE_VERSION}-sonar-dev-de.tsv\")\n",
    "sonar_test_de_path = os.path.join(HIPE2022_data_path, f\"sonar/de/HIPE-2022-{RELEASE_VERSION}-sonar-test-de.tsv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "65df2a7a",
   "metadata": {},
   "source": [
    "#### Dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5ac5fcb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "sonar_dev_de_docs = parse_tsv(file_path=sonar_dev_de_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fba9bf96",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=sonar_dev_de_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5c90f0d",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9081feb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "sonar_test_de_docs = parse_tsv(file_path=sonar_test_de_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b84855f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=sonar_test_de_docs))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}