examples/am_literals.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from rrgcn import RRGCNEmbedder\n",
    "from torch_geometric.datasets import Entities\n",
    "import torch\n",
    "from catboost import CatBoostClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report\n",
    "import rdflib\n",
    "import re\n",
    "from tqdm import tqdm\n",
    "import math\n",
    "\n",
    "# !pip install transformers\n",
    "from transformers import AutoTokenizer, AutoModelForMaskedLM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "## edited entities.py from PyG that also saves node_features\n",
    "import logging\n",
    "import os\n",
    "import os.path as osp\n",
    "from collections import Counter\n",
    "from typing import Callable, List, Optional\n",
    "\n",
    "import numpy as np\n",
    "import torch\n",
    "\n",
    "from torch_geometric.data import (\n",
    "    Data,\n",
    "    InMemoryDataset,\n",
    "    download_url,\n",
    "    extract_tar,\n",
    ")\n",
    "\n",
    "\n",
    "class Entities(InMemoryDataset):\n",
    "    url = 'https://data.dgl.ai/dataset/{}.tgz'\n",
    "\n",
    "    def __init__(self, root: str, name: str,\n",
    "                 transform: Optional[Callable] = None,\n",
    "                 pre_transform: Optional[Callable] = None):\n",
    "        self.name = name.lower()\n",
    "        assert self.name in ['aifb', 'am', 'mutag', 'bgs']\n",
    "        super().__init__(root, transform, pre_transform)\n",
    "        self.data, self.slices = torch.load(self.processed_paths[0])\n",
    "\n",
    "    @property\n",
    "    def raw_dir(self) -> str:\n",
    "        return osp.join(self.root, self.name, 'raw')\n",
    "\n",
    "    @property\n",
    "    def processed_dir(self) -> str:\n",
    "        return osp.join(self.root, self.name, 'processed')\n",
    "\n",
    "    @property\n",
    "    def num_relations(self) -> int:\n",
    "        return self.data.edge_type.max().item() + 1\n",
    "\n",
    "    @property\n",
    "    def num_classes(self) -> int:\n",
    "        return self.data.train_y.max().item() + 1\n",
    "\n",
    "    @property\n",
    "    def raw_file_names(self) -> List[str]:\n",
    "        return [\n",
    "            f'{self.name}_stripped.nt.gz',\n",
    "            'completeDataset.tsv',\n",
    "            'trainingSet.tsv',\n",
    "            'testSet.tsv',\n",
    "        ]\n",
    "\n",
    "    @property\n",
    "    def processed_file_names(self) -> str:\n",
    "        return 'data.pt'\n",
    "\n",
    "    def download(self):\n",
    "        path = download_url(self.url.format(self.name), self.root)\n",
    "        extract_tar(path, self.raw_dir)\n",
    "        os.unlink(path)\n",
    "\n",
    "    def process(self):\n",
    "        import gzip\n",
    "\n",
    "        import pandas as pd\n",
    "        import rdflib as rdf\n",
    "\n",
    "        graph_file, task_file, train_file, test_file = self.raw_paths\n",
    "\n",
    "        with hide_stdout():\n",
    "            g = rdf.Graph()\n",
    "            with gzip.open(graph_file, 'rb') as f:\n",
    "                g.parse(file=f, format='nt')\n",
    "\n",
    "        freq = Counter(g.predicates())\n",
    "\n",
    "        relations = sorted(set(g.predicates()), key=lambda p: -freq.get(p, 0))\n",
    "        subjects = set(g.subjects())\n",
    "        objects = set(g.objects())\n",
    "        nodes = list(subjects.union(objects))\n",
    "\n",
    "        N = len(nodes)\n",
    "        R = 2 * len(relations)\n",
    "\n",
    "        relations_dict = {rel: i for i, rel in enumerate(relations)}\n",
    "        nodes_dict = {node: i for i, node in enumerate(nodes)}\n",
    "\n",
    "        edges = []\n",
    "        node_features = {}\n",
    "        for s, p, o in g.triples((None, None, None)):\n",
    "            src, dst, rel = nodes_dict[s], nodes_dict[o], relations_dict[p]\n",
    "            edges.append([src, dst, 2 * rel])\n",
    "            edges.append([dst, src, 2 * rel + 1])\n",
    "\n",
    "            # SAVE LITERAL VALUES\n",
    "            if isinstance(o, rdf.Literal):\n",
    "                literal_type = p#type(o.value)\n",
    "                if literal_type not in node_features:\n",
    "                    node_features[literal_type] = [[dst], [o.value]]\n",
    "                else:\n",
    "                    node_features[literal_type][0].append(dst)\n",
    "                    node_features[literal_type][1].append(o.value)\n",
    "\n",
    "        edges = torch.tensor(edges, dtype=torch.long).t().contiguous()\n",
    "        perm = (N * R * edges[0] + R * edges[1] + edges[2]).argsort()\n",
    "        edges = edges[:, perm]\n",
    "\n",
    "        edge_index, edge_type = edges[:2], edges[2]\n",
    "\n",
    "        if self.name == 'am':\n",
    "            label_header = 'label_cateogory'\n",
    "            nodes_header = 'proxy'\n",
    "        elif self.name == 'aifb':\n",
    "            label_header = 'label_affiliation'\n",
    "            nodes_header = 'person'\n",
    "        elif self.name == 'mutag':\n",
    "            label_header = 'label_mutagenic'\n",
    "            nodes_header = 'bond'\n",
    "        elif self.name == 'bgs':\n",
    "            label_header = 'label_lithogenesis'\n",
    "            nodes_header = 'rock'\n",
    "\n",
    "        labels_df = pd.read_csv(task_file, sep='\\t')\n",
    "        labels_set = set(labels_df[label_header].values.tolist())\n",
    "        labels_dict = {lab: i for i, lab in enumerate(list(labels_set))}\n",
    "        nodes_dict = {np.unicode(key): val for key, val in nodes_dict.items()}\n",
    "\n",
    "        train_labels_df = pd.read_csv(train_file, sep='\\t')\n",
    "        train_indices, train_labels = [], []\n",
    "        for nod, lab in zip(train_labels_df[nodes_header].values,\n",
    "                            train_labels_df[label_header].values):\n",
    "            train_indices.append(nodes_dict[nod])\n",
    "            train_labels.append(labels_dict[lab])\n",
    "\n",
    "        train_idx = torch.tensor(train_indices, dtype=torch.long)\n",
    "        train_y = torch.tensor(train_labels, dtype=torch.long)\n",
    "\n",
    "        test_labels_df = pd.read_csv(test_file, sep='\\t')\n",
    "        test_indices, test_labels = [], []\n",
    "        for nod, lab in zip(test_labels_df[nodes_header].values,\n",
    "                            test_labels_df[label_header].values):\n",
    "            test_indices.append(nodes_dict[nod])\n",
    "            test_labels.append(labels_dict[lab])\n",
    "\n",
    "        test_idx = torch.tensor(test_indices, dtype=torch.long)\n",
    "        test_y = torch.tensor(test_labels, dtype=torch.long)\n",
    "\n",
    "        data = Data(edge_index=edge_index, edge_type=edge_type,\n",
    "                    train_idx=train_idx, train_y=train_y, test_idx=test_idx,\n",
    "                    test_y=test_y, num_nodes=N, node_features=node_features)\n",
    "\n",
    "        torch.save(self.collate([data]), self.processed_paths[0])\n",
    "\n",
    "    def __repr__(self) -> str:\n",
    "        return f'{self.name.upper()}{self.__class__.__name__}()'\n",
    "\n",
    "\n",
    "class hide_stdout(object):\n",
    "    def __enter__(self):\n",
    "        self.level = logging.getLogger().level\n",
    "        logging.getLogger().setLevel(logging.ERROR)\n",
    "\n",
    "    def __exit__(self, *args):\n",
    "        logging.getLogger().setLevel(self.level)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "dataset = Entities('./', 'am')\n",
    "data = dataset[0].to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "http://purl.org/collections/nl/am/dimensionPart\n",
      "['passe-partout', 'geheel', 'b-maat', 'a-maat', 'c-maat', 'c-maat', 'a-maat', 'a-maat', 'KA 12834.1', 'a']\n",
      "num entries 6028 - unique entries 577\n",
      "--\n",
      "http://purl.org/collections/nl/am/reproductionReference\n",
      "['KA_6989.JPG', 'A_10069_000.jpg', 'KA_14018.JPG', 'A_12399_000.jpg', 'O 2577.38', 'A_19506', 'O 11540.09', 'O 1320.34', 'A_50466', 'A_44633']\n",
      "num entries 110861 - unique entries 104142\n",
      "--\n",
      "http://www.w3.org/2000/01/rdf-schema#label\n",
      "['WH 1ste verdieping 2003-01-23 2003-08-22 intern 15111 Transportnummer:2003-0066 intern ', 'CBD 3 kast 84-3 1975-01-01   17471   ', ' Portretten van Nederlandse beeldende kunstenaars : repertorium [R] p. 132 ', 'Een en ander over glasgravure p. 276, t.o. p. 276 afb. ', 'AHM dep 3 blok 2-4 1975-01-01   16378   ', 'AHM textielrestauratie 2006-10-24 2008-09-03  16283   ', 'WH Palet Willet boudoir 2001-11-04 2001-08-28  26292   ', 'AHM PK stelling 27-14 II 2009-03-21   14631 doos nr 251 intern ', ' Geschiedenis van Amsterdam dl. III, p. 454 ', 'breedte 86.5 cm    ']\n",
      "num entries 341964 - unique entries 68566\n",
      "--\n",
      "http://purl.org/collections/nl/am/name\n",
      "['Stichting Kerkelijk Kunstbezit in Nederland', 'Maison Descartes', 'Janssen, Winfried', 'Velsen, Tadeus van', 'Mirto, Alfonso', 'Sotheby Mak van Waay, grote zaal', \"Cliché's van Leer & Co\", 'Linnet, Ragni', 'Landwerh, John', 'Siemensma, Ferry']\n",
      "num entries 66937 - unique entries 66937\n",
      "--\n",
      "http://purl.org/collections/nl/am/dimensionUnit\n",
      "['cm', 'cm', 'cm', 'cm', 'cm', 'cm', 'cm', 'cm', 'cm', 'cm']\n",
      "num entries 152555 - unique entries 13\n",
      "--\n",
      "http://purl.org/collections/nl/am/currentLocationLref\n",
      "['32216', '17313', '28887', '28928', '16415', '15376', '22153', '16644', '32226', '15127']\n",
      "num entries 137474 - unique entries 5109\n",
      "--\n",
      "http://purl.org/collections/nl/am/currentLocationType\n",
      "['intern', 'extern', 'intern', 'intern', 'intern', 'intern', 'extern', 'intern', 'binnenkomend', 'intern']\n",
      "num entries 63542 - unique entries 22\n",
      "--\n",
      "http://purl.org/collections/nl/am/reproductionDate\n",
      "['1998-03-16', '1997-06-05', '2009-12-01', '2009-04', '2004-10-20', '2004-03-22', '2004-10-13', '1999', '1999', '1999']\n",
      "num entries 73697 - unique entries 1269\n",
      "--\n",
      "http://purl.org/collections/nl/am/currentLocationNotes\n",
      "['doos 5, boeknr. 81', 'doos 31, boeknr. 343', 'transportformulier 2008-0032', 'doos 11a', 'doos nr 109', 'doos nr 200', 'Transportnummer:2005-0245', 'doos 17, boeknr. 346', 'doos 33, boeknr. 321', 'doos nr 160']\n",
      "num entries 53819 - unique entries 4797\n",
      "--\n",
      "http://purl.org/collections/nl/am/objectNumber\n",
      "['KA 13641', 'A 15826', 'KB 2122.7', 'TA 36547', 'KA 18210', 'DA 19.142', 'A 16256', 'KA 19183', 'PB 99', 'KA 18683']\n",
      "num entries 73448 - unique entries 73448\n",
      "--\n",
      "http://purl.org/collections/nl/am/priref\n",
      "['24940', '12843', '39315', '33995', '12711', '843', '59058', '4256', '29404', '27437']\n",
      "num entries 167399 - unique entries 75094\n",
      "--\n",
      "http://purl.org/collections/nl/am/reproductionReferenceLref\n",
      "['62510', '51562', '133951', '134204', '8884', '44329', '70500', '38746', '40582', '129574']\n",
      "num entries 114453 - unique entries 107380\n",
      "--\n",
      "http://purl.org/collections/nl/am/documentationTitle\n",
      "['Amsterdamer Schützenbildnis: Teil 5. Katalog der frühen amsterdamer Schützenbildnissse (1529-1566)', 'glazen bekers, naar de volgorde waarin zij geplaatst zijn in de vitrine', 'Amsterdams Historisch Museum : schilderijen daterend van voor 1800 : voorlopige catalogus', 'Beschrijving der schilderijen, teekeningen, prenten, prentwerken en boeken, in het Museum Fodor, te Amsterdam', 'werk van Jan en Casper Luyken', 'Frederik Sieger', 'Andriessen-kamer in de collectie van het Amsterdams Historisch Museum', 'Amsterdam in de schilderkunst', 'Vier eeuwen Herengracht : geveltekeningen van alle huizen aan de gracht, twee historische overzichten en de beschrijving van elk pand met zijn eigenaars en bewoners', 'werk van Jan en Casper Luyken']\n",
      "num entries 38416 - unique entries 5594\n",
      "--\n",
      "http://purl.org/collections/nl/am/title\n",
      "['Bak', 'Japonlijfje', 'Het opzetten van den tabernakel, De gouden kandelaar, De heerlykheyd des heeren, Viering van het loofhuttenfeest', 'Schutters van de compagnie van kapitein Adriaen Pietersz. Raep en Dirck Hasselaer', 'Historien Onses Tyds, Behelzende Saken Van Staat En Oorlogh, Voorgevallen in, en omtrent de Vereenigde Nederlanden, En door geheel Europa, Mitsgaders in meest alle de andere Deelen des Werelds. Beginnende met het Jaar 1669. daar het De Heer Lieuwe van Aitzema heeft gelaten; en eyndigende na het Jaar 1679. met het sluyten der Nimeegsche en Algemeene Vrede Door den Heer L. Sylvius. Verçiert met schoone Konst-plaaten', 'Schoenendoos (van Barbiepop)', 'Tekstpagina met opdracht uit \"Lust-hof des gemoets\"', 'Mythologische figuur musicerend', 'Namiddagjapon met sjerp', \"Stempelafdruk 'Mairie de la ville d'amsterdam'\"]\n",
      "num entries 75883 - unique entries 45768\n",
      "--\n",
      "http://www.w3.org/2004/02/skos/core#prefLabel\n",
      "['diameter handvat', 'CBD 3 stelling 3-10', 'Paradiso', 'wapenaccessoire', 'demonstratie tegen kruisraketten', 'hoogte blad', 'renteboekje', 'waardepapier', 'Vesoul', 'miniaturen']\n",
      "num entries 28048 - unique entries 28048\n",
      "--\n",
      "http://purl.org/collections/nl/am/currentLocationDateStart\n",
      "['2003-06', '1997', '1975-01-01', '1975-01-01', '2009-03-21', '2009-01-05', '2001-01-30', '1975-01-01', '1975-01-01', '1975-01-01']\n",
      "num entries 131997 - unique entries 3099\n",
      "--\n",
      "http://purl.org/collections/nl/am/dimensionValue\n",
      "['14.5', '16.11', '4.1', '10', '37.2', '7.5', '23', '5.9', '23.06', '10.5']\n",
      "num entries 151734 - unique entries 7951\n",
      "--\n",
      "http://purl.org/collections/nl/am/documentationSortyear\n",
      "['2008', '1996', '2002', '1995', '1928', '1881', '1845', '1876', '1914', '1968']\n",
      "num entries 36129 - unique entries 282\n",
      "--\n",
      "http://purl.org/collections/nl/am/productionDateStart\n",
      "['1857', '1917', '1708', '1692', '1732', '1750', '1793', '1782', '1700', '1700']\n",
      "num entries 58055 - unique entries 655\n",
      "--\n",
      "http://purl.org/collections/nl/am/selected\n",
      "['False', 'False', 'False', 'False', 'False', 'False', 'False', 'False', 'False', 'False']\n",
      "num entries 66966 - unique entries 1\n",
      "--\n",
      "http://purl.org/collections/nl/am/AHMTextsAuthorLref\n",
      "['70', '15', '5', '142', '5', '47', '5', '15', '15', '30']\n",
      "num entries 4411 - unique entries 53\n",
      "--\n",
      "http://purl.org/collections/nl/am/reproductionIdentifierURL\n",
      "['..\\\\..\\\\DAT\\\\collectie\\\\images\\\\PB_808.jpg', '..\\\\..\\\\DAT\\\\collectie\\\\images\\\\PA_324.JPG', '..\\\\..\\\\DAT\\\\collectie\\\\images\\\\KB_895.JPG', '..\\\\..\\\\DAT\\\\collectie\\\\images\\\\KA_756.JPG', '..\\\\..\\\\DAT\\\\collectie\\\\images\\\\KA_5543.JPG', '..\\\\..\\\\dat\\\\collectie\\\\images\\\\pictura2009\\\\S_A_16666_000.jpg', '..\\\\..\\\\DAT\\\\collectie\\\\images\\\\LUYKEN\\\\A_19305_1.JPG', '..\\\\..\\\\DAT\\\\collectie\\\\images\\\\KA_10360.JPG', '..\\\\..\\\\DAT\\\\collectie\\\\images\\\\LA_361_F.jpg', '..\\\\..\\\\DAT\\\\collectie\\\\images\\\\LUYKEN\\\\A_44835.JPG']\n",
      "num entries 68178 - unique entries 64260\n",
      "--\n",
      "http://purl.org/collections/nl/am/creatorRole\n",
      "['goud- en zilversmid', 'beeldhouwer', 'etser', 'etser', 'uitgever', 'etser', 'goud- en zilversmid', 'goud- en zilversmid', 'auteur', 'uitgever']\n",
      "num entries 45772 - unique entries 149\n",
      "--\n",
      "http://purl.org/collections/nl/am/exhibitionDateEnd\n",
      "['2005-03-13', '2003-01-12', '1989-04-23', '2001-01-28', '1987-01-04', '2008-11-24', '2003-01-26', '2003-08-17', '1991-01-06', '2005-01-23']\n",
      "num entries 9506 - unique entries 923\n",
      "--\n",
      "http://purl.org/collections/nl/am/productionDateEnd\n",
      "['1968', '1812', '1664', '1801', '1849', '1703', '1855', '1817', '1899', '1598']\n",
      "num entries 55886 - unique entries 653\n",
      "--\n",
      "http://purl.org/collections/nl/am/documentationPageReference\n",
      "['[p. 22]', 'p. 72-75 nr. 1, afb. p. 74', 'p. 36 nr 46', 'p. 98, 103, afb. 33', 'p. 199 nr. 3369', 'dl. 2 p. 29', 'p. 12 nr. 28', 'p. 233 afb. 1, p. 234 afb. 3', '[X] p. 11 nr. 106', '[R[ p. 145 nr. 880']\n",
      "num entries 38424 - unique entries 4645\n",
      "--\n",
      "http://purl.org/collections/nl/am/currentLocationDateEnd\n",
      "['2009-03-21', '2006-01-27', '2009-03-21', '2004-05-31', '2006-10-09', '1999', '2009-03-21', '2009-03-21', '2007-05-22', '2009-03-21']\n",
      "num entries 62446 - unique entries 2180\n",
      "--\n",
      "http://purl.org/collections/nl/am/AHMTextsPubl\n",
      "['2007-10-08', '2007-10-31', '2008-03-17', '2008-03-06', '2007-09-26', '2007-11-26', '2009-02-20', '2008-03-18', '2008-07-06', '2007-10-15']\n",
      "num entries 2275 - unique entries 84\n",
      "--\n",
      "http://purl.org/collections/nl/am/dimensionNotes\n",
      "['incl. minimale marge', 'incl. minimale marge', 'incl. minimale marge', 'incl. minimale marge', 'incl. minimale marge', 'met 25 kralen', 'incl. minimale marge', 'incl. minimale marge', 'incl. minimale marge', 'incl. scan marge']\n",
      "num entries 28665 - unique entries 779\n",
      "--\n",
      "http://purl.org/collections/nl/am/exhibitionLref\n",
      "['66', '6041', '5553', '4624', '4615', '4605', '5707', '5148', '466', '307']\n",
      "num entries 10617 - unique entries 1164\n",
      "--\n",
      "http://purl.org/collections/nl/am/alternativeNumberType\n",
      "['vE/vdK prent', 'oud inv.nr', 'vE/vdK prent', 'vE/vdK prent', 'oud inv.no.', 'vE/vdK prent', 'catalogus', 'vE/vdK prent', 'vE/vdK prent', 'vE/vdK prent']\n",
      "num entries 21313 - unique entries 113\n",
      "--\n",
      "http://purl.org/collections/nl/am/documentationShelfmark\n",
      "['O.b/Thie.', 'Q.b.alg./Alle', 'S.f.Amsterdam,AHM/tent.1994', 'R.f.2/Egge.', 'F.g/Mole.', 'Q.b.alg./Alle', 'I.b/Nieu.', 'Q.a.17e/Art', 'P.a.21E/Docu.', 'P.f/Hoop']\n",
      "num entries 29099 - unique entries 2862\n",
      "--\n",
      "http://purl.org/collections/nl/am/acquisitionDate\n",
      "['1887-10-28', '1989', '2001-02-14', '1907', '2003-11', '1907', '1989', '1972-01-04', '1907', '1975-08-14']\n",
      "num entries 64691 - unique entries 3514\n",
      "--\n",
      "http://purl.org/collections/nl/am/documentationTitleLref\n",
      "['11531', '4157', '27022', '16203', '200001058', '23343', '23343', '200010050', '28321', '10153']\n",
      "num entries 38605 - unique entries 5781\n",
      "--\n",
      "http://purl.org/collections/nl/am/reproductionFormat\n",
      "['6 x 6 cm', '167 mB', '22.5 mB', 'kleinbeeld', '300 dpi/25.5 x 16.6 cm', '37.9 mB', '6 x 6 cm', '6 x 6 cm', '6 x 6 cm', '25.2 mB']\n",
      "num entries 10444 - unique entries 927\n",
      "--\n",
      "http://purl.org/collections/nl/am/dimensionPrecision\n",
      "['ca', 'ca', 'ca', 'circa', 'circa', 'ca', 'ca', 'ca', 'ca', 'ca']\n",
      "num entries 28920 - unique entries 51\n",
      "--\n",
      "http://purl.org/collections/nl/am/AHMTextsDate\n",
      "['1998', '2009-03', '1995', '2007-11', '1995', '2003', '1998', '2003', '1979', '2009-03']\n",
      "num entries 5130 - unique entries 128\n",
      "--\n",
      "http://purl.org/collections/nl/am/birthDateStart\n",
      "['1770', '1803-05-01', '1697', '1814-08-14', '1717', '1591', '1837-06-24', '1769', '1702-03-25', '1821']\n",
      "num entries 5199 - unique entries 2313\n",
      "--\n",
      "http://purl.org/collections/nl/am/documentationTitleArticle\n",
      "['The', 'The', 'De', 'De', 'Het', 'De', 'De', 'Het', 'Het', \"'\"]\n",
      "num entries 8797 - unique entries 38\n",
      "--\n",
      "http://purl.org/collections/nl/am/alternativeNumber\n",
      "['3958', 'VdH 12', '4414 K', 'C 86', 'Blankert 304', '4221', '0027', 'cat. 1981 11', 'oka 4316/Z 158', 'Jumbo 722']\n",
      "num entries 21262 - unique entries 11125\n",
      "--\n",
      "http://purl.org/collections/nl/am/AHMTextsType\n",
      "['cat. Zilver', 'cat. Zilver', 'zaaltekst', 'zaaltekst', 'zaaltekst', 'cat. Beelden', 'zaaltekst', 'cat. Tekeningen', 'cat. Kopstukken', 'webtekst']\n",
      "num entries 7981 - unique entries 30\n",
      "--\n",
      "http://purl.org/collections/nl/am/birthDateEnd\n",
      "['1624', '1804-01-13', '1868-09-15', '1828-04-22', '1701', '1725', '1597', '1686', '1882-05-13', '1762-01-22']\n",
      "num entries 3323 - unique entries 2086\n",
      "--\n",
      "http://purl.org/collections/nl/am/exhibitionTitle\n",
      "['De stad door de jaren : de geschiedenis van Amsterdam 1880-1998', 'Jan Eissenloeffel (1876-1957)', 'De Hollandsche schilderkunst van Jeroen Bosch tot Rembrandt', 'Toen hier... Hongerwinter en bevrijding in Amsterdam', 'Mijn hoofddoek', 'Servies in de ontvangstruimte', 'Keuze uit de collectie Fodor', 'Servies in de ontvangstruimte', 'Een huis vol bloemen : zomerse kleurenpracht in donkere jaargetijden', 'ADAM, man en mode']\n",
      "num entries 10471 - unique entries 1094\n",
      "--\n",
      "http://purl.org/collections/nl/am/alternativeNumberInstitution\n",
      "['voorlopig', 'Backerstichting', 'Koninklijk Oudheidkundig Genootschap', 'Backerstichting', 'Stadhuisbibliotheek', 'Backerstichting', 'voorlopig', 'ICN', 'Backerstichting', 'voorlopig']\n",
      "num entries 892 - unique entries 59\n",
      "--\n",
      "http://purl.org/collections/nl/am/usedFor\n",
      "['Vastricx, Hendrik Willemsz', \"Maatschappij tot nut van 't algemeen\", \"d'Ailly, A.J.\", 'Hooft, Corneils Pietersz.', 'Claassen-Huyer, Margaretha Carolina', 'Vrolijk, Johannes Martinus', 'Arondéus, Willem', '(Museum Fodor)', 'vrijmetselaarsloge', 'Stichting geschiedkundig medisch-pharmaceutisch museum']\n",
      "num entries 3883 - unique entries 3857\n",
      "--\n",
      "http://purl.org/collections/nl/am/birthPlace\n",
      "['Amsterdam', 'Antwerpen', 'Naaldwijk', 'Amsterdam', 'Düsseldorf', 'Amsterdam', 'Amsterdam', 'Santpoort', 'Amsterdam', 'Hamburg']\n",
      "num entries 3065 - unique entries 741\n",
      "--\n",
      "http://purl.org/collections/nl/am/currentLocationFitness\n",
      "['tentoonstelling , bruikleencontract : AHM98.02/2', 'tentoonstelling Suasso juwelen', 'intern', 'bruikleen uit, kort , bruikleencontract : UB00.004', 'voor restauratie en bruikleen', 'tentoonstelling Japan', 'intern', 'langdurig bruikleen', 'checken', 'langdurig bruikleen']\n",
      "num entries 4857 - unique entries 502\n",
      "--\n",
      "http://purl.org/collections/nl/am/exhibitionDateStart\n",
      "['1982-02-05', '1990-10-23', '2008-03-01', '2005-09-23', '2009-05-17', '1999-12-04', '2007-11-29', '2008-03-01', '2008-03-01', '1996-03-07']\n",
      "num entries 10018 - unique entries 1009\n",
      "--\n",
      "http://purl.org/collections/nl/am/creatorQualifier\n",
      "['naar', 'naar', 'kopie naar', 'naar', 'naar', 'naar', 'naar', 'naar', 'naar', 'waarschijnlijk']\n",
      "num entries 5100 - unique entries 87\n",
      "--\n",
      "http://purl.org/collections/nl/am/AHMTextsTekst\n",
      "[\"in 1656 ontving Quellinus 168 gulden voor het vervaardigen van vier `wapens vanden heere commissarissen,/ door ordre der heeren ende borger Myn heer graaf ende mijn heer spigel'. Deze vermelding heeft vrijwel zeker betrekking op de vier rijk gesneden wapenborden van leden van de families Roeters, Reynst, Bas en Trip (zie ook cat. nrs. 98, 100, 101), die bij de inwijding van het nieuwe stadhuis in 1655 commissarissen waren van de Wisselbank.\\nDe Wisselbank was gehuisvest in een aantal vertrekken op de begane grond in de zuidoosthoek van het 17de-eeuwse stadhuis. Vermoedelijk waren de schilden aangebracht boven de vier deuren in het kantoor van de commisarissen.\", \"Liefdadige instellingen bezaten in de achttiende en negentiende eeuw veelal zilverwerk ten gebruike der 'Heeren Regenten en Mevrouwen Regentessen'. Het Diaconie Oude Vrouwenhuis vormde hierop geen uitzondering. Het bestuur van het huis bestond uit een college van diakenen en diaconessen, elk met een eigen vertrek aan weerszijden van de kerkzaal (Bakker/Fleurbaay/Gerlagh 1988, p. 323 nr. 329). Over de bestelling en het gebruik van het zilver van de diaconie is weinig bekend.\\nWaarschijnlijk werden losse lepels, vorken en messen en later ook wel complete couverts op kosten van een zojuist verkozen diaken of diacones besteld. De gegraveerde opschriften verwijzen naar de naam van de verkozene en het jaar waarin de verkiezing plaatsvond.\\nWanneer een diaken of diacones zijn of haar bestuursfunctie beëindigde of overleed, bleef het eetgerei deel uitmaken van de inventaris van het Oude Vrouwenhuis. Twee vermeldingen in het archief van de firma As. Bonebakker & Zoon lijken deze veronderstelling te bevestigen.\\nOp 10 februari 1853 bestelde J.C. la Verge Jolles '1 lepel en 1 vork kk 105 à 108 f. 11,35/ werkloon voor het graveren van inscriptie f. 3,90'. Op 24 februari 1858 kocht mej. A.H. Wendelaar bij Bonebakker een tafelcouvert, inclusief graveringen, voor een bedrag van f. 15,75 (GAA, PA 406, Deb. boek 1853, p. 40; Deb. boek 1858, p. 45) (vriendelijke mededeling van de heer B.J. van Benthem).\\nDe achttiende-eeuwse personen (alleen diaconessen) zijn in eerste instantie geïdentificeerd op grond van een geschilderde naamlijst uit 1783, in 2001 aanwezig in de kerkzaal van het verpleeghuis Amstelhof. De volledige titel van deze lijst luidt: NAMEN VAN DE EERWAARDE ZUSTEREN DIACONESSEN/ VAN DE WARE gereformeerde kerke Jesu Christi TOT AMSTERDAM/ welke bij GELEGENHEID der VIERING van het EERSTE EEUWFEEST/ DER BOUW en STICHTING van het DIACONIE OUDE VROUWEN HUIS/ in den Dienst WAREN en Vervolgens zijn AANGEKOMEN ANNO 1783/ (afgekort tot: N1783). Twee Herenboekjes, berustend bij het Gemeentearchief Amsterdam, verschaften aanvullende informatie (afgekort tot: H1787, H1791).\\nVoor de identificatie van de negentiende-eeuwse personen (diakenen en diaconessen) vormde het archief van de Nederlands Hervormde Gemeente (GAA, PA 377) een belangrijke bron. In het bijzonder is gebruik gemaakt van: Verdeeling van het diakende DIENSTWERK in AMSTERDAM 1842-1857, waarin: 'Namen en woonplaatsen van de Diakenen in het jaar ...' en 'Namen der in Leven zijnde en alhier woonende ouderlingen en diakenen en de Jaren hunner Verkiezing' (GAA, PA 377A/254A). Het archiefonderzoek heeft meer gegevens opgeleverd dan in bijgaande lijst kon worden opgenomen. Deze informatie is te raadplegen in het Documentatie archief van het Amsterdams Historisch Museum (met vriendelijke dank aan de heer J. Wit voor het door hem verrichtte onderzoek in genoemde archieven).\\nIn 1994 kwam het complete zilverbezit van het Diaconie Oude Vrouwenhuis onder de hamer (Glerum 1994, p. 32-35 nr. 201-233). Het Amsterdams Historisch Museum kon op deze veiling slechts twee stukken verwerven (cat.nr. KA 19509, KA 19510). Het hier besproken tafelzilver werd door de Diaconie der Hervormde Gemeente aan de veiling onttrokken en alsnog aan het museum geschonken. Zie over het Diaconie Oude Vrouwenhuis het commentaar bij cat.nr. KA 19509.\\nZie voor de zilveren tafelcouverts van het belendende Diaconie Corvershof cat.nr. KA 19522.\", 'Het Rijksmuseum te Amsterdam bezit een soortgelijke bokaal van de hand van Schneider, wiens stijl wordt gekenmerkt door cartouches die enigszins op de vleugels van een vleermuis lijken. Vijf van de zes op dit glas aangebrachte monogrammen komen overeen met die op de bokaal in het Amsterdams Historisch Museum. Het zesde naamcijfer, NtS, verwijst vrijwel zeker naar mr Nicolaas Tersmitten. Hierdoor - en door de afbeelding van een wevende man en een spinnende vrouw - kan de bokaal in het Rijksmuseum in verband worden gebracht met het bestuur van het Nieuwe Werkhuis in Amsterdam (Ritsema van Eck 1995, p. 376 afb., p. 377 nr 458 afb.).', 'Het schilderij van Emanuel de Witte (1617-1692), waarnaar Stolker deze tekening vervaardigde, bevond zich in het kabinet van Jan Snellen (1711-1787), koopman en suikerraffinadeur in Rotterdam, tevens verzamelaar van kunst en naturali‰n. De huidige verblijfplaats van het schilderij is onbekend. Het feit dat Stolker maar liefst drie schilderijen van Emanuel de Witte uit Snellens collectie kopieerde, geeft aan dat Stolker Snellen goed gekend moet hebben. Twee van de drie tekeningen, waaronder deze, kwamen na de veiling van de nalatenschap van Jan Stolker in de verzameling van Cornelis Ploos van Amstel terecht.\\nIn de grote collectie lijntekeningen van Stolker in Rotterdam, bevindt zich ook een tekening met dit kerkinterieur (afb. a).', 'Burgerweeshuis\\nDe regenten en regentessen van het Burgerweeshuis kwamen geregeld bijeen om te vergaderen. De vergaderingen werden onderbroken om de maaltijd te nuttigen. In 1759 plaatste het bestuur van het Burgerweeshuis een grote bestelling luxe tafelzilver bij verschillende Amsterdamse zilversmeden. In hetzelfde jaar voorzag de graveur Jan Delhoff al deze voorwerpen van het embleem van het Burgerweeshuis, een duif in een stalenkrans.', \"Een groep vechtende bedelaars is afgebeeld onder het motto “all'-arm!”. Dit is een woordcombinatie van “allen arm” en de waarschuwingskreet “alarm”, die afkomstig is van het Franse “à l’arme” (te wapen). Op spottende wijze wordt uitgebeeld dat waar armoede heerst geweld onvermijdelijk is.\", \"De Fransman Édouard Baldus is vooral bekend door de foto's die hij in de jaren vijftig en zestig in Zuid-Frankrijk maakte. Hij heeft zowel Romeinse ruïnes als recent gebouwde spoorwegen in beeld gebracht. Baldus heeft veel van zijn werk gemaakt in opdracht van de maatschappij die de spoorlijn Parijs-Lyon-Toulon aanlegde. Het 'oude' landschap dat door deze spoorlijn werd doorsneden is even prominent aanwezig als de spoorlijn zelf. Sommige locaties zijn meer dan eens door Baldus gefotografeerd. De afdrukken in Willets bezit dateren waarschijnlijk uit de jaren '60 of vroege jaren '70. Dat kan afgeleid worden uit het formaat van de foto's en de wijze van signeren.\", 'Kopie naar cat.nr. 249. Tegenhanger van het voorgaande nr., zie aldaar. Op de achterzijde: \"Clara Magdalena Dedel .. Juni 1778, dochter van Jan Hudde Dedel, getrouwd 12 mei 1745 met Hendrick Bicker. Zoon van Mr. Jan Bicker en van Johanna Sara Pels\". De kopiïst heeft het kapsel en de borstversiering gewijzigd.', 'Eén der exemplaren van een compositie waarvan het origineel soms aan Alonso Sanchez Coello (1531-1590), vaker aan diens leermeester Antonio Moro wordt toegeschreven. Friedländer en Marlier nemen aan dat het exemplaar in het Escorial, Madrid het origineel van Moro is (G. Marlier, Anthonis Mor van Dashorst, Brussel 1934, p. 97, nr. 14; M.J. Friedländer, Die Altniederländische Malerei XIII, Leiden 1936, p. 172, nr. 356). Zij achten dat stuk in 1557 geschilderd, volgens Marlier na de in dat jaar geleverde slag bij Saint Quentin: vandaar de commando-staf in Philips´ rechter hand.\\nOm zijn hals draagt de vorst het teken van de orde van het Gulden Vlies. Op zijn borst, op het harnas, is een madonna in een stralenkrans afgebeeld.\\nAndere goede exemplaren bevinden zich te: 1. Cambridge, Fitzwilliam Museum, cat. 1960, nr. M 64, met afb.; 2. Berlijn, Kaiser Friedrich Museum, cat. 1909, nr. 406 B, met afb., als door Coello; 3. Leuven, stadhuis (vermeld door Marlier). Replieken werden reeds vermeld in de vroege 17de eeuw (cat. Cambridge, met verwijzing naar: V. von Loya, in: Jahrbuch der Kunsthistorischen Sammlungen des allerhöchtsten Kaiserhauses 27, 1907, p. 116).\\nVolgens Van Mander zou tijdens Moro´s verblijf aan het hof te Madrid de omgang tussen schilder en koning Philips II uiterst familiair zijn geweest. Aan de waarheid van de anekdotes die hij hieromtrent vertelt, is echter getwijfeld (C. van Mander, het Schilder-Boeck .., Haarlem 1604, fol. 231; Friedländer, op.cit., p. 123).', 'This ox was won by Jacob Reyersz Boon in the parrot shooting contest, as the inscription on the frame states. The inscription on the stone gives the animal’s height as six-anda-half foot; he was 10 foot in circumference and weighed over 1,912 pounds; he was no less than 3.30 metres long.  The sadly anonymous artist emphasises that not a ‘crumb’ of this is untrue.\\nBesides taking home an enormous ox, the winner of the annual shooting contest also got to wear the regalia of the militia king: a sceptre and chain.']\n",
      "num entries 7336 - unique entries 5188\n",
      "--\n",
      "http://purl.org/collections/nl/am/deathPlace\n",
      "['Den Haag', 'Amsterdam', 'Haarlem', 'Amsterdam', 'Amsterdam', 'Amsterdam', 'München', 'Amsterdam', 'Haarlem', 'Parijs']\n",
      "num entries 2694 - unique entries 426\n",
      "--\n",
      "http://purl.org/collections/nl/am/biography\n",
      "['zilversmid, werkzaam 1529-1561', 'beeldhouwer', 'Schepen te Dordrecht; Raad en Schepen te Amsterdam', 'Burgemeester van Amsterdam 1842-1852', 'werkzaam in Amsterdam 1869-na 1938)', 'niet in Thieme Becker', 'regent van het oudezijds huiszitten-aalmoezeniershuis, tevens regent van het weduwenhof; 1806', 'regent van het leprozenhuis 1814-1826', 'huwde 3 maart 1910 Joseph Vintura (1880-?)', 'zilversmid, werkzaam 1855-1858']\n",
      "num entries 1953 - unique entries 1683\n",
      "--\n",
      "http://purl.org/collections/nl/am/exhibitionNotes\n",
      "['Object zelf is tentoonstelling', 'Alleen KB 2337.1 t/m 8 gaan naar de tentoonstelling.', 'showroom', 'Object zelf is tentoonstelling', 'Uitgeleend via het RMA', '-', 'op het kaartje: Egbert Gerbrantsz en zijn vrouw zijn op het schilderij omringd door vergankelijkheidssymbolen, zoals gebruikelijk in portretkunst van de 16e eeuw', 'cat.nr. samen met SA 25978', 'Alleen KA 18069.2 gaat naar de tentoonstelling', 'Object is zelf gelijk aan deze tentoonstelling.']\n",
      "num entries 211 - unique entries 96\n",
      "--\n",
      "http://purl.org/collections/nl/am/AHMTextsAuthor\n",
      "['Vreeken, Bert', 'Vreeken, Bert', 'Rooseboom, Hans', 'Dekker, Annemarie den', 'Blankert, Albert', 'Blankert, Albert', 'Blankert, Albert', 'Vreeken, Bert', 'Jonker, Michiel', 'Middelkoop, Norbert']\n",
      "num entries 4407 - unique entries 52\n",
      "--\n",
      "http://purl.org/collections/nl/am/deathDateEnd\n",
      "['1944', '1821/02/12', '1712', '1776/10/03', '1972-10-08', '1786', '1693', '1816-01-30', '1702', '1843']\n",
      "num entries 3074 - unique entries 2040\n",
      "--\n",
      "http://purl.org/collections/nl/am/relatedObjectNotes\n",
      "['p. 159', \"Rechts in het vertrek op 'Drie regententen en de binnenmoeder van het Leprozenhuis' van Louis Moritz is 'De Dam in 1604 tijdens de laatste Leprozenommegang op Koppertjesmaandag' van Adriaen van Nieulandt afgebeeld.\", 'p. 214', 'p. 99', 'p. 85', 'p. 148', 'p. 15', 'p. 123', 'p. 166', 'p. 325']\n",
      "num entries 1044 - unique entries 421\n",
      "--\n",
      "http://purl.org/collections/nl/am/creditLine\n",
      "['Amsterdams Historisch Museum, legaat C.J. Fodor', 'Amsterdams Historisch Museum, schenking kinderen C.P. van Eeghen', 'Amsterdams Historisch Museum, bruikleen van Stichting Spirit', 'Amsterdams Historisch Museum, schenking kinderen C.P. van Eeghen', 'Amsterdams Historisch Museum, bruikleen van Stichting Spirit', 'Amsterdams Historisch Museum, schenking kinderen C.P. van Eeghen', 'Amsterdams Historisch Museum, legaat C.J. Fodor', 'Amsterdams Historisch Museum, legaat C.J. Fodor', 'Amsterdams Historisch Museum, bruikleen Rijksmuseum Amsterdam', 'Amsterdams Historisch Museum, bruikleen van Stichting Spirit']\n",
      "num entries 6954 - unique entries 176\n",
      "--\n",
      "http://purl.org/collections/nl/am/exhibitionCatalogueNumber\n",
      "['343', '39', 'nr. 30', '10', '221', '421', 'AHM', '35', '3', '851']\n",
      "num entries 802 - unique entries 412\n",
      "--\n",
      "http://purl.org/collections/nl/am/relatedObjectTitle\n",
      "['Veren waaier, in doos (doos)', 'Reclameletters firma T.M. Mendelson (letter E)', 'franje', 'Waaiersteun', 'Maatstok', 'Gevelplaat met naam firma T.M. Mendelson in doos (doos)', 'band', 'stempels met stempeldoos Fa. T.M. Mendelson', 'Waaiersteun', 'lappen met gitten']\n",
      "num entries 1761 - unique entries 68\n",
      "--\n",
      "http://www.w3.org/2004/02/skos/core#altLabel\n",
      "['beursgebouwen', 'boekwinkel', 'Barbizon, School van', 'dominees', 'epidemieën', 'Paris', 'lijkdichten', 'papierrestauratie', 'huwelijksgedichten', 'historiologie']\n",
      "num entries 558 - unique entries 546\n",
      "--\n",
      "http://purl.org/collections/nl/am/birthDatePrecision\n",
      "['ca', 'ca. 1497', 'ca.', 'circa', '?', 'ca.', 'ca', '1597', '1617', 'waarschijnlijk 1633']\n",
      "num entries 286 - unique entries 167\n",
      "--\n",
      "http://purl.org/collections/nl/am/value\n",
      "['motief', 'Spitzenbegriff', 'COLL', 'CATEGORY', 'ROLE', 'LOANCAT', 'acquisition', 'activity', 'CONCEPT', 'materiaal']\n",
      "num entries 202 - unique entries 197\n",
      "--\n",
      "http://purl.org/collections/nl/am/occupation\n",
      "['schilder', 'droogscheerder', '1539-41 burgemeester', 'goud- en zilversmid', 'beeldhouwer', 'diaconesse', 'drukker', 'tentoonstellingsmaker/curator', 'notaris', 'koopman']\n",
      "num entries 786 - unique entries 245\n",
      "--\n",
      "http://purl.org/collections/nl/am/AHMTextsReference\n",
      "['..\\\\..\\\\DAT\\\\Collectie\\\\objectteksten\\\\KB_871_KB_872.pdf', '..\\\\..\\\\DAT\\\\collectie\\\\objectteksten\\\\SB 5812 zaaltekst.doc', '..\\\\..\\\\DAT\\\\collectie\\\\objectteksten\\\\KA_3686_1_37_KA_18444_1_2_KA_18086.pdf', '..\\\\..\\\\DAT\\\\collectie\\\\objectdocumentatie\\\\glas 1998\\\\collectie\\\\KA_18249\\\\index.html', '..\\\\..\\\\DAT\\\\Collectie\\\\objectteksten\\\\SB_6374_SB_6383.pdf', '..\\\\..\\\\DAT\\\\Collectie\\\\objectteksten\\\\KB_3768_KB_3769_KB_3770_KB_3771_KB_3772_KB_3773.pdf', '..\\\\..\\\\DAT\\\\collectie\\\\objectdocumentatie\\\\glas 1998\\\\collectie\\\\KA_5312\\\\index.html', '..\\\\..\\\\DAT\\\\Collectie\\\\objectteksten\\\\KB_768.pdf', '..\\\\..\\\\DAT\\\\collectie\\\\objectdocumentatie\\\\glas 1998\\\\collectie\\\\KA_15356\\\\index.html', '..\\\\..\\\\DAT\\\\collectie\\\\objectteksten\\\\KA_3686_1_37_KA_18444_1_2_KA_18086.pdf']\n",
      "num entries 2088 - unique entries 856\n",
      "--\n",
      "http://purl.org/collections/nl/am/exhibitionVenue\n",
      "['The J. Paul Getty Museum', 'Waag, de', 'Historisch Museum', 'Princessehof', \"Cabinet des Estampes, Museé d'Art en d'Histoire\", 'Princessehof', 'Franckesche Stiftungen', 'Stichting Projecten De Nieuwe Kerk  Amsterdam', 'Stichting Stedelijk Museum Amsterdam', 'Universiteitsbibliotheek  (Amsterdam)']\n",
      "num entries 1514 - unique entries 247\n",
      "--\n",
      "http://purl.org/collections/nl/am/deathDateStart\n",
      "['1625', '1941', '1675 (na)', '1987-06-21', '1879-10-10', '1604', '1916', '1824-02-24', '1661', '1860']\n",
      "num entries 4887 - unique entries 2187\n",
      "--\n",
      "http://purl.org/collections/nl/am/ownershipHistoryFreeText\n",
      "['Legaat A. van der Hoop, 1854; bruikleen aan Rijksmuseum, 1885', 'De pakje behoorde tot een schenking van 17.2.1978 aan het Nederlands Kostuum Museum in Den Haag door Mevr. J.A: Zeldenrust-De Beaufort, Frankrijk en was afkomstig uit de nalatenschap van haar moeder Mevr. De Beaufort, wonende op het huis “De Treek” te Leusden.', 'Bruikleen Spirit, 1962; schoorsteenstuk Regentenkamer Burgerweeshuis, thans onderdeel van het AHM', \"Veiling Van der Zande 1855, p. 346, nr. 3032: 'Ostade (Adrien Van). Cinq paysans assis à boire et à fumer. Une femme leur apporte un pot de bière ; à gauche, un chien couché. Morceau d'une admirable exécution, à plume et lavé d'encre de Chine. On y a joint l'eau-fort qu'en a gravée, en plus petit, F. Hillemacher' (fr. 1.700,-) ; Veiling Norblin 1860, p. 18, nr. 111 \\nOstade (Adrien Van). Cinq paysans assis à boire et à fumer. Une femme leur apporte un pot de bière ; à gauche, un chien couché. Morceau d'une admirable exécution, à plume et lavé d'encre de Chine. On y a joint l'eau-fort qu'en a gravée, en plus petit, F. Hillemacher. Ce beau dessin provient de la collection Van den Zande. La description que nous en donnons est extraite du catalogue rédigé par M. Guichardot' (fr. 3020,- , aan C.J. Fodor) ; Collectie C.J. Fodor, legaat 1860\", 'Verzameling Christina Catharina Kemper-de Vries, Leiden, waarschijnlijk ná 1824; van Christina Catharina Kemper-de Vries, gekocht door Adriaan van der Hoop door bemiddeling van haar broer, kunsthandelaar Jeronimo de Vries, 1833; legaat aan de Stad Amsterdam, 1854; Museum Van der Hoop, 1854-1885; bruikleen aan Rijksmuseum, 1885', 'Aankoop 1936, van  Kunsthandel E. Speelman, Londen, (Fl. 60,-), (volgens documentatiearchief), vermeld in het jaarverslag over dat jaar, p. 16', 'Diaconie Oude Vrouwenhuis, Amsterdam; geschenk van de Diaconie der Hervormde Gemeente, Amsterdam, 1994', 'Verzameling W.J.R. Dreesmann, Amsterdam; veiling Frederik Muller 22/25-3-1960, nr. 152', 'Het kostuum is gedragen door J.H. Scheltema, van 1904-1921 gemeenteraadslid van Amsterdam voor de Bond van Vrije Liberalen, hij was tevens lid van de Amsterdamsche Eerewacht. Het kostuum is geschonken door zijn jongste zoon, Mr. C. Scheltema. (schr. mw. E.A.M. Scheltema-Vriesendorp, 1983/12/16)', 'Veiling Marie-Caroline de Bourbon, Duchesse de Berry (Parijs), Londen (Christie’s), april 1834, nr. 101, niet verkocht; veiling Marie-Caroline de Bourbon, Duchesse de Berry, Parijs, 4 april 1837, nr. 87, aan kunsthandelaar H,ris voor Adriaan van der Hoop door bemiddeling van de kunsthandelaar Albertus Brondgeest; legaat aan de Stad Amsterdam, 1854; Museum Van der Hoop, 1854-1885; bruikleen aan Rijksmuseum, 1885']\n",
      "num entries 3412 - unique entries 1749\n",
      "--\n",
      "http://purl.org/collections/nl/am/source\n",
      "['AHM', 'AAT', 'AHM', 'AHM', 'AAT zonder hierarchie', 'niet in AAT', 'AHM', 'niet in AAT', 'AAT', 'AAT']\n",
      "num entries 1096 - unique entries 34\n",
      "--\n",
      "http://purl.org/collections/nl/am/equivalentName\n",
      "['Internationaal Instituut voor Sociale Geschiedenis', 'V.V.H.K.', 'Bouts, Dirk', 'Eijl, dhr. G.G.L.F.Th. van', 'Sotheby Mak van Waay B.V.', 'Voltaire', 'Verster, Floris', 'Wolf, N.H.', 'Linde, Hans van der', 'Protestantse Diaconie Amsterdam']\n",
      "num entries 1015 - unique entries 764\n",
      "--\n",
      "http://purl.org/collections/nl/am/use\n",
      "['Amsterdamsch Studenten Corps', 'Grégoire, Jan', 'Groen, Marga', 'Kunstgezelschap M.A.B.', 'Iliohan, Mevrouw J.', 'Stichting Hulp na Onderzoek', 'A. Boersma', \"Sa'di, Mušarrif al-Din b. Muslih\", 'O+S', 'Bezige Bij, De']\n",
      "num entries 1855 - unique entries 1425\n",
      "--\n",
      "http://purl.org/collections/nl/am/scopeNote\n",
      "['Drinkgerei gemaakt van een hoorn, in zilver of brons gemonteerd.', 'Kleine vijlen die van metaal of karton zijn gemaakt en worden gebruikt voor het bijwerken, gladmaken en scherp maken van vingernagels. Hebben soms een punt om het vuil onder de nagels te verwijderen.', '(bron: GAA)', '(bron: GAA)', '(bron: GAA)', '(bron: GAA)', 'decoratieve kunststijl van de Renaissance-barok (AAT-NL)', 'bewerkte huid van kalf, schaap, geit enz. geschikt om erop te schrijven, boeken in te binden of als trommelvel', 'Objecten dan wel voorgevormde of buigzame bedekkingen, bedoeld om voorwerpen of stoffen te bevatten, te bewaren of te verschepen.', '(bron: GAA)']\n",
      "num entries 726 - unique entries 540\n",
      "--\n",
      "http://purl.org/collections/nl/am/creator\n",
      "['Stereo beschuit- en koekfabriek N. V. Rotterdam', 'J. F. Jarvis', 'Underwood & Underwood', 'Underwood & Underwood', 'Underwood & Underwood', 'Underwood & Underwood', 'J. F. Jarvis', 'Underwood & Underwood', 'Jarvis, J. F.', 'Underwood & Underwood']\n",
      "num entries 226 - unique entries 23\n",
      "--\n",
      "http://purl.org/collections/nl/am/deathDatePrecision\n",
      "['na 1729', 'ca.', 'na 1690', 'vermoedelijk 1807', 'vóór', '05-31', '1951-11-02', \"1626-'27\", '1840-06-19', '12-29']\n",
      "num entries 185 - unique entries 131\n",
      "--\n",
      "http://purl.org/collections/nl/am/birthNotes\n",
      "['Ca. 1625', 'Gedoopt: 1734/07/18', 'doop', '1690 (?)', 'Gedoopt: 1720/11/13', 'ca. 1815', 'Ca. 1565', 'volgens Willigen 1643/44', 'Gedoopt op 14 december 1806', 'Gedoopt: 1666/12/21']\n",
      "num entries 455 - unique entries 408\n",
      "--\n",
      "http://purl.org/collections/nl/am/exhibitionObjectLocation\n",
      "['V. m', 'kast', '9V2M/06', 'kast', 'V.h', '9V2M/04', 'V. x', 'V. x', '06VR/04', '06VR/08']\n",
      "num entries 250 - unique entries 154\n",
      "--\n",
      "http://purl.org/collections/nl/am/exhibitionCode\n",
      "['1.3', '1.4', '6.1.3', '6.2.3', '1.3', '1.6', '3.2.3', '6.1.3', '1.3', '1.3']\n",
      "num entries 295 - unique entries 40\n",
      "--\n",
      "http://purl.org/collections/nl/am/documentationAuthor\n",
      "['Schipper, Daniël', 'Lavèn, Annemarie', 'Schipper, Daniël', 'Lavèn, Annemarie', 'Schipper, Daniël', 'Dehé, John', 'Schipper, Daniël', 'Schipper, Daniël', 'Lavèn, Annemarie', 'Schoots, Hans']\n",
      "num entries 22 - unique entries 5\n",
      "--\n",
      "http://purl.org/collections/nl/am/deathNotes\n",
      "['(ca.)', 'ca. 1611', 'Bij Wurzbach: 1652/06/26', 'Gedoopt: 1764/06/13', 'Geboren: 1750/03/16 (Scheen/Thieme-Becker)', 'Gedoopt: 1595/04/06', 'op.c.1850', 'ca.', '1583', 'Ca. 1737']\n",
      "num entries 408 - unique entries 363\n",
      "--\n",
      "http://purl.org/collections/nl/am/alternativeNumberDate\n",
      "['2003-11-06', '2005-09-28', '2003-11-06', '2007', '1995', '1993', '2007', '2004-04', '2003-11-06', '1975']\n",
      "num entries 578 - unique entries 74\n",
      "--\n",
      "http://purl.org/collections/nl/am/currentLocation\n",
      "['AHM dep 3 kast 20-5', 'WH Ontmoet de Willets', 'AHM dep 3 kast 20-5', 'AHM dep 3 kast 20-5', 'Bijkerk, A.Th.', 'AHM dep 3 kast 20-5', 'WH Ontmoet de Willets', 'WH - W10 d', 'AHM dep 3 kast 20-5', 'WH Ontmoet de Willets']\n",
      "num entries 382 - unique entries 36\n",
      "--\n",
      "http://purl.org/collections/nl/am/reproductionNotes\n",
      "['Opmerking Alice de Keijser, d.d. 2008-04-09: crinoline wordt hier \"binnenste buiten en achterste voren\" getoond. Foto dus niet gebruiken voor publicatie!', 'reproductie toegevoegd door adapl: jweemhoff (2003-04-24)', 'Stempel op achterzijde', '(E22)', 'voorzijde', 'reproductie toegevoegd door adapl: jweemhoff (2003-04-24)', 'Stempel op achterzijde', 'na restauratie 1970-72', 'detail', 'detail']\n",
      "num entries 134 - unique entries 38\n",
      "--\n",
      "http://purl.org/collections/nl/am/relatedObjectAssociation\n",
      "['pendant', 'onderdeel', 'pendant', 'pendant', 'pendant', 'pendant', 'pendant', 'pendant', 'pendant', 'pendant']\n",
      "num entries 113 - unique entries 6\n",
      "--\n",
      "http://purl.org/collections/nl/am/nationality\n",
      "['Nederlandse', 'Nederlandse', 'Duitse', 'Nederlandse', 'Nederlandse', 'Nederlandse', 'Nederlandse', 'Nederlandse', 'Nederlandse', 'Duitse']\n",
      "num entries 239 - unique entries 33\n",
      "--\n",
      "http://purl.org/collections/nl/am/objectName\n",
      "['stereokaart', 'stereokaart', 'stereokaart', 'stereokaart', 'kleinwerk', 'stereokaart', 'stereokaart', 'stereokaart', 'stereokaart', 'stereokaart']\n",
      "num entries 418 - unique entries 5\n",
      "--\n",
      "http://purl.org/collections/nl/am/partsTitle\n",
      "['diverse fournituren, strook imitatie persianer', 'placemat Schreierstoren', 'placemat Schreierstoren', 'diverse fournituren, sierband', 'placemat met hoekhuis trapgevel', 'placemat Koepelkerk', 'Reclameletters firma T.M. Mendelson (letter L)', 'Bril met opzetstukken in doos (doos)', 'vingerdoekje met hoekhuis trapgevel', 'vingerdoekje met hoekhuis trapgevel']\n",
      "num entries 68 - unique entries 32\n",
      "--\n",
      "http://purl.org/collections/nl/am/relatedObjectReference\n",
      "['Ka 13601', 'PB 944', 'TA 15893', 'A 18857 X', '03.06.19.003', 'PB 944', '03.06.22.006.1/2', 'PB 945a', 'KA 6758', 'PB 945a']\n",
      "num entries 220 - unique entries 113\n",
      "--\n",
      "http://purl.org/collections/nl/am/associationSubject\n",
      "['Grand Canyon', 'Martinique', 'Niagara watervallen', 'Thun', 'Brooklyn Bridge', 'Minnesota', 'Cascade du Dard', 'Droogbak', 'Galta tempel', 'Chamonix']\n",
      "num entries 166 - unique entries 132\n",
      "--\n",
      "http://purl.org/collections/nl/am/contentMotifGeneral\n",
      "['groep personen', 'groep personen', 'groep personen', 'groep personen', 'groep personen', 'groep personen', 'groep personen', 'groep personen', 'groep personen', 'groep personen']\n",
      "num entries 38 - unique entries 3\n",
      "--\n",
      "http://purl.org/collections/nl/am/partsReference\n",
      "['KA 2901', 'KB 2633.1/2', 'KB 2135.2', 'KA 19764.71', 'KA 20653.79', 'KA 2897', 'KA 20653.114', 'KA 7203', 'KA 20653.90', 'KA 5975']\n",
      "num entries 11 - unique entries 11\n",
      "--\n",
      "http://purl.org/collections/nl/am/contentSubject\n",
      "['lelie', 'vulkaanuitbarsting', 'Haarlemmerweg 485', 'japonse rok', 'vulkaanuitbarsting', 'Niagara watervallen', 'japonse rok', 'Korsjespoortsteeg 23', 'Buitenkant', 'japonse rok']\n",
      "num entries 43 - unique entries 21\n",
      "--\n",
      "http://purl.org/collections/nl/am/partOfReference\n",
      "['KA 1519, KA 1521', 'KA 1519, KA 1521', 'KB 1727.1/4', 'KA 1519, KA 1521', 'KB 1727.1/4', 'KA 1519, KA 1521', 'KA 1519, KA 1521', 'KB 1727.1/4', 'KA 1519, KA 1521', 'KA 1519, KA 1521']\n",
      "num entries 19 - unique entries 3\n",
      "--\n",
      "http://purl.org/collections/nl/am/associationPerson\n",
      "['Marriott Hotel', 'Patria biscuitfabriek', 'Koopmans, Hyke', 'Faith71', 'Paulowna, Anna', 'Bitterswijk, Marje', 'J. Nooitgedagt', 'S. S. Mosel', 'Aëgerter, Laurence', 'Redlight Fashion Amsterdam']\n",
      "num entries 55 - unique entries 35\n",
      "--\n",
      "http://purl.org/collections/nl/am/technique\n",
      "['geplastificeerd', 'geplastificeerd', 'geplastificeerd', 'geplastificeerd', 'geplastificeerd', 'tamboureren', 'geplastificeerd', 'geplastificeerd', 'geplastificeerd', 'geplastificeerd']\n",
      "num entries 18 - unique entries 2\n",
      "--\n",
      "http://www.w3.org/1999/02/22-rdf-syntax-ns#value\n",
      "['\\n      ', '\\t', '\\n      ', '\\t', '\\n      ', '\\t']\n",
      "num entries 6 - unique entries 2\n",
      "--\n",
      "http://purl.org/collections/nl/am/contentPersonName\n",
      "['Lutherse kerk', 'Kasteel Heidelberg', 'Edinburgh Castle', 'Jenner, Edward', 'Witte Huis', 'Fontein, Dieuwke', 'Witte Huis', 'Paulowna, Anna', 'Bank of England', 'Paulowna, Anna']\n",
      "num entries 15 - unique entries 12\n",
      "--\n",
      "http://purl.org/collections/nl/am/reproductionCreator\n",
      "['Tuijl, Karin van', 'Boom, Eric van den (medailleur)', 'Aëgerter, Laurence', 'Aëgerter, Laurence', 'Aëgerter, Laurence', 'Aëgerter, Laurence', 'Eichelsheim, Leon', 'Eichelsheim, Leon']\n",
      "num entries 8 - unique entries 4\n",
      "--\n",
      "http://purl.org/dc/terms/title\n",
      "['Amsterdam Museum low precision vocabulary alignments', 'Amsterdam Museum (vocabularies)', 'Amsterdam Museum (people database)', 'Amsterdam Museum (metadata)', 'Amsterdam Museum vocabulary alignments', 'Amsterdam Museum', 'Amsterdam Museum (thesaurus)', 'Amsterdam Museum mid precision vocabulary alignments', 'Amsterdam Museum high precision vocabulary alignments']\n",
      "num entries 9 - unique entries 9\n",
      "--\n",
      "http://purl.org/collections/nl/am/productionPlace\n",
      "['Venice, CA']\n",
      "num entries 1 - unique entries 1\n",
      "--\n",
      "http://purl.org/collections/nl/am/exhibitionOrganiser\n",
      "['Palais Fesch', 'Geelvinck Hinlopenhuis', 'Palais Fesch', 'Historisch Museum Het Palthe-Huis, Stichting']\n",
      "num entries 4 - unique entries 3\n",
      "--\n",
      "http://purl.org/collections/nl/am/productionPeriod\n",
      "['1903']\n",
      "num entries 1 - unique entries 1\n",
      "--\n",
      "http://purl.org/collections/nl/am/documentationNotes\n",
      "['ref.']\n",
      "num entries 1 - unique entries 1\n",
      "--\n",
      "http://purl.org/collections/nl/am/dimensionType\n",
      "['diepte uitgeklapt', 'hoogte uitgeklapt']\n",
      "num entries 2 - unique entries 2\n",
      "--\n",
      "http://purl.org/collections/nl/am/lang\n",
      "['neutral']\n",
      "num entries 1 - unique entries 1\n",
      "--\n"
     ]
    }
   ],
   "source": [
    "for k, v in data.node_features.items():\n",
    "    print(k)\n",
    "    print(v[1][0:10])\n",
    "    print(f'num entries {len(v[1])} - unique entries {len(set(v[1]))}')\n",
    "    print('--')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Divide literals in: \n",
    "# categoricals: will be modelled as if they are regular nodes\n",
    "# numericals: will be modelled as univariate numbers\n",
    "# dates: will be modelled as dates\n",
    "# strings: will be modelled as textual strings\n",
    "# all other literal types will be removed!\n",
    "am = rdflib.Namespace(\"http://purl.org/collections/nl/am/\")\n",
    "categoricals = [\n",
    "    am[\"dimensionPart\"],\n",
    "    am[\"dimensionUnit\"],\n",
    "    am[\"currentLocationLref\"],\n",
    "    am[\"currentLocationType\"],\n",
    "    am[\"priref\"],\n",
    "    am[\"AHMTextsAuthorLref\"],\n",
    "    am[\"exhibitionLref\"],\n",
    "    am[\"alternativeNumberType\"],\n",
    "    am[\"documentationShelfmark\"],\n",
    "    am[\"documentationTitleLref\"],\n",
    "    am[\"documentationTitleArticle\"],\n",
    "    am[\"alternativeNumberInstitution\"],\n",
    "    am[\"exhibitionVenue\"],\n",
    "    am[\"relatedObjectAssociation\"]\n",
    "]\n",
    "numericals = [am[\"dimensionValue\"]]\n",
    "dates = [\n",
    "    am[\"reproductionDate\"],\n",
    "    am[\"currentLocationDateStart\"],\n",
    "    am[\"documentationSortyear\"],\n",
    "    am[\"productionDateStart\"],\n",
    "    am[\"exhibitionDateEnd\"],\n",
    "    am[\"productionDateEnd\"],\n",
    "    am[\"currentLocationDateEnd\"],\n",
    "    am[\"AHMTextsPubl\"],\n",
    "    am[\"acquisitionDate\"],\n",
    "    am[\"AHMTextsDate\"],\n",
    "    am[\"birthDateEnd\"],\n",
    "    am[\"exhibitionDateStart\"],\n",
    "    am[\"deathDateEnd\"],\n",
    "    am[\"deathDateStart\"],\n",
    "]\n",
    "strings = [\n",
    "    am[\"creatorRole\"],\n",
    "    am[\"name\"],\n",
    "    am[\"documentationTitle\"],\n",
    "    am[\"title\"],\n",
    "    rdflib.URIRef(\"http://www.w3.org/2004/02/skos/core#prefLabel\"),\n",
    "    am[\"AHMTextsType\"],\n",
    "    am[\"exhibitionTitle\"],\n",
    "    am[\"usedFor\"],\n",
    "    am[\"birthPlace\"],\n",
    "    am[\"currentLocationFitness\"],\n",
    "    am[\"creatorQualifier\"],\n",
    "    am[\"deathPlace\"],\n",
    "    am[\"AHMTextsTekst\"],\n",
    "    am[\"biography\"],\n",
    "    am[\"relatedObjectNotes\"],\n",
    "    am[\"creditLine\"],\n",
    "    am[\"relatedObjectTitle\"],\n",
    "    rdflib.URIRef(\"http://www.w3.org/2004/02/skos/core#altLabel\"),\n",
    "    am[\"occupation\"],\n",
    "    am[\"source\"],\n",
    "    am[\"equivalentName\"],\n",
    "    am[\"nationality\"],\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Store literal types remove literals unaccounted for in previous step\n",
    "types_to_remove = set(data.node_features.keys()).difference(\n",
    "    set(strings).union(set(categoricals)).union(set(dates)).union(set(categoricals))\n",
    ")\n",
    "removed_literal_idxs = []\n",
    "for t in types_to_remove:\n",
    "    removed_literal_idxs.append(torch.tensor(data.node_features[t][0], dtype=torch.long))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "processed_node_features = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def date_to_number(date: str):\n",
    "    if date is None:\n",
    "        return None\n",
    "\n",
    "    date = date.replace(\"?\", \"\").strip()\n",
    "    date = date.replace(\"(\", \"\").strip()\n",
    "    date = date.replace(\")\", \"\").strip()\n",
    "    date = re.sub(r\"(\\D*)$\", \"\", date).strip()\n",
    "    date = re.sub(r\"^(\\D*)\", \"\", date).strip()\n",
    "\n",
    "    date = date.replace(\"/\", \"-\")\n",
    "    date = date.replace(\"\\\\\", \"-\")\n",
    "    date = date.replace(\".\", \"-\")\n",
    "    date = date.replace(\" \", \"-\")\n",
    "\n",
    "    if \"-\" in date:\n",
    "        parts = date.split(\"-\")\n",
    "    else:\n",
    "        parts = [date]\n",
    "\n",
    "    parts = [p for p in parts if len(p.strip())]\n",
    "    values = [1, 1 / 12, 1 / 365]\n",
    "    val = 0\n",
    "    for v, p in zip(values, parts):\n",
    "        try:\n",
    "            val += v * int(p)\n",
    "        except:\n",
    "            return torch.nan\n",
    "    return val\n",
    "\n",
    "\n",
    "for l in dates:\n",
    "    indices = torch.tensor(data.node_features[l][0], dtype=torch.long)\n",
    "    float_dates = torch.tensor(\n",
    "        [date_to_number(d) for d in data.node_features[l][1]], dtype=torch.float32\n",
    "    )\n",
    "\n",
    "    # remove invalid nodes from graph\n",
    "    removed_literal_idxs.append(indices[torch.isnan(float_dates)])\n",
    "\n",
    "    indices = indices[~torch.isnan(float_dates)]\n",
    "    float_dates = float_dates[~torch.isnan(float_dates)]\n",
    "\n",
    "    processed_node_features[len(processed_node_features)] = [\n",
    "        indices,\n",
    "        float_dates.reshape(-1, 1),\n",
    "    ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "for l in numericals:\n",
    "    processed_numericals = []\n",
    "    for n in data.node_features[l][1]:\n",
    "        try:\n",
    "            processed_numericals.append(float(n.replace(\",\", \".\")))\n",
    "        except:\n",
    "            processed_numericals.append(torch.nan)\n",
    "\n",
    "    indices = torch.tensor(data.node_features[l][0], dtype=torch.long)\n",
    "    numerical_features = torch.tensor(processed_numericals, dtype=torch.float32)\n",
    "\n",
    "\n",
    "    # remove invalid nodes from graph\n",
    "    removed_literal_idxs.append(indices[torch.isnan(numerical_features)])\n",
    "\n",
    "    indices = indices[~torch.isnan(numerical_features)]\n",
    "    numerical_features = numerical_features[~torch.isnan(numerical_features)]\n",
    "\n",
    "    processed_node_features[len(processed_node_features)] = [\n",
    "        indices,\n",
    "        numerical_features.reshape(-1, 1),\n",
    "    ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 22/22 [22:21<00:00, 60.99s/it]  \n"
     ]
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"GroNLP/bert-base-dutch-cased\")\n",
    "model = AutoModelForMaskedLM.from_pretrained(\"GroNLP/bert-base-dutch-cased\",)\n",
    "model = model.to(device)\n",
    "\n",
    "def mean_pooling(model_output, attention_mask):\n",
    "    token_embeddings = model_output[\n",
    "        0\n",
    "    ]  # First element of model_output contains all token embeddings\n",
    "    input_mask_expanded = (\n",
    "        attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()\n",
    "    )\n",
    "    sum_embeddings = torch.sum(token_embeddings * input_mask_expanded, 1)\n",
    "    sum_mask = torch.clamp(input_mask_expanded.sum(1), min=1e-9)\n",
    "    return sum_embeddings / sum_mask\n",
    "\n",
    "\n",
    "batch_size = 64\n",
    "for l in tqdm(strings):\n",
    "    model.eval()\n",
    "    indices = torch.tensor(data.node_features[l][0], dtype=torch.long)\n",
    "\n",
    "    sents = data.node_features[l][1]\n",
    "    num_batches = math.ceil(len(sents) / batch_size)\n",
    "\n",
    "    s_embs = []\n",
    "    for b in range(num_batches):\n",
    "        inputs = tokenizer(\n",
    "            sents[b * batch_size : (b + 1) * batch_size],\n",
    "            padding=\"max_length\",\n",
    "            return_tensors=\"pt\",\n",
    "            max_length=256,\n",
    "            truncation=True\n",
    "        ).to(device)\n",
    "        s_emb = mean_pooling(\n",
    "            model._modules[\"bert\"](\n",
    "                **inputs\n",
    "            ),\n",
    "            inputs[\"attention_mask\"]\n",
    "        ).detach().cpu()\n",
    "        s_embs.append(s_emb)\n",
    "\n",
    "    s_emb = torch.vstack(s_embs)\n",
    "    \n",
    "    processed_node_features[len(processed_node_features)] = [\n",
    "        indices,\n",
    "        s_emb,\n",
    "    ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "literals = torch.hstack(removed_literal_idxs).squeeze().to(device)\n",
    "mask = torch.isin(data.edge_index[0], literals) | torch.isin(\n",
    "    data.edge_index[1], literals\n",
    ")\n",
    "edge_index = data.edge_index[:, ~mask]\n",
    "edge_type = data.edge_type[~mask]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_idx, val_idx, y_train, y_val = train_test_split(\n",
    "    data.train_idx,\n",
    "    data.train_y,\n",
    "    stratify=data.train_y.cpu().numpy(),\n",
    "    test_size=0.1,\n",
    "    random_state=42,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedder = RRGCNEmbedder(\n",
    "    num_nodes=data.num_nodes,\n",
    "    num_relations=dataset.num_relations,\n",
    "    num_layers=5,\n",
    "    emb_size=512,\n",
    "    device=device,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [03:00<00:00, 180.20s/it]\n"
     ]
    }
   ],
   "source": [
    "# for node features to work well, they have to be normalized\n",
    "# you can choose \"standard\" for StandardScaler, \"robust\" for RobustScaler, \"quantile\"\n",
    "# for QuantileTransformer and \"power\" for PowerTransformer\n",
    "#\n",
    "# you could also pass sklearn compatible scalers by passing a dict keyed by\n",
    "# literal type, e.g.:\n",
    "# {0: StandardScaler(), 1: RobustScaler()}\n",
    "from sklearn.preprocessing import StandardScaler, QuantileTransformer\n",
    "\n",
    "train_embs = embedder.embeddings(\n",
    "    edge_index,\n",
    "    edge_type,\n",
    "    node_features=processed_node_features,\n",
    "    node_features_scalers={\n",
    "        k: None\n",
    "        if v[1].shape[1] > 1\n",
    "        else QuantileTransformer(output_distribution=\"normal\")\n",
    "        for k, v in processed_node_features.items()\n",
    "    },\n",
    "    idx=train_idx,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [03:01<00:00, 181.36s/it]\n",
      "100%|██████████| 1/1 [03:04<00:00, 184.37s/it]\n"
     ]
    }
   ],
   "source": [
    "# only fit node feature scalers on nodes reachable from train nodes,\n",
    "# for val and test nodes, reuse the fit scalers using embedder.get_last_fit_scalers()\n",
    "val_embs = embedder.embeddings(\n",
    "    edge_index,\n",
    "    edge_type,\n",
    "    node_features=processed_node_features,\n",
    "    node_features_scalers=embedder.get_last_fit_scalers(),\n",
    "    idx=val_idx,\n",
    ")\n",
    "test_embs = embedder.embeddings(\n",
    "    edge_index,\n",
    "    edge_type,\n",
    "    node_features=processed_node_features,\n",
    "    node_features_scalers=embedder.get_last_fit_scalers(),\n",
    "    idx=data.test_idx,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: less than 75% gpu memory available for training. Free: 8260.25 Total: 32510.5\n"
     ]
    }
   ],
   "source": [
    "task_type = \"GPU\" if torch.cuda.is_available() else \"CPU\"\n",
    "clf = CatBoostClassifier(\n",
    "    iterations=10_000,\n",
    "    learning_rate=0.10,\n",
    "    early_stopping_rounds=100,\n",
    "    task_type=task_type,\n",
    "    random_seed=42,\n",
    "    use_best_model=True,\n",
    "    auto_class_weights=\"Balanced\",\n",
    "    verbose=False\n",
    ")\n",
    "clf = clf.fit(\n",
    "    train_embs.cpu().numpy(),\n",
    "    y_train.cpu().numpy(),\n",
    "    eval_set=(val_embs.cpu().numpy(), y_val.cpu().numpy()),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.88      0.88      0.88         8\n",
      "           1       0.97      1.00      0.99        69\n",
      "           2       1.00      0.80      0.89         5\n",
      "           3       0.93      0.81      0.87        16\n",
      "           4       0.89      0.73      0.80        11\n",
      "           5       1.00      1.00      1.00        11\n",
      "           6       1.00      1.00      1.00         3\n",
      "           7       0.70      1.00      0.82        23\n",
      "           8       0.94      0.88      0.91        17\n",
      "           9       0.83      0.60      0.70        25\n",
      "          10       0.55      0.60      0.57        10\n",
      "\n",
      "    accuracy                           0.88       198\n",
      "   macro avg       0.88      0.85      0.86       198\n",
      "weighted avg       0.89      0.88      0.88       198\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(data.test_y.cpu().numpy(), clf.predict(test_embs.cpu().numpy())))"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "d4d1e4263499bec80672ea0156c357c1ee493ec2b1c70f0acce89fc37c4a6abe"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}