examples/notebooks/PGFrames and sematic encoding tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7f47539b",
   "metadata": {},
   "source": [
    "## Prerequisites and installation instructions\n",
    "\n",
    "In order to run this notebook install BlueGraph using:\n",
    "\n",
    " ```\n",
    " pip install bluegraph\n",
    " ```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "desperate-florida",
   "metadata": {},
   "source": [
    "# Introduction to PGFrames and semantic encoding\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fourth-offer",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from nltk.corpus import words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ongoing-senator",
   "metadata": {},
   "outputs": [],
   "source": [
    "from bluegraph import PandasPGFrame\n",
    "from bluegraph.preprocess import ScikitLearnPGEncoder"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "taken-memorabilia",
   "metadata": {},
   "source": [
    "__NB:__ If an nltk error occurs, run the following code (the 'words' corpus needs to be downloaded for semantic encoding of text properties):\n",
    "\n",
    "```\n",
    "import nltk\n",
    "nltk.download('words')\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "advisory-source",
   "metadata": {},
   "source": [
    "## Example 1: small property graph"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "exceptional-signature",
   "metadata": {},
   "source": [
    "Intialize a `PandasPGFrame` given a node and edge list."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "naval-sudan",
   "metadata": {},
   "outputs": [],
   "source": [
    "nodes = [\"Alice\", \"Bob\", \"Eric\", \"John\", \"Anna\", \"Laura\", \"Matt\"]\n",
    "\n",
    "sources = [\n",
    "    \"Alice\", \"Alice\", \"Bob\", \"Bob\", \"Bob\", \"Eric\", \"Anna\", \"Anna\", \"Matt\"\n",
    "]\n",
    "targets = [\n",
    "    \"Bob\", \"Eric\", \"Eric\", \"John\", \"Anna\", \"Anna\", \"Laura\", \"John\", \"John\"\n",
    "]\n",
    "edges = list(zip(sources, targets))\n",
    "\n",
    "frame = PandasPGFrame(nodes=nodes, edges=edges)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "practical-mortgage",
   "metadata": {},
   "source": [
    "Get nodes and edges as lists."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "damaged-winner",
   "metadata": {},
   "outputs": [],
   "source": [
    "frame.nodes()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "italic-incidence",
   "metadata": {},
   "outputs": [],
   "source": [
    "frame.edges()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "allied-portal",
   "metadata": {},
   "source": [
    "Add properties to nodes and edges. Here, all the properties have type `numeric`. Other available types are: `categorical` and `text`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "solved-malawi",
   "metadata": {},
   "outputs": [],
   "source": [
    "age = [25, 9, 70, 42, 26, 35, 36]\n",
    "frame.add_node_properties(\n",
    "    {\n",
    "        \"@id\": nodes,\n",
    "        \"age\": age\n",
    "    }, prop_type=\"numeric\")\n",
    "\n",
    "height = [180, 122, 173, 194, 172, 156, 177]\n",
    "frame.add_node_properties(\n",
    "    {\n",
    "        \"@id\": nodes,\n",
    "        \"height\": height\n",
    "    }, prop_type=\"numeric\")\n",
    "\n",
    "weight = [75, 43, 68, 82, 70, 59, 81]\n",
    "frame.add_node_properties(\n",
    "    {\n",
    "        \"@id\": nodes,\n",
    "        \"weight\": weight\n",
    "    }, prop_type=\"numeric\")\n",
    "\n",
    "\n",
    "weights = [1.0, 2.2, 0.3, 4.1, 1.5, 21.0, 1.0, 2.5, 7.5]\n",
    "edge_weight = pd.DataFrame({\n",
    "    \"@source_id\": sources,\n",
    "    \"@target_id\": targets,\n",
    "    \"distance\": weights\n",
    "})\n",
    "frame.add_edge_properties(edge_weight, prop_type=\"numeric\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "informative-wallet",
   "metadata": {},
   "source": [
    "Get nodes and edges as dataframes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "descending-bangladesh",
   "metadata": {},
   "outputs": [],
   "source": [
    "frame.nodes(raw_frame=True).sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "maritime-seattle",
   "metadata": {},
   "outputs": [],
   "source": [
    "frame.edges(raw_frame=True).sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "leading-lodge",
   "metadata": {},
   "source": [
    "## Example 2: Random graph with a given density\n",
    "\n",
    "In this example we will generate a small random graph given a specified density value (i.e. ratio of edges realized of all possible edges between distinct pairs of nodes)."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "heavy-builder",
   "metadata": {},
   "source": [
    "### Create a PandasPGFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "similar-watershed",
   "metadata": {},
   "outputs": [],
   "source": [
    "N = 70  # number of nodes\n",
    "density = 0.1  # density value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "colonial-parcel",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helper functions for graph generation\n",
    "\n",
    "def generate_targets(nodes, s, density=0.2):\n",
    "    edges = []\n",
    "    for t in nodes:\n",
    "        if s < t:\n",
    "            edge = np.random.choice([0, 1], p=[1 - density, density])\n",
    "            if edge:\n",
    "                \n",
    "                edges.append([s, t])\n",
    "    return edges\n",
    "\n",
    "\n",
    "def random_pgframe(n_nodes, density):\n",
    "    nodes = list(range(n_nodes))\n",
    "\n",
    "    edges = sum(\n",
    "        map(lambda x: generate_targets(nodes, x, density), nodes), [])\n",
    "    edges = pd.DataFrame(\n",
    "        edges, columns=[\"@source_id\", \"@target_id\"])\n",
    "    edges_df = edges.set_index([\"@source_id\", \"@target_id\"])\n",
    "    frame = PandasPGFrame(nodes=nodes, edges=edges_df.index)\n",
    "    return frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "temporal-target",
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_frame = random_pgframe(N, density)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "lovely-marker",
   "metadata": {},
   "source": [
    "Get nodes and edges as dataframes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "developing-colonial",
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_frame.nodes(raw_frame=True).sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "macro-birmingham",
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_frame.edges(raw_frame=True).sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "african-football",
   "metadata": {},
   "source": [
    "### Add node and edge types\n",
    "\n",
    "Here we generate random types for nodes and edges."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "baking-bracket",
   "metadata": {},
   "outputs": [],
   "source": [
    "types = [\"Apple\", \"Orange\", \"Carrot\"]\n",
    "node_types = {\n",
    "    n: np.random.choice(types, p=[0.5, 0.4, 0.1])\n",
    "    for n in range(N)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "spiritual-desert",
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_frame.add_node_types(node_types)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "internal-michigan",
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_frame.nodes(raw_frame=True).sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "tight-village",
   "metadata": {},
   "outputs": [],
   "source": [
    "types = [\"isFriend\", \"isEnemy\"]\n",
    "edge_types = {\n",
    "    e: np.random.choice(types, p=[0.8, 0.2])\n",
    "    for e in graph_frame.edges()\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "adult-celtic",
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_frame.add_edge_types(edge_types)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "restricted-robinson",
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_frame.edges(raw_frame=True).sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cathedral-stock",
   "metadata": {},
   "source": [
    "### Add node and edge properties\n",
    "\n",
    "We add node properties of different data types (`numeric`, `categorical`, `text`) randomly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "afraid-domain",
   "metadata": {},
   "outputs": [],
   "source": [
    "weight = pd.DataFrame(\n",
    "    [\n",
    "        (n, np.random.normal(loc=35, scale=5))\n",
    "        for n in graph_frame.nodes()\n",
    "    ], \n",
    "    columns=[\"@id\", \"weight\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "differential-satellite",
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_frame.add_node_properties(weight, prop_type=\"numeric\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "imposed-train",
   "metadata": {},
   "outputs": [],
   "source": [
    "colors = [\"red\", \"green\", \"blue\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "employed-fraud",
   "metadata": {},
   "outputs": [],
   "source": [
    "colors = pd.DataFrame(\n",
    "    [\n",
    "        (n, np.random.choice(colors))\n",
    "        for n in graph_frame.nodes()\n",
    "    ], \n",
    "    columns=[\"@id\", \"color\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "loving-southwest",
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_frame.add_node_properties(colors, prop_type=\"category\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "young-shirt",
   "metadata": {},
   "outputs": [],
   "source": [
    "desc = pd.DataFrame(\n",
    "    [\n",
    "        (n, ' '.join(random.sample(words.words(), 20)))\n",
    "        for n in graph_frame.nodes()\n",
    "    ], \n",
    "    columns=[\"@id\", \"desc\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "derived-green",
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_frame.add_node_properties(desc, prop_type=\"text\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "worldwide-weapon",
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_frame.nodes(raw_frame=True).sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "featured-oliver",
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_frame._node_prop_types"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "potential-holiday",
   "metadata": {},
   "source": [
    "We add edge properties of different data types (`numeric`, `categorical`, `text`) randomly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "systematic-intensity",
   "metadata": {},
   "outputs": [],
   "source": [
    "years = pd.DataFrame(\n",
    "    [\n",
    "        (s, t, np.random.randint(0, 20))\n",
    "        for s, t in graph_frame.edges()\n",
    "    ], \n",
    "    columns=[\"@source_id\", \"@target_id\", \"n_years\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "canadian-wright",
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_frame.add_edge_properties(years, prop_type=\"numeric\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "hired-leader",
   "metadata": {},
   "outputs": [],
   "source": [
    "shapes = [\"dashed\", \"dotted\", \"solid\"]\n",
    "shapes = pd.DataFrame(\n",
    "    [\n",
    "        (s, t, np.random.choice(shapes))\n",
    "        for s, t, in graph_frame.edges()\n",
    "    ], \n",
    "    columns=[\"@source_id\", \"@target_id\", \"shapes\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "progressive-billy",
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_frame.add_edge_properties(shapes, prop_type=\"category\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "familiar-prophet",
   "metadata": {},
   "outputs": [],
   "source": [
    "desc = pd.DataFrame(\n",
    "    [\n",
    "        (s, t, ' '.join(random.sample(words.words(), 20)))\n",
    "        for s, t, in graph_frame.edges()\n",
    "    ], \n",
    "    columns=[\"@source_id\", \"@target_id\", \"desc\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "indirect-skirt",
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_frame.add_edge_properties(desc, prop_type=\"text\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "optional-result",
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_frame.edges(raw_frame=True).sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dominant-intent",
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_frame._edge_prop_types"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "attended-battlefield",
   "metadata": {},
   "source": [
    "### Perform semantic encoding of properties\n",
    "\n",
    "BlueGraph allows to convert node/edge properties of different data types into numerical vectors."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "driving-stocks",
   "metadata": {},
   "source": [
    "Create a encoder object for homogeneous encoding (properties of all the nodes (edges) are encoded with feature vectors of the same length independently of their type)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "supreme-activation",
   "metadata": {},
   "outputs": [],
   "source": [
    "hom_encoder = ScikitLearnPGEncoder(\n",
    "    node_properties=[\"weight\", \"color\", \"desc\"],\n",
    "    edge_properties=[\"n_years\", \"shapes\", \"desc\"],\n",
    "    edge_features=True,\n",
    "    heterogeneous=False,\n",
    "    encode_types=True,\n",
    "    drop_types=True,\n",
    "    text_encoding=\"tfidf\",\n",
    "    standardize_numeric=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "understood-execution",
   "metadata": {},
   "outputs": [],
   "source": [
    "transformed_frame = hom_encoder.fit_transform(graph_frame)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "clean-geography",
   "metadata": {},
   "outputs": [],
   "source": [
    "transformed_frame.nodes(raw_frame=True).sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "pharmaceutical-eating",
   "metadata": {},
   "source": [
    "We can inspect encoding models for different node and edge properties created by BlueGraph."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "legal-astronomy",
   "metadata": {},
   "outputs": [],
   "source": [
    "hom_encoder._node_encoders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ruled-party",
   "metadata": {},
   "outputs": [],
   "source": [
    "transformed_frame.edges(raw_frame=True).sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "atomic-armstrong",
   "metadata": {},
   "outputs": [],
   "source": [
    "hom_encoder._edge_encoders"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "relevant-ministry",
   "metadata": {},
   "source": [
    "### Convert PGFrames to JSON"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "varied-mexican",
   "metadata": {},
   "outputs": [],
   "source": [
    "json_repr = graph_frame.to_json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "indie-labor",
   "metadata": {},
   "outputs": [],
   "source": [
    "json_repr[\"nodes\"][:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "instrumental-cross",
   "metadata": {},
   "outputs": [],
   "source": [
    "json_repr[\"edges\"][:2]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "stretch-wednesday",
   "metadata": {},
   "source": [
    "Create a new `PandasPGFrame` from the generated representation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eastern-indication",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_frame = PandasPGFrame.from_json(json_repr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "collect-abraham",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "new_frame.nodes(raw_frame=True).sample(5)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "clean_bg",
   "language": "python",
   "name": "clean_bg"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}