notebooks/readme.md examples.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "98d22aea",
   "metadata": {},
   "source": [
    "### What is the sentiment of this film review?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "303edda7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<html lang=\"en\">\n",
       "<script src=\"https://requirejs.org/docs/release/2.3.6/minified/require.js\"></script>\n",
       "<script>\n",
       "    var ecco_url = 'https://storage.googleapis.com/ml-intro/ecco/'\n",
       "    //var ecco_url = 'http://localhost:8000/'\n",
       "\n",
       "    if (window.ecco === undefined) window.ecco = {}\n",
       "\n",
       "    // Setup the paths of the script we'll be using\n",
       "    requirejs.config({\n",
       "        urlArgs: \"bust=\" + (new Date()).getTime(),\n",
       "        nodeRequire: require,\n",
       "        paths: {\n",
       "            d3: \"https://d3js.org/d3.v6.min\", // This is only for use in setup.html and basic.html\n",
       "            \"d3-array\": \"https://d3js.org/d3-array.v2.min\",\n",
       "            jquery: \"https://code.jquery.com/jquery-3.5.1.min\",\n",
       "            ecco: ecco_url + 'js/0.0.6/ecco-bundle.min',\n",
       "            xregexp: 'https://cdnjs.cloudflare.com/ajax/libs/xregexp/3.2.0/xregexp-all.min'\n",
       "        }\n",
       "    });\n",
       "\n",
       "    // Add the css file\n",
       "    //requirejs(['d3'],\n",
       "    //    function (d3) {\n",
       "    //        d3.select('#css').attr('href', ecco_url + 'html/styles.css')\n",
       "    //    })\n",
       "\n",
       "    console.log('Ecco initialize!!')\n",
       "\n",
       "    // returns a 'basic' object. basic.init() selects the html div we'll be\n",
       "    // rendering the html into, adds styles.css to the document.\n",
       "    define('basic', ['d3'],\n",
       "        function (d3) {\n",
       "            return {\n",
       "                init: function (viz_id = null) {\n",
       "                    if (viz_id == null) {\n",
       "                        viz_id = \"viz_\" + Math.round(Math.random() * 10000000)\n",
       "                    }\n",
       "                    // Select the div rendered below, change its id\n",
       "                    const div = d3.select('#basic').attr('id', viz_id),\n",
       "                        div_parent = d3.select('#' + viz_id).node().parentNode\n",
       "\n",
       "                    // Link to CSS file\n",
       "                    d3.select(div_parent).insert('link')\n",
       "                        .attr('rel', 'stylesheet')\n",
       "                        .attr('type', 'text/css')\n",
       "                        .attr('href', ecco_url + 'html/0.0.2/styles.css')\n",
       "\n",
       "                    return viz_id\n",
       "                }\n",
       "            }\n",
       "        }, function (err) {\n",
       "            console.log(err);\n",
       "        }\n",
       "    )\n",
       "</script>\n",
       "\n",
       "<head>\n",
       "    <link id='css' rel=\"stylesheet\" type=\"text/css\">\n",
       "</head>\n",
       "<div id=\"basic\"></div>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "         requirejs( ['basic', 'ecco'], function(basic, ecco){\n",
       "            basic.init('viz_253143') // Python needs to know the viz id. Used for each output token.\n",
       "            window.ecco['viz_253143'] = new ecco.renderOutputSequence({\n",
       "                    parentDiv: 'viz_253143',\n",
       "                    data: {\"tokens\": [{\"token\": \"\", \"is_partial\": false, \"position\": 0, \"token_id\": 3, \"type\": \"input\"}, {\"token\": \"s\", \"is_partial\": true, \"position\": 1, \"token_id\": 7, \"type\": \"input\"}, {\"token\": \"s\", \"is_partial\": true, \"position\": 2, \"token_id\": 7, \"type\": \"input\"}, {\"token\": \"t\", \"is_partial\": true, \"position\": 3, \"token_id\": 17, \"type\": \"input\"}, {\"token\": \"2\", \"is_partial\": true, \"position\": 4, \"token_id\": 357, \"type\": \"input\"}, {\"token\": \"sentence\", \"is_partial\": false, \"position\": 5, \"token_id\": 7142, \"type\": \"input\"}, {\"token\": \":\", \"is_partial\": true, \"position\": 6, \"token_id\": 10, \"type\": \"input\"}, {\"token\": \"Den\", \"is_partial\": false, \"position\": 7, \"token_id\": 3128, \"type\": \"input\"}, {\"token\": \"is\", \"is_partial\": true, \"position\": 8, \"token_id\": 159, \"type\": \"input\"}, {\"token\": \"Ville\", \"is_partial\": false, \"position\": 9, \"token_id\": 15626, \"type\": \"input\"}, {\"token\": \"neuve\", \"is_partial\": true, \"position\": 10, \"token_id\": 26445, \"type\": \"input\"}, {\"token\": \"'\", \"is_partial\": true, \"position\": 11, \"token_id\": 31, \"type\": \"input\"}, {\"token\": \"s\", \"is_partial\": true, \"position\": 12, \"token_id\": 7, \"type\": \"input\"}, {\"token\": \"Dun\", \"is_partial\": false, \"position\": 13, \"token_id\": 6393, \"type\": \"input\"}, {\"token\": \"e\", \"is_partial\": true, \"position\": 14, \"token_id\": 15, \"type\": \"input\"}, {\"token\": \"looks\", \"is_partial\": false, \"position\": 15, \"token_id\": 1416, \"type\": \"input\"}, {\"token\": \"and\", \"is_partial\": false, \"position\": 16, \"token_id\": 11, \"type\": \"input\"}, {\"token\": \"sounds\", \"is_partial\": false, \"position\": 17, \"token_id\": 2993, \"type\": \"input\"}, {\"token\": \"amazing\", \"is_partial\": false, \"position\": 18, \"token_id\": 1237, \"type\": \"input\"}, {\"token\": \"--\", \"is_partial\": false, \"position\": 19, \"token_id\": 1636, \"type\": \"input\"}, {\"token\": \"and\", \"is_partial\": false, \"position\": 20, \"token_id\": 11, \"type\": \"input\"}, {\"token\": \"once\", \"is_partial\": false, \"position\": 21, \"token_id\": 728, \"type\": \"input\"}, {\"token\": \"the\", \"is_partial\": false, \"position\": 22, \"token_id\": 8, \"type\": \"input\"}, {\"token\": \"(\", \"is_partial\": false, \"position\": 23, \"token_id\": 41, \"type\": \"input\"}, {\"token\": \"a\", \"is_partial\": true, \"position\": 24, \"token_id\": 9, \"type\": \"input\"}, {\"token\": \"d\", \"is_partial\": true, \"position\": 25, \"token_id\": 26, \"type\": \"input\"}, {\"token\": \"mitted\", \"is_partial\": true, \"position\": 26, \"token_id\": 16030, \"type\": \"input\"}, {\"token\": \"ly\", \"is_partial\": true, \"position\": 27, \"token_id\": 120, \"type\": \"input\"}, {\"token\": \"slow\", \"is_partial\": false, \"position\": 28, \"token_id\": 2684, \"type\": \"input\"}, {\"token\": \"-\", \"is_partial\": true, \"position\": 29, \"token_id\": 18, \"type\": \"input\"}, {\"token\": \"building\", \"is_partial\": true, \"position\": 30, \"token_id\": 10905, \"type\": \"input\"}, {\"token\": \")\", \"is_partial\": true, \"position\": 31, \"token_id\": 61, \"type\": \"input\"}, {\"token\": \"story\", \"is_partial\": false, \"position\": 32, \"token_id\": 733, \"type\": \"input\"}, {\"token\": \"gets\", \"is_partial\": false, \"position\": 33, \"token_id\": 2347, \"type\": \"input\"}, {\"token\": \"you\", \"is_partial\": false, \"position\": 34, \"token_id\": 25, \"type\": \"input\"}, {\"token\": \"hooked\", \"is_partial\": false, \"position\": 35, \"token_id\": 20949, \"type\": \"input\"}, {\"token\": \",\", \"is_partial\": true, \"position\": 36, \"token_id\": 6, \"type\": \"input\"}, {\"token\": \"you\", \"is_partial\": false, \"position\": 37, \"token_id\": 25, \"type\": \"input\"}, {\"token\": \"'\", \"is_partial\": true, \"position\": 38, \"token_id\": 31, \"type\": \"input\"}, {\"token\": \"ll\", \"is_partial\": true, \"position\": 39, \"token_id\": 195, \"type\": \"input\"}, {\"token\": \"be\", \"is_partial\": false, \"position\": 40, \"token_id\": 36, \"type\": \"input\"}, {\"token\": \"on\", \"is_partial\": false, \"position\": 41, \"token_id\": 30, \"type\": \"input\"}, {\"token\": \"the\", \"is_partial\": false, \"position\": 42, \"token_id\": 8, \"type\": \"input\"}, {\"token\": \"edge\", \"is_partial\": false, \"position\": 43, \"token_id\": 3023, \"type\": \"input\"}, {\"token\": \"of\", \"is_partial\": false, \"position\": 44, \"token_id\": 13, \"type\": \"input\"}, {\"token\": \"your\", \"is_partial\": false, \"position\": 45, \"token_id\": 39, \"type\": \"input\"}, {\"token\": \"seat\", \"is_partial\": false, \"position\": 46, \"token_id\": 3143, \"type\": \"input\"}, {\"token\": \"for\", \"is_partial\": false, \"position\": 47, \"token_id\": 21, \"type\": \"input\"}, {\"token\": \"the\", \"is_partial\": false, \"position\": 48, \"token_id\": 8, \"type\": \"input\"}, {\"token\": \"sequel\", \"is_partial\": false, \"position\": 49, \"token_id\": 19826, \"type\": \"input\"}, {\"token\": \".\", \"is_partial\": true, \"position\": 50, \"token_id\": 5, \"type\": \"input\"}, {\"token\": \"</s>\", \"is_partial\": true, \"position\": 51, \"token_id\": 1, \"type\": \"input\"}]},\n",
       "                    tokenization_config: {\"token_prefix\": \"\\u2581\", \"partial_token_prefix\": \"\"}\n",
       "            \n",
       "            })\n",
       "         }, function (err) {\n",
       "            console.log(err);\n",
       "        })\n",
       "        "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "        // We don't really need these require scripts. But this is to avert\n",
       "        //this code from running before display_input_sequence which DOES require external files\n",
       "        requirejs(['basic', 'ecco'], function(basic, ecco){\n",
       "                console.log('addToken viz_id', 'viz_253143');\n",
       "                window.ecco['viz_253143'].addToken({\"token\": \"<pad>\", \"is_partial\": true, \"token_id\": 0, \"position\": 52, \"type\": \"output\"})\n",
       "                window.ecco['viz_253143'].redraw()\n",
       "        })\n",
       "        "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "        // We don't really need these require scripts. But this is to avert\n",
       "        //this code from running before display_input_sequence which DOES require external files\n",
       "        requirejs(['basic', 'ecco'], function(basic, ecco){\n",
       "                console.log('addToken viz_id', 'viz_253143');\n",
       "                window.ecco['viz_253143'].addToken({\"token\": \"positive\", \"is_partial\": false, \"token_id\": 1465, \"position\": 52, \"type\": \"output\"})\n",
       "                window.ecco['viz_253143'].redraw()\n",
       "        })\n",
       "        "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "!pip install ecco\n",
    "import ecco\n",
    "\n",
    "lm = ecco.from_pretrained('t5-small')\n",
    "review=\"\"\"Denis Villeneuve's Dune looks and sounds amazing -- and once the \n",
    "(admittedly slow-building) story gets you hooked, you'll be on the edge \n",
    "of your seat for the sequel.\"\"\"\n",
    "\n",
    "output = lm.generate(f\"sst2 sentence: {review}\", generate=1, do_sample=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "841af8d6",
   "metadata": {},
   "source": [
    "\n",
    "Use a large language model (T5 in this case) to detect text sentiment. In addition to the sentiment, see the tokens the model broke the text into (which can help debug some edge cases).\n",
    "\n",
    "### Which words in this review lead the model to classify its sentiment as \"negative\"?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2ccc9007",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<html lang=\"en\">\n",
       "<script src=\"https://requirejs.org/docs/release/2.3.6/minified/require.js\"></script>\n",
       "<script>\n",
       "    var ecco_url = 'https://storage.googleapis.com/ml-intro/ecco/'\n",
       "    //var ecco_url = 'http://localhost:8000/'\n",
       "\n",
       "    if (window.ecco === undefined) window.ecco = {}\n",
       "\n",
       "    // Setup the paths of the script we'll be using\n",
       "    requirejs.config({\n",
       "        urlArgs: \"bust=\" + (new Date()).getTime(),\n",
       "        nodeRequire: require,\n",
       "        paths: {\n",
       "            d3: \"https://d3js.org/d3.v6.min\", // This is only for use in setup.html and basic.html\n",
       "            \"d3-array\": \"https://d3js.org/d3-array.v2.min\",\n",
       "            jquery: \"https://code.jquery.com/jquery-3.5.1.min\",\n",
       "            ecco: ecco_url + 'js/0.0.6/ecco-bundle.min',\n",
       "            xregexp: 'https://cdnjs.cloudflare.com/ajax/libs/xregexp/3.2.0/xregexp-all.min'\n",
       "        }\n",
       "    });\n",
       "\n",
       "    // Add the css file\n",
       "    //requirejs(['d3'],\n",
       "    //    function (d3) {\n",
       "    //        d3.select('#css').attr('href', ecco_url + 'html/styles.css')\n",
       "    //    })\n",
       "\n",
       "    console.log('Ecco initialize!!')\n",
       "\n",
       "    // returns a 'basic' object. basic.init() selects the html div we'll be\n",
       "    // rendering the html into, adds styles.css to the document.\n",
       "    define('basic', ['d3'],\n",
       "        function (d3) {\n",
       "            return {\n",
       "                init: function (viz_id = null) {\n",
       "                    if (viz_id == null) {\n",
       "                        viz_id = \"viz_\" + Math.round(Math.random() * 10000000)\n",
       "                    }\n",
       "                    // Select the div rendered below, change its id\n",
       "                    const div = d3.select('#basic').attr('id', viz_id),\n",
       "                        div_parent = d3.select('#' + viz_id).node().parentNode\n",
       "\n",
       "                    // Link to CSS file\n",
       "                    d3.select(div_parent).insert('link')\n",
       "                        .attr('rel', 'stylesheet')\n",
       "                        .attr('type', 'text/css')\n",
       "                        .attr('href', ecco_url + 'html/0.0.2/styles.css')\n",
       "\n",
       "                    return viz_id\n",
       "                }\n",
       "            }\n",
       "        }, function (err) {\n",
       "            console.log(err);\n",
       "        }\n",
       "    )\n",
       "</script>\n",
       "\n",
       "<head>\n",
       "    <link id='css' rel=\"stylesheet\" type=\"text/css\">\n",
       "</head>\n",
       "<div id=\"basic\"></div>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "             requirejs(['basic', 'ecco'], function(basic, ecco){\n",
       "                const viz_id = basic.init()\n",
       "                console.log(viz_id)\n",
       "                // ecco.interactiveTokens(viz_id, {})\n",
       "                window.ecco[viz_id] = new ecco.MinimalHighlighter({\n",
       "                    parentDiv: viz_id,\n",
       "                    data: {\"tokens\": [{\"token\": \"\", \"token_id\": 3, \"is_partial\": false, \"type\": \"input\", \"value\": \"0.0\", \"position\": 0}, {\"token\": \"s\", \"token_id\": 7, \"is_partial\": true, \"type\": \"input\", \"value\": \"0.0\", \"position\": 1}, {\"token\": \"s\", \"token_id\": 7, \"is_partial\": true, \"type\": \"input\", \"value\": \"0.0\", \"position\": 2}, {\"token\": \"t\", \"token_id\": 17, \"is_partial\": true, \"type\": \"input\", \"value\": \"0.0\", \"position\": 3}, {\"token\": \"2\", \"token_id\": 357, \"is_partial\": true, \"type\": \"input\", \"value\": \"0.0\", \"position\": 4}, {\"token\": \"sentence\", \"token_id\": 7142, \"is_partial\": false, \"type\": \"input\", \"value\": \"0.0\", \"position\": 5}, {\"token\": \":\", \"token_id\": 10, \"is_partial\": true, \"type\": \"input\", \"value\": \"0.0\", \"position\": 6}, {\"token\": \"I\", \"token_id\": 27, \"is_partial\": false, \"type\": \"input\", \"value\": \"0.0\", \"position\": 7}, {\"token\": \"have\", \"token_id\": 43, \"is_partial\": false, \"type\": \"input\", \"value\": \"0.0\", \"position\": 8}, {\"token\": \"\", \"token_id\": 3, \"is_partial\": false, \"type\": \"input\", \"value\": \"0.0\", \"position\": 9}, {\"token\": \"a\", \"token_id\": 9, \"is_partial\": true, \"type\": \"input\", \"value\": \"0.0\", \"position\": 10}, {\"token\": \"well\", \"token_id\": 168, \"is_partial\": false, \"type\": \"input\", \"value\": \"0.0\", \"position\": 11}, {\"token\": \"-\", \"token_id\": 18, \"is_partial\": true, \"type\": \"input\", \"value\": \"0.0\", \"position\": 12}, {\"token\": \"document\", \"token_id\": 28244, \"is_partial\": true, \"type\": \"input\", \"value\": \"0.0\", \"position\": 13}, {\"token\": \"e\", \"token_id\": 15, \"is_partial\": true, \"type\": \"input\", \"value\": \"0.0\", \"position\": 14}, {\"token\": \"d\", \"token_id\": 26, \"is_partial\": true, \"type\": \"input\", \"value\": \"0.0\", \"position\": 15}, {\"token\": \"weakness\", \"token_id\": 18724, \"is_partial\": false, \"type\": \"input\", \"value\": \"0.0\", \"position\": 16}, {\"token\": \"for\", \"token_id\": 21, \"is_partial\": false, \"type\": \"input\", \"value\": \"0.0\", \"position\": 17}, {\"token\": \"sci\", \"token_id\": 17201, \"is_partial\": false, \"type\": \"input\", \"value\": \"0.0\", \"position\": 18}, {\"token\": \"-\", \"token_id\": 18, \"is_partial\": true, \"type\": \"input\", \"value\": \"0.0\", \"position\": 19}, {\"token\": \"f\", \"token_id\": 89, \"is_partial\": true, \"type\": \"input\", \"value\": \"0.0\", \"position\": 20}, {\"token\": \"i\", \"token_id\": 23, \"is_partial\": true, \"type\": \"input\", \"value\": \"0.0\", \"position\": 21}, {\"token\": \"and\", \"token_id\": 11, \"is_partial\": false, \"type\": \"input\", \"value\": \"0.0\", \"position\": 22}, {\"token\": \"expected\", \"token_id\": 1644, \"is_partial\": false, \"type\": \"input\", \"value\": \"0.0\", \"position\": 23}, {\"token\": \"Dun\", \"token_id\": 6393, \"is_partial\": false, \"type\": \"input\", \"value\": \"0.0\", \"position\": 24}, {\"token\": \"e\", \"token_id\": 15, \"is_partial\": true, \"type\": \"input\", \"value\": \"0.0\", \"position\": 25}, {\"token\": \"to\", \"token_id\": 12, \"is_partial\": false, \"type\": \"input\", \"value\": \"0.0\", \"position\": 26}, {\"token\": \"feed\", \"token_id\": 3305, \"is_partial\": false, \"type\": \"input\", \"value\": \"0.0\", \"position\": 27}, {\"token\": \"my\", \"token_id\": 82, \"is_partial\": false, \"type\": \"input\", \"value\": \"0.0\", \"position\": 28}, {\"token\": \"soul\", \"token_id\": 3668, \"is_partial\": false, \"type\": \"input\", \"value\": \"0.0\", \"position\": 29}, {\"token\": \".\", \"token_id\": 5, \"is_partial\": true, \"type\": \"input\", \"value\": \"0.0\", \"position\": 30}, {\"token\": \"I\", \"token_id\": 27, \"is_partial\": false, \"type\": \"input\", \"value\": \"0.0\", \"position\": 31}, {\"token\": \"didn\", \"token_id\": 737, \"is_partial\": false, \"type\": \"input\", \"value\": \"0.0\", \"position\": 32}, {\"token\": \"'\", \"token_id\": 31, \"is_partial\": true, \"type\": \"input\", \"value\": \"0.0\", \"position\": 33}, {\"token\": \"t\", \"token_id\": 17, \"is_partial\": true, \"type\": \"input\", \"value\": \"0.0\", \"position\": 34}, {\"token\": \"expect\", \"token_id\": 1672, \"is_partial\": false, \"type\": \"input\", \"value\": \"0.0\", \"position\": 35}, {\"token\": \"it\", \"token_id\": 34, \"is_partial\": false, \"type\": \"input\", \"value\": \"0.0\", \"position\": 36}, {\"token\": \"to\", \"token_id\": 12, \"is_partial\": false, \"type\": \"input\", \"value\": \"0.0\", \"position\": 37}, {\"token\": \"entirely\", \"token_id\": 4585, \"is_partial\": false, \"type\": \"input\", \"value\": \"0.0\", \"position\": 38}, {\"token\": \"blow\", \"token_id\": 6019, \"is_partial\": false, \"type\": \"input\", \"value\": \"0.0\", \"position\": 39}, {\"token\": \"my\", \"token_id\": 82, \"is_partial\": false, \"type\": \"input\", \"value\": \"0.0\", \"position\": 40}, {\"token\": \"mind\", \"token_id\": 809, \"is_partial\": false, \"type\": \"input\", \"value\": \"0.0\", \"position\": 41}, {\"token\": \".\", \"token_id\": 5, \"is_partial\": true, \"type\": \"input\", \"value\": \"0.0\", \"position\": 42}, {\"token\": \"</s>\", \"token_id\": 1, \"is_partial\": true, \"type\": \"input\", \"value\": \"0.0\", \"position\": 43}, {\"token\": \"<pad>\", \"token_id\": 0, \"is_partial\": true, \"type\": \"output\", \"value\": \"0.0\", \"position\": 44}, {\"token\": \"negative\", \"token_id\": 2841, \"is_partial\": false, \"type\": \"output\", \"value\": \"0\", \"position\": 45}], \"attributions\": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.015366007730092928, 0.01080238781296824, 0.006555946170426004, 0.010511936850245894, 0.010819682866485133, 0.006844987607900387, 0.020658293022999482, 0.006296534427043782, 0.010746173953253644, 0.053511958015525406, 0.0191044238209213, 0.01732939294141331, 0.006677608375787354, 0.005542894937869959, 0.006098651460333583, 0.010431829408538517, 0.03813108944213731, 0.016817739782625886, 0.007239562440818059, 0.013129857479506924, 0.023573667254844183, 0.012310466506667122, 0.021918936427546172, 0.01544644939260225, 0.012138432623264809, 0.025486921542021913, 0.021626885586587718, 0.02884554139095014, 0.051023453347653, 0.02008861105220378, 0.015324896740912057, 0.025577892380170466, 0.03673365271098042, 0.012070500463006385, 0.01515605255148629, 0.01877452266354365, 0.0, 0.0]]},\n",
       "                    preset: 'viridis',\n",
       "                    tokenization_config: {\"token_prefix\": \"\\u2581\", \"partial_token_prefix\": \"\"}\n",
       "\n",
       "             })\n",
       "\n",
       "             window.ecco[viz_id].init();\n",
       "             window.ecco[viz_id].selectFirstToken();\n",
       "\n",
       "             }, function (err) {\n",
       "                console.log(err);\n",
       "            })"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import ecco\n",
    "lm = ecco.from_pretrained('t5-small', verbose=False)\n",
    "review=\"\"\"I have a well-documented weakness for sci-fi and expected Dune \n",
    "to feed my soul. I didn't expect it to entirely blow my mind.\"\"\"\n",
    "\n",
    "output = lm.generate(f\"sst2 sentence: {review}\", generate=1, do_sample=False, attribution=['ig'])\n",
    "output.primary_attributions(attr_method='ig', ignore_tokens=[0,1,2,3,4,5,6,43,44])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "54463ec8",
   "metadata": {},
   "source": [
    "\n",
    "Feature attribution using Integrated Gradients helps you explore model decisions. In this case, switching \"weakness\" to \"inclination\" allows the model to correctly switch the prediction to *positive*.\n",
    "\n",
    "### Explore the world knowledge of GPT models by posing fill-in-the blank questions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4fb43dac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<html lang=\"en\">\n",
       "<script src=\"https://requirejs.org/docs/release/2.3.6/minified/require.js\"></script>\n",
       "<script>\n",
       "    var ecco_url = 'https://storage.googleapis.com/ml-intro/ecco/'\n",
       "    //var ecco_url = 'http://localhost:8000/'\n",
       "\n",
       "    if (window.ecco === undefined) window.ecco = {}\n",
       "\n",
       "    // Setup the paths of the script we'll be using\n",
       "    requirejs.config({\n",
       "        urlArgs: \"bust=\" + (new Date()).getTime(),\n",
       "        nodeRequire: require,\n",
       "        paths: {\n",
       "            d3: \"https://d3js.org/d3.v6.min\", // This is only for use in setup.html and basic.html\n",
       "            \"d3-array\": \"https://d3js.org/d3-array.v2.min\",\n",
       "            jquery: \"https://code.jquery.com/jquery-3.5.1.min\",\n",
       "            ecco: ecco_url + 'js/0.0.6/ecco-bundle.min',\n",
       "            xregexp: 'https://cdnjs.cloudflare.com/ajax/libs/xregexp/3.2.0/xregexp-all.min'\n",
       "        }\n",
       "    });\n",
       "\n",
       "    // Add the css file\n",
       "    //requirejs(['d3'],\n",
       "    //    function (d3) {\n",
       "    //        d3.select('#css').attr('href', ecco_url + 'html/styles.css')\n",
       "    //    })\n",
       "\n",
       "    console.log('Ecco initialize!!')\n",
       "\n",
       "    // returns a 'basic' object. basic.init() selects the html div we'll be\n",
       "    // rendering the html into, adds styles.css to the document.\n",
       "    define('basic', ['d3'],\n",
       "        function (d3) {\n",
       "            return {\n",
       "                init: function (viz_id = null) {\n",
       "                    if (viz_id == null) {\n",
       "                        viz_id = \"viz_\" + Math.round(Math.random() * 10000000)\n",
       "                    }\n",
       "                    // Select the div rendered below, change its id\n",
       "                    const div = d3.select('#basic').attr('id', viz_id),\n",
       "                        div_parent = d3.select('#' + viz_id).node().parentNode\n",
       "\n",
       "                    // Link to CSS file\n",
       "                    d3.select(div_parent).insert('link')\n",
       "                        .attr('rel', 'stylesheet')\n",
       "                        .attr('type', 'text/css')\n",
       "                        .attr('href', ecco_url + 'html/0.0.2/styles.css')\n",
       "\n",
       "                    return viz_id\n",
       "                }\n",
       "            }\n",
       "        }, function (err) {\n",
       "            console.log(err);\n",
       "        }\n",
       "    )\n",
       "</script>\n",
       "\n",
       "<head>\n",
       "    <link id='css' rel=\"stylesheet\" type=\"text/css\">\n",
       "</head>\n",
       "<div id=\"basic\"></div>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "         requirejs( ['basic', 'ecco'], function(basic, ecco){\n",
       "            basic.init('viz_859508') // Python needs to know the viz id. Used for each output token.\n",
       "            window.ecco['viz_859508'] = new ecco.renderOutputSequence({\n",
       "                    parentDiv: 'viz_859508',\n",
       "                    data: {\"tokens\": [{\"token\": \"Heath\", \"is_partial\": false, \"position\": 0, \"token_id\": 25846, \"type\": \"input\"}, {\"token\": \"row\", \"is_partial\": true, \"position\": 1, \"token_id\": 808, \"type\": \"input\"}, {\"token\": \"airport\", \"is_partial\": false, \"position\": 2, \"token_id\": 9003, \"type\": \"input\"}, {\"token\": \"is\", \"is_partial\": false, \"position\": 3, \"token_id\": 318, \"type\": \"input\"}, {\"token\": \"located\", \"is_partial\": false, \"position\": 4, \"token_id\": 5140, \"type\": \"input\"}, {\"token\": \"in\", \"is_partial\": false, \"position\": 5, \"token_id\": 287, \"type\": \"input\"}, {\"token\": \"the\", \"is_partial\": false, \"position\": 6, \"token_id\": 262, \"type\": \"input\"}, {\"token\": \"city\", \"is_partial\": false, \"position\": 7, \"token_id\": 1748, \"type\": \"input\"}, {\"token\": \"of\", \"is_partial\": false, \"position\": 8, \"token_id\": 286, \"type\": \"input\"}]},\n",
       "                    tokenization_config: {\"token_prefix\": \"\\u0120\", \"partial_token_prefix\": \"\"}\n",
       "            \n",
       "            })\n",
       "         }, function (err) {\n",
       "            console.log(err);\n",
       "        })\n",
       "        "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "        // We don't really need these require scripts. But this is to avert\n",
       "        //this code from running before display_input_sequence which DOES require external files\n",
       "        requirejs(['basic', 'ecco'], function(basic, ecco){\n",
       "                console.log('addToken viz_id', 'viz_859508');\n",
       "                window.ecco['viz_859508'].addToken({\"token\": \"London\", \"is_partial\": false, \"token_id\": 3576, \"position\": 9, \"type\": \"output\"})\n",
       "                window.ecco['viz_859508'].redraw()\n",
       "        })\n",
       "        "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import ecco\n",
    "lm = ecco.from_pretrained('gpt2')\n",
    "prompt=\"\"\" Heathrow airport is located in the city of\"\"\"\n",
    "\n",
    "output = lm.generate(prompt, generate=1, do_sample=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "94416bbd",
   "metadata": {},
   "source": [
    "\n",
    "Does GPT2 know where Heathrow Airport is? Yes. It does.\n",
    "\n",
    "### What other cities/words did the model consider in addition to London?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a65ab8fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import ecco\n",
    "lm = ecco.from_pretrained('gpt2', verbose=False)\n",
    "prompt=\"\"\" Heathrow airport is located in the city of\"\"\"\n",
    "\n",
    "output = lm.generate(prompt, generate=1, do_sample=False, attribution=['ig'])\n",
    "output.primary_attributions(attr_method='ig')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89d1acc5",
   "metadata": {},
   "source": [
    "\n",
    "Visuals the candidate output tokens and their probability scores.\n",
    "\n",
    "### Which input words lead it to think of London?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "96f36843",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<html lang=\"en\">\n",
       "<script src=\"https://requirejs.org/docs/release/2.3.6/minified/require.js\"></script>\n",
       "<script>\n",
       "    var ecco_url = 'https://storage.googleapis.com/ml-intro/ecco/'\n",
       "    //var ecco_url = 'http://localhost:8000/'\n",
       "\n",
       "    if (window.ecco === undefined) window.ecco = {}\n",
       "\n",
       "    // Setup the paths of the script we'll be using\n",
       "    requirejs.config({\n",
       "        urlArgs: \"bust=\" + (new Date()).getTime(),\n",
       "        nodeRequire: require,\n",
       "        paths: {\n",
       "            d3: \"https://d3js.org/d3.v6.min\", // This is only for use in setup.html and basic.html\n",
       "            \"d3-array\": \"https://d3js.org/d3-array.v2.min\",\n",
       "            jquery: \"https://code.jquery.com/jquery-3.5.1.min\",\n",
       "            ecco: ecco_url + 'js/0.0.6/ecco-bundle.min',\n",
       "            xregexp: 'https://cdnjs.cloudflare.com/ajax/libs/xregexp/3.2.0/xregexp-all.min'\n",
       "        }\n",
       "    });\n",
       "\n",
       "    // Add the css file\n",
       "    //requirejs(['d3'],\n",
       "    //    function (d3) {\n",
       "    //        d3.select('#css').attr('href', ecco_url + 'html/styles.css')\n",
       "    //    })\n",
       "\n",
       "    console.log('Ecco initialize!!')\n",
       "\n",
       "    // returns a 'basic' object. basic.init() selects the html div we'll be\n",
       "    // rendering the html into, adds styles.css to the document.\n",
       "    define('basic', ['d3'],\n",
       "        function (d3) {\n",
       "            return {\n",
       "                init: function (viz_id = null) {\n",
       "                    if (viz_id == null) {\n",
       "                        viz_id = \"viz_\" + Math.round(Math.random() * 10000000)\n",
       "                    }\n",
       "                    // Select the div rendered below, change its id\n",
       "                    const div = d3.select('#basic').attr('id', viz_id),\n",
       "                        div_parent = d3.select('#' + viz_id).node().parentNode\n",
       "\n",
       "                    // Link to CSS file\n",
       "                    d3.select(div_parent).insert('link')\n",
       "                        .attr('rel', 'stylesheet')\n",
       "                        .attr('type', 'text/css')\n",
       "                        .attr('href', ecco_url + 'html/0.0.2/styles.css')\n",
       "\n",
       "                    return viz_id\n",
       "                }\n",
       "            }\n",
       "        }, function (err) {\n",
       "            console.log(err);\n",
       "        }\n",
       "    )\n",
       "</script>\n",
       "\n",
       "<head>\n",
       "    <link id='css' rel=\"stylesheet\" type=\"text/css\">\n",
       "</head>\n",
       "<div id=\"basic\"></div>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "         requirejs(['basic', 'ecco'], function(basic, ecco){\n",
       "            const viz_id = basic.init()\n",
       "\n",
       "\n",
       "            let pred = new ecco.LayerPredictions({\n",
       "                parentDiv: viz_id,\n",
       "                data:[[{\"token\": \" London\", \"prob\": \"0.10360387\", \"ranking\": 1, \"layer\": 11}, {\"token\": \" Birmingham\", \"prob\": \"0.07902661\", \"ranking\": 2, \"layer\": 11}, {\"token\": \" Manchester\", \"prob\": \"0.044098165\", \"ranking\": 3, \"layer\": 11}, {\"token\": \" Liverpool\", \"prob\": \"0.018532543\", \"ranking\": 4, \"layer\": 11}, {\"token\": \" Bristol\", \"prob\": \"0.018447196\", \"ranking\": 5, \"layer\": 11}, {\"token\": \" Glasgow\", \"prob\": \"0.017188776\", \"ranking\": 6, \"layer\": 11}, {\"token\": \" W\", \"prob\": \"0.013818842\", \"ranking\": 7, \"layer\": 11}, {\"token\": \" Leeds\", \"prob\": \"0.012471717\", \"ranking\": 8, \"layer\": 11}, {\"token\": \" Edinburgh\", \"prob\": \"0.011846515\", \"ranking\": 9, \"layer\": 11}, {\"token\": \" L\", \"prob\": \"0.01184525\", \"ranking\": 10, \"layer\": 11}]]\n",
       "            })\n",
       "            pred.init()\n",
       "         }, function (err) {\n",
       "            console.log(viz_id, err);\n",
       "        })"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "output.layer_predictions(position=9, layer=11)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2963976f",
   "metadata": {},
   "source": [
    "\n",
    "### At which layers did the model gather confidence that London is the right answer?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7d61e231",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\msx\\pycharmprojects\\ecco\\src\\ecco\\output.py:473: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  token_id = torch.tensor(self.token_ids[0][offset + j])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAO0AAAGxCAYAAACOdjG3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABTsUlEQVR4nO3dd1gU19fA8S+9KwIqAjbEhhXB3kBEjQWixh4FC2rs0WjUmMQSo1GjYskv9hY1tkTUxBZ7LEGwxd5QFEvEAgrS5/2DsC8bQBbc3dmF+8mzT2Bm7uxZHs/emTsz9xhIkiQhCILeMJQ7AEEQ8kckrSDoGZG0gqBnRNIKgp4RSSsIekYkrSDoGZG0gqBnRNIKgp4RSSvojMTERGbOnMm9e/fkDkWnGYg7olRnb29PuXLl5A5DZzx48ICYmBi17OvZs2d069aNpk2bsmfPHpYtW0ajRo3Usu/CxljuAPRJuXLliNi4Tu4wdMJr5/L4+vqqZV/Xr1+nd+/ezJo1i7Zt2zJ48GC6d+/O2LFj6dGjh1reozARh8eCrA4dOkSvXr3YsGEDbdu2BaB8+fIcPHiQ9evXM3PmTMTBoDKRtIJsVq5cybRp09i3bx81atRQWlesWDFCQ0N5/Pgx/fv3JykpSaYodY9IWkHr0tPTmTBhAgcPHmT//v2ULl06x+2MjY1ZsmQJnp6efPDBBzx//lzLkeomkbSCViUkJNCtWzfMzMzYvHkzFhYWebYZOXIk48aNw8/Pj5s3b2ohSt0mklbQmsePH+Pn54e/vz8zZszA0FD1f34dOnRgzZo1dO/enWPHjmkwSt0nklbQiosXL9K+fXu+/fZbAgMDC7SPOnXqsHfvXiZPnsy6dUV3FF8tSRsUFETHjh3VsSutiIqKolOnTlhZWeHg4MCoUaNITk7Wehwbf9/LR599Tu1uvZi0aEmO2yzdso3qnbtx6uKlbOtevX5Nk8AB9Jk0Jd9ttem3335jwIABbN26lZYtW77XvsqUKcOBAwfYtWsXU6ZMIT09XU1R6o9C29OmpqbmeKkgLS2NDh068Pr1a06cOMHmzZvZvn0748aN03qMJUvYMbRbV7r4+uS4PurxE/afOk3JEiVyXP/9+o24urgUqK22LF68mPnz53Pw4EEqV66sln1aWVmxbds2UlJS6NmzJ2/fvlXLfvWFVpJ2/vz51K5dGysrK5ydnRk0aBCvXr0CID4+nmLFirF9+3alNgcPHsTExISnT58CEB0dTc+ePSlRogQlSpSgQ4cO3Lp1S7H91KlTqVmzJmvXrqVSpUqYmZkRHx+fLZYDBw5w5coVNmzYQL169fDz82POnDmsWLGCuLg4zf0RctCmcUNaN2yArY1NjutnrFjJuH4fY2Kc/R6Y89dvcCsqii6tck74d7XVhtTUVEaOHElERAR79+7Fzs5Orfs3NDTku+++o02bNrRp00bx76Qo0ErSGhoasnDhQq5cucKmTZsICwtj5MiRQMa3Zq9evVi9erVSm9WrV9OxY0dKly5NQkICPj4+mJubc+zYMU6fPk2ZMmVo3bo1CQkJijaRkZFs2rSJbdu2cfHiRczNzbPFcvr0aapXr07ZsmUVy9q2bUtSUhIREREa+gvk376TpzE1NqGlZ71s69LS0pixYhVfDh6IgUH+2mpDXFwcH374IU5OTqxZswZTU1ONvdegQYOYOnUq7dq14/Llyxp7H12ilaQdM2YMrVq1okKFCrRs2ZI5c+awdetWxflIcHAwBw4cIDo6GoCXL1+yc+dOBg4cCMDPP/+MJEmsWbOG2rVrU61aNZYtW8abN2/Ys2eP4n2Sk5MVPWjNmjUxzqGXefLkSbbrgg4ODhgZGfHkyRNN/QnyJf7tWxZs3MTkQf1zXL/ht73UruxGjUqV8t1W06KiovDz8yMwMJBJkyZhkNO3ipr5+vqyefNm+vXrx/79+zX+fnLTStIePnwYPz8/XFxcsLGxoUuXLiQnJyuSxMvLi1q1ailGBDdt2oSdnR0ffPABABEREURGRmJjY4O1tTXW1tYUL16cly9fcufOHcX7uLi45HqhvqCWL1+Ol5cXXl5eWru4v+Tnrfi3bIFzqVLZ1v3z4gU//fY7Y/r0yndbTTt//jz+/v4sWrSIbt26afW9q1Wrxv79+5k1axYrVqzQ6ntrm8ZPeO7fv0+HDh0IDg5m+vTp2Nvbc+7cOXr16qU0Yjto0CBCQkKYPHkyq1evJjAwECMjIyDjDpq6devy888/Z9t/1nMlKyurPONxdHTk5MmTSstiYmJIS0vD0dEx2/aDBw9m8ODBAHh4eKj2od/TmUt/8+T5C37el9FrvIiLY+y8+Qzs/CEVnZ149vIVnUZ9CkBicjJJyck07z+IoyuXvbNtcJcPNRr3ihUrmD9/Pg0bNtTo++SmZMmS7Nu3j+rVq3P27Fk6depEp06dZIlFkzSetOHh4SQnJ7NgwQJFEmY9pM3Up08fxo8fz5IlSzh37pxSgtarV4/Nmzfj4OCAra3te8XTuHFjvvnmGx4+fIjLvyOvBw8exMzMDE9Pz/fad36lpqWRlpZGeno6aenpJCUnY2RkxOppX5OalqrYrvv4SXzeP5Dm9epiYmzMH8uWKtbt/fMUv534kyWTJuTZVhss374h/VGUVt4rJ6ZkfJEvX75cthg0TW1JGxcXx4ULF5SW2draUrlyZdLT01m4cCFdunThzJkzLFy4MFt7W1tbunXrxrhx42jRooXS5YE+ffowb948AgICmD59OuXKlePBgweEhoYydOjQfF1KaNOmDTVq1KBfv358//33PH/+nPHjxxMcHEyxYsUK+vEL5MdtO1i6ZZvi993HTjC8RzdG9OyutJ2hoSHFrK2w+veWv6yXcWysLDE2MlIsK1HM5p1tNU769yVojNqS9sSJE9kOH7t27cr27dsJCQnhu+++Y8qUKTRp0oR58+bl+JzkwIEDWb9+vWIAKpOlpSXHjx9n4sSJdOvWjdjYWJycnPDx8aFEPq9DGhkZ8dtvvzFs2DCaNm2KhYUFffr0Ye7cufn/0O9pRM/u2RI0J4eW/5Drus6tfOicy2WfvNoK+kmnZq7YsmULQ4YM4dGjR1haWsodTjYeHh7iIfh/ZT4EHx4erlg2bNgw+rVvSwOPuvIFBtT376JTl+/UTSdmrkhISODJkyd8++23BAcH62TCCioSh8capxO3Mc6ZM4eqVatiZ2fHl19+KXc4gqDTdCJpp06dSkpKCkeOHNH6YJCgbhJIMr8KOZ1IWkFQp9jYWAYPHszu3bvlDkUjdOKcVhDUqXjx4oX6Oq3oaQVBz4ieNp9eO5eXOwTdJlEkzivlJHpaQdAzoqfNJ5sHkXKHoBNel60odwhFlkhaQb2KyGUXOYnDY0HQM6KnFdRM3MeoaSJpBfUSOatx4vBYEPSMSFqh0BG3MaogKCiImJiYHKeR0UWjR4/m5MmTXL58GUdHR+7duyd3SIWI/KPHhf02xkJ7TpuamoqRkVGOU3imp6cTGBjI33//zYEDB2SILsPG3/fx69Fj3LwfRYdmTZk1chgAtx88ZOLipTx4kjEBt7trRb4Y2B+3shlzWq3auYudR4/x6FkMJYrZ0KttGwZ+6K/Yr+/QETyPfYXRvwWu6latyqqvvtDypxM0pchVGICMUhUjR46kSpUqmvnAKippV4KhXbtkqxJQyq4EIZ99ypl1qzi1ZiWt6nsxbn6IYr2ExOxRw/lr/WpWTJnMxr37+e1P5Rkmf5g4gYiN64nYuF67CSv3Y3lF4BpxkaswoEvaNGpI64b1sbWxVlpezMoK51KlMDAwQELC0NCQqCwTqQ/6MIAarq4YGxlR0dkJ3wZenL9+Q9vhCzLRyuHxmDFjFD9XqFCBOXPmEBAQwLp16zA0NCQ4OJhGjRoRHR2Ns7OzosLAtm0ZMxVmrTCQebi7bNkySpUqxZ49e+jePWNytMwKA+qesFwuDfr2JyExkXRJYmTPnCf/liSJiGvX6e7XWmn5hJAlpEvpVK9YkfH9+lCtQgUtRKyISovvVfRoJWkPHz7MrFmzuHbtGrGxsaSlpSkqDDg5OSlVGJg8efI7KwxklZCQoJUKA5mDGtqqMJApbMMaEhIT2Xn0GE4lS+a4zZIt20hPT6dLK2/FsrljRuBe0RUJiQ2//U7wjG/5bdECiqkwmft7E9dpNU7jh8eZFQaqV6/Otm3biIiIUBwK/7fCwNq1awFyrTBw4cIFpdfNmzcZMmSIYh+qVBjIr8GDBxMeHk54eDj29vZq339eLM3N6dnGj4mLlvI8NlZp3cbf9xF67Dg/fjERUxMTxfJ61aphbmaKhZkZg7t0xsbSioir17UUsaQDr8KtyFUY0EfpkkRichJPn7/AvnhxAHYcOsKKX0PZMGMqjnl8mRgYZAxeaUXRyBtZFbkKAwC3b9/mzZs3PHr0iOTkZEXc7u7uGi3L+F+5lQX56/IVStjYULV8ed4mJRKyeQvFrKyp5OIMwO7jJ1i4aTNrp31FWUfl04FHz2J48jyGmpXckKR0fvp9Hy9fv8ajWlWtfS5Bs4pchQHIOBQ/duyY4vfMuCMjI6mgxQGbH7f/wtKt/3+pa/fxEwzv/hFuZV2YuXINT188x8zUlFpubqyYMgmzf79QQjZv4dXrN3T/fLKibacWzZk6JJj4t2+ZtmwVD54+xdTEhOoVK7D8i0mUyKVwtfqJrlbTRIWBfPDw8CBiwxq5w9AJr8tWzLnCQCtvGtSpLWNkUL9nH1FhQNNEhQFBUJ1OPDAgKgwUJnKPHOvMgaPG6ETSigoDhYzIWY3SiaQVChO5M1YSj+YJQr7oQG8nHs0ThHwrAseoMhKHx4KgZ0RPm09iku486MDhcWEnkjafrK9fljsEnfCmWs1c1ois1TRxeCwIekb0tIJ6iY5W40TSCmomslbTRNIK6iVyVuNE0gpqJrJW08RAlCDoGbUkbVBQEB07dlTHrjTu4sWL9OrVi7Jly2JhYUHVqlWZM2cO6enpcodWSEiIeY81q9AeHudWYSAiIoKSJUuyYcMGypUrR1hYGMHBwaSmpjJ58uRc9qZdv5/5ix9+3cXj589xsC3OzOCBeFWtwukrV/lm/U88fv6C2pVcmRk8AGcHBwD2/hXGhv0HuR71gFquFVk3+XOZP4WgKUWuwsCAAQNYtGgR3t7euLq60rNnTz755BN27NihuT9APpy6fIXvt2xnZvAAzi7/gfWTJ1K2ZElevn7N6EVLGdm1M6d/WEyNChUYt/RHRbviVlb0bevHoI7tZYxe0AZRYYCMSekKMteUJiz5ZSfDPuxEHbdKGBoaUtquBKXtSnAwPAI3ZyfaNaiPmakJw7sEcCPqAXcfPQagSc0afNCwAaXknq1SQhwea1iRrzBw7tw51q5dy8aNG9X3gQsoLT2dy5H38PGoS9vPJpKckoKvpwef9ezO7ehHVC1XVrGtpZkZZUuV4nZ0NK5OZWSMWtA2rfS0hw8fxs/PDxcXF2xsbOjSpYuiwgCgVGEAeGeFAWtra6ytrSlevDgvX758rwoDN27coEOHDowZM4auXbvmuM3y5cvx8vLCy8tL4xUGnsfGkpqWxoGz4fw0ZSK/fDOVa/ej+DF0NwmJidhYWChtb2NpQXxiokZjyi9JkmR/FXZFtsLA9evX8fb2pmfPnsyePTvX7bRZYSBzitQ+fq0paWtLCRsbAtu14fjFv7E0N+fNW+UEffP2LVY6XmRMUL8iWWHg6tWrtGrViu7du7NgwYL33p+6FLeywtGuBFkHvA3I+MXN2YnQP08plickJfHgn2e4OTtrO0xBZmrraTMrDGR93bt3T6nCQGRkJJs3by5QhYHSpUsTEBDAsWPHiIyM5Pjx44wbN05pBFkVV65cwcfHB29vbyZPnsyTJ08UL13QuXkzNh48xPO4OGLj41m3/wDedWvT2rMetx5Gc+BsOEnJKfxv5y6qlHVRnM9mVChIITU9jXRJIik5hZTUVO1/ALkHoYrA4XGRqzCwbds2/vnnH7Zs2cKWLVuU1unC+dDQgE68fP2G9hMmYWpiQrsG9Rni3wkzUxMWjhzGzA0b+fzHFdSu5Mr3w4Yq2u06eYovVvz/CLzHoCF82Kwp3w4emNPbCHpMVBjIBw8PD8J/XCJ3GDrhTbWaOVYY6Nu0EQ1q5faAvHY06B+cZ4WBAQMGsGfPHkqVKsXly9knNkhKSqJfv35ERERgb2/Pli1bFCVjZs2axapVqzAyMmLRokW0bdsWgH379jF69GjS0tIYNGgQEydOVPtnAx259zghIYG7d++KCgOFgdyHxir2QUFBQezbty/X9atWraJEiRLcvn2bTz/9lM8/z7jD7OrVq/z8889cuXKFffv2MWzYMNL+LaQ2fPhw9u7dy9WrV9m8eTNXr15Vy5/0v3QiaUWFAUHbWrRogZ2dXa7rQ0NDCQwMBOCjjz7i0KFDSJJEaGgoPXv2xMzMjIoVK+Lm5kZYWBhhYWG4ubnh6uqKqakpPXv2JDQ0VCOx60TSigoDhYzMPW1MTIzi2rqXl1eB5kCOjo6mbNmMm1mMjY0pXrw4z58/V1oOGfcGREdH57pcEwrtAwNC0eXg4KB0rl3YiKQV1Ez696XfnJ2defDgAS4uLqSmphIbG4u9vb1ieaaHDx/i/O+18tyWq5tOHB4LhYikA68CWrJkCUuWZFwd8Pf3V9xWu337dlq1aoWBgQH+/v78/PPPJCUlERkZya1bt2jQoAH169fn1q1bREZGkpyczM8//4y/v3/Bg3kH0dMKaiZ/T5tZgKtTp0506tQpx2169erF0aNHiYmJwcXFhWnTpnH9+nWaNm0KZNwz0LdvX9zc3LCzs1PcoVejRg26d++Ou7s7xsbGLF26VHGn35IlS2jbti1paWkMGDCAGjVqaOTz6dR1Wl3n4eHB0aNH5Q5DZ+R4nbZRfRrU1Mw/VlU1CP6kQJXgO3bsyC+//ILpv/eA6yrR0+aT9dWLcoegE96415E7BLXL6Z54XSTOaQVBz4ikFdRM7lEoUVRaEPJH/nEoUVRaEPJHB7K2kBNJK6ifyFmNEkkraIDIWk0SA1GCesk/DiUGolQRFBRETEyMXlznevbsGR9//DF///03z58/p1SpUvj7+/Ptt99SvHhxucMjcNZcLt65q7jLpnQJW36f/Q0Ae07/xYLtv/Dq9Rsa13Dnm4FB2FpnTGZ359FjvtmwkSv3oihhY834Hh/R2rOeDJ9A/nPawj4QVWh72tTU1BynjzE0NKRz587s3r2bmzdvsnbtWg4dOkRwcLAMUeZsSt/eRCxbQsSyJYqEvRUdzdR1G/hu8ECOL/oeCzNTZqzPmKs5NS2NESFLaFmnNqeXLmRaUF8+X7aKezoy75WgXkWuLIi9vT1Dhw7F09OT8uXL4+vry7Bhwzhx4oTm/gBqsOf0X/jUrYNX1SpYmZszsksAByPOEf82kcjHT/jnVSyBbf0wMjSkkXt1PCq7sevkGbnDFjSgyJcFefToEb/88gstW7ZU74d+Dwu2/0KTEZ/S55vZhF27AZBRYaCsi2KbcqVKYWJszL1/v9T+S0LiloYewn4nuaeaKQK30mslaceMGUOrVq2oUKECLVu2ZM6cOWzdulVRXjI4OJgDBw4onvTPLAuSOStj1rIgtWvXplq1aixbtow3b94onUdnlgWpV68eNWvWxNg491P2Xr16YWlpibOzMzY2NqxZs0aDfwHVje3elQNzZnF0wRy6ebdgWMhiov75h4TEJKz/W2HAwoL4t4lUcCyNfTEbVu/dT0pqKicvX+Hs9ZskZpkMXtA9L1++5MqVK9y9ezdfpVaLbFmQBQsWcO7cOUJDQ7l7965SvaGstFkWBKBOJVesLMwxNTHhw2ZNqOfm9m+FAbNsJUDeJL7FysIcE2NjFo8azrGLf9Ni9Ges3XeAdg28KC1HUTExevxOsbGxfPvtt9SqVYtGjRoxZMgQunfvTvny5enWrRtHjhzJcx8av06bWRYkODiY6dOnY29vz7lz5+jVq1e2siAhISFMnjw517IgWasOZMo6OVd+yoI4Ojri6OhItWrVsLOzo3nz5kyZMkVpnh/IKAsyePBggGzzOmuFgQESGRUGrkf9/8wID/55RnJKKhX+/ZKqWtaF9ZPGK9b3/mY2AU0baztaxOjxu3300Uf069ePEydOZKuWERERwYYNG7h79262ub+zKpJlQf4r89AkKSlJ7fvOj7j4BC7dvUv9qlUxMjJk719nibhxk8l9epCSlkbvGbMJv3ET9wrlWfxrKH6e9bCyyDhvv/HgIRVKlyZdSmfz4aM8e/WKzs2aaP9DyJ+zOu3gwYO5rvP09MTT0zPPfagtaTPLgmRla2urVBakS5cunDlzpkBlQebNm0dAQADTp0+nXLlyPHjwgNDQUIYOHaq0bV727NnD8+fP8fT0xNramitXrjB+/HgaNWqEm5tbQT++WqSmpbHol53cffwEIwNDKpZxZPGo4VRwdATg68CPmbBsFbFv3tCoRnVmDuyvaLvr5Gl2HP+TlLQ0PKtUZuX4sZiamMjwKUTWqkKSJDZu3Mjdu3f56quviIqK4smTJzRo0CDPtkWuLIi5uTk//vgj165dIykpibJly9K5c2eNzQafH3bFbNj69ZRc13ds3JCOjRvmuG58z26M79lNU6EJajZs2DAMDQ05fPgwX331FTY2NnTt2pWzZ8/m2VanppvRi7IgP4TIHYZOeONeJ+fpZjzr0MC9uoyRQYMRnxZouhltqlevHufOncPDw4Pz588DUKdOHS5ezHtmFJ24I0qUBRHUSZdHjzOZmJiQlpaGwb91TZ89e4ahoWrpqBNJK8qCCOqUOXqc20yMumDUqFF07tyZf/75hy+++IJmzZoxefJkldrqxKN5U6dOZerUqXKHIahDEbkr6X2kp6dTsWJF5syZo6gRtHPnTqpXV+20QieSVhCKEkNDQ4YPH8758+epVq1a/ttrICahKNOBO6L0ga+vLzt27ChQIXORtIIgg2XLltGtWzdMTU2xsbHBxsZG5YqR4vA4nwrjJN1qJc5pVfL69esCtxVJK6iXHh2iym3Xrl0cP34cAG9vbzp27KhSO5G0+WR947LcIeiEN1Vryh2CXps4cSJnz56lT58+AISEhHDy5ElmzZqVZ1uRtEKho0rVPLn9/vvvXLhwQXFDRWBgIB4eHiJpBZnIfHisy4/mZfXq1SvFo6WxsbEqtxNJKwgymDRpEh4eHvj4+CBJEsePH2f27NkqtRVJK6iXGIhSSa9evfD29lY81fPdd9/h+O8jmHkR12kF9ZJ7Ujc9udz066+/Ymlpib+/P/7+/pibm7Nz506V2oqkFQQZTJs2TWlyfFtbW6ZNm6ZS2yJXYSCrmJgY6tSpw6NHj3j27BkODg5yhwTA76f/4oedu3gc8xwH2+LMHDwQEyMjFm3/lSv37mNkaED9atX4ol9vSv47/U5cfAKzftrEiYt/A9CztQ8junyo/eDF4bFKcpp9MTU1VaW2hfacNjU1FSMjI8Xzijnp378/devW5dGjR1qM7N1O/X2F77dsZ/6IodRyrcizVxmjijcePKCbT0tCatfEyNCQb9Zv5Ivlq1k+YSwAszdu5m1SMgcXzOFF3GsGzJ6Lk4M9XVo0l/PjCLnw8vJi7NixDB8+HIAlS5aoND8UFMEKA5lCQkJISEhg3Lhx6v/A72HJLzsZ9mEn6rhVwtDQkNJ2JShtV4IWdWrTrmF9rC0ssDAzo4+fL+eyfP6j5y8ysOMHWJiZ4VzSgS4tm/PLsT/l+RDigYE8LV68GFNTU3r06EGPHj0wNzdn6dKlKrUtkhUGzp8/z3fffcf69etVni1AG9LS07kceY8Xr1/TdtxEfEaN45t1P+U46Xj49Ru4OTsrLcv6xIgkSdx+KEOFAUElVlZWzJ49m/DwcP766y8mTZqk8hTARa7CQHx8PD179mTx4sU4/+cfvdyex8aSmpbGgbPh/PTlRH6ZOZVr96P4MVR52pQbUQ/4YeduPuvVXbGsWe2arNzzO/Fv33L/6VN+Pf4nb4tohQF9mG6md+/exMXFER8fT61atXB3d2fu3LkqtS1yFQZGjRpFs2bN6Nq1q0qxa7PCgJmpKQB9/FpT0taWEjY2BH7QhuP/Di4B3H/6lCHzFjD54154Va2iWD65b2/MTExpN34SIxYspn2jhjjayVVhQN5LPvow3czVq1cpVqwYO3fu5IMPPiAyMpINGzao1FbjSZtZYaB69eps27aNiIgIxaHwfysMrF27FiDXCgMXLlxQet28eZMhQ4Yo9qHK4cWhQ4dYu3YtxsbGGBsb4+vrC2RUHPjiiy+ybT948GDCw8MJDw/H3t6+wH8HVRS3ssLRrgRZx84M+P9fomNiGDh7HkMDOuH/n4nIba2tmTtsMCeWLGT37G9IlyRquVbUaLw5kfsSrZ5cpiUlJYWUlBR27tyJv78/JiYm7xw0zarIVRg4cOCA0pfF2bNnGTBgAEePHs3XpOea0rlFMzYeOESzWrUwNjZi3b4DeNetzdMXLxkway69W/vS09cnW7uop/9QzNISGytLTv59mW1HjrHui89l+ASCKoYMGUKFChWoU6cOLVq04P79+9p/CF5fKgxUqVJF6feYmBgAqlWrphPXaYcGdOLl6ze0nzAJUxMT2jWozxD/Tqzc8zsP/nnG0l9DWfprqGL7iJX/A+DKvXvM/mkzrxPeUt6xNHM+GUxlFxnO2fVoBFdOo0aNYtSoUYrfy5Urp1LxLSiCFQZ0nYmxMV8F9eWroL5Ky4d3CWB4l4Bc233QsAEfNMy7pISgmwwMDN5ZmlVpW1FhQHUeHh6EL1sidxg64U3VmjlWGPi4Wk0aFGCGQXVqOGm8zlcYeB86cUdUQkICT548ERUGCgU9Gg3SUzpxZ4GoMCAUNZ6enixdupSXL1/mu61OJO3UqVNJSUnhyJEjKo+gCYI+27JlC48ePaJ+/fr07NmT/fv3qzwHsk4krVCIyH3fsZ4cmbu5uTFz5kxu3rxJ7969GTBgAOXLl+frr7/mxYsX72wrklYQZHLp0iXGjRvH+PHj6dq1K9u2baNYsWK0atXqne10YiBKKET0qLeTk6enJ7a2tgwcOJDZs2djZmYGQMOGDTl58uQ724qkzScx328e9OleQhlt27YNV1fXHNf98ssv72wrDo8FQQbFixdn1KhR1KtXD09PT0aPHq3yAymip80nq5Nn5Q5BJ8Q3rZ/zCh04PNaHycp79uxJixYt2LFjBwAbN26kR48e/PHHH3m2FUkrFDr6MFn548ePle5JmDJlClu2bFGprTg8FtRPXPLJU5s2bfj5559JT08nPT2drVu30rZtW5Xaip5WELTIxsYGAwMDJEli4cKF9O2b8WBIWloa1tbWzJs3L899iKQV1EuPejs5vE9d2kzi8FhQM7mPjXX7G+PevXvvXC9JEg8fPnznNqKnFdRL9/NGVuPHjyc9PZ2AgAA8PT0pWbIkiYmJ3L59myNHjnDo0CGmTZuGi4tLrvsokhUGcpqL53//+x9Dhw6VIRqhKNm2bRtXr15l48aNrF69msePH2NpaUn16tVp3749X3zxRa5T/2YqtD1tXhUGVqxYQceOHRW/Z62rIpfk1BRmbN/C6ZvXiU1IoKy9A592DKB59RoAnLl5nW92bOXxyxfUKl+Bb3v1xckuY7I5/9kzePTyRZZ9pdKsmjs/BH+i3Q8heto8ubu7M3PmzAK3L7IVBmxtbXF0dFS8LCws1P/B8yk1LR1H2xKsG/Epf307j1HtOzF23SqiXzzn5Zs3jF6zgpEfdOTUzLnULFuOcev/f4L3XRO/JPy7BYR/t4Czs+fjaFuCtnXraf9DyH06WwS+MIpkhQGA0aNH4+DgQP369fnxxx9zLIikbZZmZgxv1wFnO3sMDQ3xrlELFzt7rjyI4uClC7g5lqFt3XqYmZgwrG0HbjyK5u7TJ9n2E37nNi/j3+BXu672P4SgcVo5PB4zZozi5woVKjBnzhwCAgJYt24dhoaGBAcH06hRI6Kjo3F2dlZUGNi2bRugXGEg83B32bJllCpVij179tC9e8ZM+5kVBvKasHz69On4+PhgbW3NoUOHGDduHDExMUyZMkUzf4ACinkdx71n/+DmWIYtJ09Q1en/Z1e0NDOjrL0Dt588xrW0cjHi0LNn8KtdF8t/nxzRLvHAgKZpJWkPHz7MrFmzuHbtGrGxsaSlpSkqDDg5OSlVGJg8efI7KwxklZCQkO8KA4DS7WN169YlLS2NmTNn5pi0y5cvV9wSp+kKA1mlpKXx+Ya1BNRviGtpRxKSkyhhZa20jbW5BfFJiUrL3iYnc+DiBZYMGoKg26Kjo7l//75SicsWLVrk2U7jSZtZYSA4OJjp06djb2/PuXPn6NWrV7YKAyEhIUyePDnXCgNZJzDPZGdnp/hZ1QJG/9WwYUPi4uJ4+vRptqQfPHgwgwcPBsg2RaympKenM2njOkyMjfiia8ZUs5amZrz5T4LGJyViZaZ8CvDHpQsUt7SkfiX5J14Xcvf555+zZcsW3N3dFf/ODQwMdCNpda3CQE4uXLiAubm5RvadX5Ik8eWWjcS8juPH4GGY/Ps3q+RYhtCzfym2S0hK4kHMM9wcyyi1Dz37F/71G6pcYkLtishg0PvauXMnN27cUDz8nh9qG4jKrDCQ9XXv3j2lCgORkZFs3ry5QBUGSpcuTUBAAMeOHSMyMpLjx48zbtw4pRFkVezevZsVK1Zw+fJl7ty5w8qVK/nqq68YPHhwgf6A6jZ928/cffqEpYOGYv5vQS6A1rXrcPvJIw5cPE9SSgr/O7CXKk7OSuezT169JOz2TQLqN5Qj9AxyjxzryReGq6srKSkpBWpb5CoMmJiY8MMPPzB27FjS09NxdXVl+vTpiorccnr04jlbT/+JqbExLb+apFg+tXsvOno2YGFQMDN/2crEjeuoXa4C8/oNUGq/OzyMOuUrUs6hpLZDF/LJ0tKSunXr4uvrq9RZLFq0KM+2osJAPnh4eHB2pmo1RAu7+Kb1c64wULYaDSrLXGFg1iSdrzCQWdb1vwIDA/NsqxN3RIkKA4WNzvQDOiswMJC3b98SFRVF1apV89VWJ57yERUGChlxTpun3bt3U7duXdq1awdkDIb6+/ur1FYnklZUGChE5E5YPUnaqVOnEhYWprhiUbduXe7evatSW51IWkEoakxMTLI9pGJoqFo6iqQViqQBAwZQqlQpatb8/3msX7x4gZ+fH5UrV8bPz09RHEuSJEaNGoWbmxu1a9fm3Llz7/3+NWrUYNOmTaSlpXHr1i1GjhxJkyZNVGorklZQr39vPZbzpYqgoCD27duntGz27Nn4+vpy69YtfH19mT17NgB79+7l1q1b3Lp1i+XLl/PJJ+//uOPixYu5cuUKZmZm9O7dm2LFihESEqJSW5G0QpHUokULpVtgAUJDQxWXXAIDA9m5c6dieb9+/TAwMKBRo0a8evWKx48fv9f7b968mZkzZ3L27FnOnj3LzJkz+frrr1VqqxOXfPRJrpN0Cxl0oCxITEwMXl5eit+z3j/+Lk+fPqVMmYzbQh0dHZWe5S5btqxiOxcXF6KjoxXbFsSOHTswNzenT58+AIwYMYK3b9+q1FYkbT5Z/nZa7hB0QkKHxnKHkCsHBwelmz4KwsDAQKP3b+/YsQN/f38MDQ3Zt28ftra2rFq1SqW24vBYUC+5L/e8RydfunRpxWHv48ePKVWqFADOzs48ePBAsd3Dhw9xdnbOcR95efHiBS9evODt27esXLmSOXPmYGNjo1Jd2kwiaQX109Ok9ff3V9xeuG7dOgICAhTL169fjyRJnDlzhuLFixf40NjT0xMvLy88PT3x8fHh1atX/Pbbb4rlqhCHx0Kho0oBrl69enH06FFiYmJwcXFh2rRpTJw4ke7du7Nq1SrKly/P1q1bAWjfvj2///47bm5uWFpasmbNmgLHFhkZWeC2mUTSCuqlAwNRqhTg2rx5c47LDx06lG2ZgYEBS5cuVUtsmVJSUvjf//7H8ePHAfD29mbIkCGYmJjk2VYkrSDI4JNPPiElJYVhw4YBsGHDBj755BNWrlyZZ1uRtIIgg7Nnz3Lx4kXF761ataJOnToqtVXLQFRQUJDSxN/64KeffqJu3bqYm5vj4OBAv3795A6pcJB7EEreI3OVGRkZKU1KePfuXcV0THkptD3tuyoMLFq0iFmzZjF37lwaNWrE27dvuXnzpgxRvtudZ0+Y+ft2rj5+QAlLa8b5+dO6eh1S0lKZsGM9Vx494FHsC1YHjqBBBR2ZyE0HEkcfKsHPnTsXHx8fXF1dkSSJ+/fvZ5v7OzdFrsLAq1evmDRpEuvXr+fjjz/Gzc2NWrVq0bVrV839AQogNT2NUT+vpGWVGpycMIupnXow6defuPf8HwDqlXNldpePcbAWjzL+V+ZAlK4mLECzZs24desWixYtYvHixdy4cYOmTZuq1LbIVRg4cOAAaWlpPH36FHd3d5ydnencubPKzzJqS2TMU/55HUu/Rt4YGRrSsGIV6patyO6LZzExMqZvI2/qlauEkVyzLuZG7kNjPTk8bty4MWZmZtSuXZvatWtjZmZG48aq3WVW5CoM3L17l/T0dL755hsWLlyInZ2douLAtWvXdHqqGwmJW8/e70Z1zdOjzJHBkydPiI6O5u3bt5w/f57MKdri4uKUOqB3KXIVBtLT00lJSWHRokW0adMGgI0bN+Lo6Mju3btznCVSDhXsS2NvZcOaU4fp28ibsHu3CL93hwYVdeTc9V3EOW2u9u/fz9q1a3n48CHjxo1TJG2xYsX49ttvVdpHkaswkHn7mbu7u2JZ8eLFcXJyIioqKtv2cpUFMTEyIqTnQGbt3cGqk39Qw6kcbWvUxdRIx8cOdaCjVeXmCrkEBgYSGBjIjh07CjyOUuQqDGSe7N+4cUNRbfvNmzc8fvyY8uXLZ9tejrIgmaqWdmZt0CjF731WLSCgTgOtxiBoxvsMfBa5CgNVqlQhICCA0aNHc/LkSa5evUr//v0pVaqUzl1rvvE0mqTUFN6mJLPm1GFi3sTxYd2M6gHJqakkpWbMUJ+SlkZSago6NIW1oEFqS9rMCgNZX5999hm1a9cmJCSE+fPn4+7uzsqVK5k3b16O+xg4cCDJycm5VhhwdXWlW7duVKtWjcDAQF6+fJnvCgOQcctY48aN6dSpE02bNiUxMZFDhw7p3CDU7ovh+Hz/JS3mfsFfkTdZ0XcYpsYZB0cdl8zEc+ZnPH0dy5Cf/ofnzM94FKvao10aJffIsY5/b2UOrr7PgwOiwkA+eHh4EDZ+ltxh6ISEDo1zrjBg50Z91/xNvq1ujZZ+pbMVBurVq8e5c+cU/y8InRjVEBUGChEd6O10efTY3t6eNm3aEBkZmePk5Lt27cpzHzqRtHPmzGHmzJk0a9ZMVBgoDMToca5+++03zp07R9++fRk3blyB9qETSTt16lSmTp0qdxiCOkjI/jytLjM1NaVRo0acOnWKkiVL8ubNGwCsra1V3oeYbkYQZPD06VM8PDyoUaMG7u7ueHp6cvnyZZXaiqQV1EvukWM96eQHDx7M/PnzuX//PlFRUXz//fcqTfMKOnJ4LBQiOjDdjD6Ij4/Hx8dH8bu3t3eOT6XlRCStIMjA1dWVGTNm0LdvXyBjUgZXV1eV2oqkzSddnqRbZ4hLPnlavXo1X3/9NV26dMHAwIDmzZur/BC8SNp8svz1pNwh6ISEzrk8sK0D55W6fMknU4kSJVi0aFGB2oqkFdRKkiRxD7SGiaQV1EsHetrCTiStoF4iaTVOJK2gZiJrVTFq1Khsy4oXL46Xl5eihlBuxM0VgvqJmyvylJiYyIULF6hcuTKVK1fm0qVLPHz4kFWrVinNqZYT0dMK6qVHiSOnS5cucfLkScVsLp988gnNmzfnzz//pFatWu9sK3paQb3k7mWl/79Ou3v3bs1/3gJ6+fKl4mEByLhD6sWLFxgZGWFmZvbOtmrpaYOCgoiJiclx7idds3btWvr375/jurCwMOrXr6/liN7t96vn+N+f+3kS9wp7KxtmduyFZ9lK7Lt2nqUn9vH0dSyONraM9u6Ab5V3f0MXFfpwnXbChAnUrVsXb29vJEni+PHjTJ48mfj4eFq3bv3OtoX28Di3siA9evSgXbt2SsvGjx/PqVOnVC7qqy2nIm+w4Mge5n3Yj1pO5Xj2Jg6Ap69fMXHXRhZ/NJBmrtU4fucq435dx/5hX2JvZZPHXjVM3HuskoEDB9K+fXvCwsIA+Pbbb3FycgIySoa8S5ErC2JhYYGjo6PiVaxYMXbv3s3AgQNzrPsjp6Un9jG0WRvqOFfA0MCQ0ja2lLax5WlcLMXMLWheqToGBga0dKuBhYkpD15qb4rXXOnA4bG+SE9Pp2TJkpQoUYLbt28ratXmRSs9bWZZEFdXV+7fv8/IkSMZOXIkGzZsUCoL8tFHHyna5FQWpEmTJhw7dgxTU1PmzZtH69atlaoCZC0LYmpqmmNZkP/aunUr8fHxDBgwQGOfvyDS0tO58vgBPpVr8MH/ZpKclkKryrUY16oTNcqUxdW+NEduXaZFJXeO3r6CqbExVUqVkTvsDHqUOHL5/PPP2bJlCzVq1MDQMKPvNDAwoEWLFnm2LXJlQf5r+fLldOzYEUdHR/V9YDV4Hv+a1PQ0Dly/yPq+IzA2NGLk9lUsO3WQ0S070KmWFxNCN5CcmoqJkRHfdw7E0vTdAxjaI7I2Lzt37uTGjRt5DjrlRCuHx4cPH8bPzw8XFxdsbGzo0qWLoiwIoFQWBHhnWRBra2usra0pXrw4L1++zHdZkKyuXLnC6dOnCQ4OznWb5cuX4+XlhZeXl1YrDJgZmwDQx7M5Ja2LU8LSmsAG3py4c43TkTeYf3g3a/oM5/znc1nTZwRf/76F60+jtRZfruQ+NNaT7wtXV1dSUlIK1LbIlQXJavny5ZQtWzbbwFRWclUYKG5hSWkbW8h6nv3vz9f/eYRnuUrULFMOgFpO5ajtVJ7T925SrbSz1mLMkR4ljpwsLS2pW7cuvr6+Sr2tKk/+FLmyIJkSExPZsGEDo0aNUpxT6JrOtRuwKfwEzVyrYWxoxIawo7Ss5E7NMmVZdfoQ159GU620M9eePCTiwV161FOtvqlGidFjlfj7++c4haoq1Ja0mWVBsrK1tVUqC9KlSxfOnDlToLIg8+bNIyAggOnTp1OuXDkePHhAaGgoQ4cOVdpWVdu3byc2NlbnBqCyGtK0DS8T4um47FtMjU1oW60ug5v6YWZswrBmbfn0l7U8T3iNnaUVwU1a09S1mtwhZxA5m6fAwMACt1Vb0maWBcmqa9eubN++nZCQEL777jumTJlCkyZNmDdvXo4lJQcOHMj69etzLQsyceJEunXrRmxsLE5OTvj4+BSoLAjAihUraNu2LeXKlStQe20wMTLiy3Yf8WW7j7Kt6+3VnN5ezWWISngf3bt3Z+vWrdSqVSvHS4yXLl3Kcx+iLEg+eHh4EDZ6ptxh6ISEzk1zLAvSx7A89cvKW0PXfcUEWrVqpZPTzTx+/JgyZcpw//79HNfnVLnxv3TijihRFqQQ0YFzWl2+jTGzPvLVq1cVV0cy/fjjjwwdOjTPfejECMycOXOoWrUqdnZ2oiyIUCTMmDGDw4cPK36fM2cOoaGhKrXViZ5WlAUpRMQlH5Xs2rWLjh07MnfuXPbt28f169f1K2mFQkQHDo/1gYODA7t27aJ169Z4enqyfft2le99F0krqJ/I2VzZ2NgoJWdycjJ3795VJG1cXFye+xBJK6iV6Gjf7fXr1++9D5G0+ZTrJN1CBnFOq7Lo6Gju379PamqqYpnOPOUjCIKyzEfz3N3dFbf36tSjeYWJxfJDcoegE94O9s15hYQ4PlaBzj+aJwiCMp1+NE8oYsQ5rUp0+tE8oagRw8eq0IlH8wQBED2tit7n0TxxTiuol9xTzUj6MVn5rVu3+Oijj3B3d8fV1VXxUoXoaQX10oG7K3T5KZ9M/fv3Z9q0aXz66accOXKENWvWkJ6erlJbtfS0QUFBdOzYUR270oqzZ8/SunVrbG1tsbW1xdfXVzFptPCedKCn1Qdv377F19cXSZIoX748U6dO5bffflOpbaHtaXOrMPDmzRvatWtHhw4dOHPmDJIkMXPmTNq2bUtUVBQ2NjLP0J/FxCNb+OvRHd6mJuNgYU3/2i3pWq0+F/+JYkn4Qa4+j8bIwBCvMhWZ1LgTJS2LyR2yoCIzMzPS09OpXLkyS5YswdnZWam2z7sUuQoD169f58WLF0ybNo1q1apRvXp1ZsyYwatXr7hx44bm/ggFMKiuN/t7TuBM4FQWtenH4ogDXImJJi7pLR9Va8D+HhPY33MCViZmfHl8e947FHRGSEgICQkJLFq0iIiICDZs2MD69etVaquVpM2sMHDlyhU2bdpEWFgYI0eOBFCqMJBVThUGzM3NOXbsGKdPn6ZMmTK0bt2ahIQERZusFQYuXryYY4WBqlWrUrJkSVatWkVSUhJJSUmsWLGCcuXKUaNGDc3+IfLJrURpTI0yDoYM/v3vQdxzmpetSlvXWlibmmNhbEov98acf5rz9CXaJ/3/ea1cLz1Qv359rK2tcXFxYc2aNWzbto3bt2+r1LbIVRiwsbHh6NGjfPjhh8yaNUsR08GDB7GwsNDQX6Dgvjm5k9Cb50hMS6G6vRMtylbNtk3Ek0jcbFWfpF2j9Oi8Ug5xcXEsXbqU6Oho/P398fPzY+nSpXz//ffUrl2bPn365LmPIldh4O3btwwYMIBGjRpx5swZTp48iYeHBwEBATkeTstVYSDTlKYfciZwKus6DsG3Qg1MjJS/Z288f8yP5w8ztuEHuexBy+QehNLxL4y+ffty48YNatWqxcqVK/Hx8WHbtm38+uuvujNzha5VGNi0aRN37txRqsK9adMmSpQowa+//srHH3+stL1cFQayMjI0pJ5jBfbcPs/Wq2foUzPj8cCo2BiG7V/LxEYd8XSsKEts2ejRIaoc7t69y99//w1k/JsvU6YMUVFRKhWLy1TkKgwkJCRgYGCgVFXA0NAQAwMDla+TySUtPZ0Hr18A8Oj1S4L3rmKIRys6Va4nc2T/IXI2VyYmJoqfjYyMcHFxyVfCghoPjzMrDGR93bt3T6nCQGRkJJs3by5QhYHSpUsTEBDAsWPHiIyM5Pjx44wbN05pBFkVfn5+xMXFMWzYMK5du8aVK1fo378/RkZGtGrV6n3/DGrz/O0b9t65SEJKEmnp6Zx8eJO9dy/S0KkST+NjGfj7Snq5N6Z79YZyh6pM7kNjHf/CuHjxIsWKFaNYsWLY2Nhw6dIlxc/Fiql2ya7IVRioVq0au3fvZtq0aTRu3BgDAwPq1q3L3r17cXFxyf8H1xADYMu1v5hxcifpkkQZa1smNOqIT3l3/nfuDx6+fsEP5w7xw7n/f743LGiafAFn0oPEkVNaWtp770NUGMgHDw8P/uot5mWGjIfgc6ow0DuuDPXLVJIxMmhy+HsiIiJkjUGTdOKOKFFhQBBUpxNP+YgKA4WI3OezOnPcqDk6kbRTp04lJSWFI0eOqHwyLugouRO2CCStThweC4WHhIQODZMUSiJpBfUqIr2dnETSCuolklbjRNLmU67z/QoZxG2MGqcTA1GCIKhO9LT5ZDxup9wh6ITU7z/MeYU4PNY40dMKgp4RSSuomYTWZqh4j5krBgwYQKlSpahZs6Zi2YsXL/Dz86Ny5cr4+fnx8uVLTf2R3otIWkG95L6xQsVD86CgIPbt26e0bPbs2fj6+nLr1i18fX2ZPXt2Af4AmieSVlAvuRNWxaRt0aKF0gQKAKGhoYqZ/wMDA9m5c2c+P7x2iIEoQb104JJPTEwMXl5eit+zzj7yLk+fPqVMmTIAODo6KmYC1TUiaQX10oHRYwcHB6VHBgvCwMAg25zZuqJIVhg4dOgQTZo0wcbGBkdHRz7//HNSU1PlDqtQkNCLcagclS5dmsePHwPw+PFjSpUqpZ4/ipoV2nPa1NTUHG9cv3jxIu3bt8fPz4/z58+zZcsWdu3axcSJE2WIMn8M7K0wG+2LZUhPLL7vjmnvhmCoY72B3Oez75G0/v7+ihlB161bR0BAQMF3pkFFrsLAli1bcHd3Z9q0abi5udGyZUvmzJnD0qVLef36teb+CGpg+nEjpLhEEsZtJXHaLoyqlMbYJ/s8yLLSga5Wlap5vXr1onHjxty4cQMXFxdWrVrFxIkTOXjwIJUrV+aPP/7Q2S9yrZzTZlYYcHV15f79+4wcOZKRI0eyYcMGpQoDH330kaJNThUGmjRpwrFjxzA1NWXevHm0bt2aa9euKWa6yFphwNTUNMdZ7pKSkrItt7CwIDExkYiICLy9vTX6t3gfhg7WJB++DqnpSHGJpF6OxtDJVu6wlMk/DqVS1bzNmzfnuPzQoUM5LtclWulpx4wZQ6tWrahQoYKiZ9u6datiytLg4GAOHDhAdHQ0gKLCQOYEb1krDNSuXZtq1aqxbNky3rx5ozQda2aFgXr16lGzZk2MjbN/J7Vt25a//vqLn376idTUVKKjo5k+fTqA4nxGV6UcvIZR/YpgaoSBrSXGtZxJu/xI7rAELStyFQbatGnDvHnzGD58OObm5lSpUoX27dsDKM2FnEnuCgNZpd16iqGzLZaLe2M5rxtp956Tdj5K1pgE7dN40mZWGKhevTrbtm0jIiJCUWzrvxUG1q5dC5BrhYH/zqt88+ZNhgwZotiHKhUGAMaOHcurV6+IiooiJiZGMeCQUyXuwYMHEx4eTnh4OPb29gX6G6iFAZiPaU3aufskDN9I/OifMbA0xeQjT/liyoEOnNLqRSX491HkKgxkMjAwwMnJCcg4vylbtiz16unYTP1ZWZlhaG9Nyr/ntKQmkXryNqadPUjZrkPThb7nCK466EMl+PehtqTNrDCQla2trVKFgS5dunDmzJkCVRiYN28eAQEBTJ8+nXLlyvHgwQNCQ0MZOnSo0raqmDt3Lu3atcPQ0JBffvmF2bNns3XrVsWXik56k0T6s9eYeFclZf8VMDPGuIkb6Q9fyh2ZMh1I2sJObYfHmRUGsr4+++wzateuTUhICPPnz8fd3Z2VK1cyb968HPcxcOBAkpOTc60w4OrqSrdu3ahWrRqBgYG8fPky3xUGAPbu3Uvz5s3x8vLit99+IzQ0lA8//LAgH1urkn44glFNZywX9MDy2y6Qlk7SlrNyh6VE7kNjuUeutUFUGMgHDw8P/vQcI3cYOiH1+w9zrDDQ/b4DXiWzjw1oU8u/FxfqCgM6cUdUQkICd+/eFRUGCgF139xUkFdhH4jSiaQVFQYKGZmzNnMgqlOnTlr4sNqnE0krKgwIgurEo3mCeonRY40TSSuoVxEZwZWTThweC4WI3KNQUuEfiBI9raBW0r//yUncESUoyXWSbiGDOKfVOJG0+WS55YTcIeiEhB7Nc1wuIc5pNU0kraBeoqfVOJG0gnqJpNU4MXosCHpGJK2gVnI/4SMegheEfJP/+Fhc8hGEfBCjx5onklZQL/k72kIvz6QNCgoiJiYmx3mddNHo0aM5efIkly9fxtHRkXv37mXb5u+//2bEiBGEhYVhZ2fHkCFD+PLLL3WudkvQlh+49Pg+Rv/OElnaujh7BkwkLOo2s478ypPXrzA0MMTTxZUvWnWhtE1xmSNGJK0W6GVPm5qaipGRUY5Jlp6eTmBgIH///TcHDhzItj4uLg4/Pz9atGjB2bNnuX79Ov3798fKyopx48ZpI/x8mdyqMx/VbqS0zNW+NMu6DqaUdXGSU1NZfHIvM/7YzpLOA3PZixaJBwY07r1Hj3Wp5AfA4sWLGTlyJFWqVMlx/caNG0lISGDdunXUrFmTjz76iM8//5z58+ejQzPvvJODlQ2lrP+/VzU0NCTqlbxzMgva895Jm1ny48qVK2zatImwsDBGjhwJoFTyI6ucSn6Ym5tz7NgxTp8+TZkyZWjdujUJCQmKNllLfly8eDHHkh+qOH36NM2bN8fCwkKxrG3btjx69CjHQ2m5hfz5O82WfsnHmxcT9uC2YvnjuJc0XvIFniETWRd+lAH1veULMgu5L/foyffue3nvw+MxY8Yofq5QoQJz5swhICCAdevWYWhoSHBwMI0aNSI6OhpnZ2dFyY9t27YByiU/Mg93ly1bRqlSpdizZw/du3cH/r/kR14VBPLy5MkTXFxclJZl7vPJkydUrFjxvfavTmNbdKCSfWlMDI3Ze+M8I35dzfZ+Yyln60CZYiU4PWImsW8T2P73GSravd/fRX3kz5zM67SdOnUqlFPOvHdPq0slPzRBzrIgtcuUx8rUHFNjYwJq1MfDuQIn7l5T2qa4hSUBNbwYGbqa1PQ0rcanq8QcUe+giyU/8uLo6Kg4l86U+bujo2O27XWmLAhggEGOA7Op6em8SHhDfFKS1mPKRgcegi/s3itps5b8aNy4MVWqVOHRo+xV3Pr06cPDhw8VJT/69++vWFevXj1u376Ng4MDbm5uSi87O7v3CS9HjRs35sSJEyQmJiqWHTx4ECcnJypUqKD29yuouMS3nLx3naTUFFLT09hzLYKIh3dpVqEaB29dIvLFP6RLGck69+guqpdypriF/FPPyn0+K85p/6VPJT9u377NmzdvePToEcnJyYq43d3dMTU1pXfv3kybNo2goCCmTJnCzZs3mT17Nl9//bVOXadNTU9j0Z/7iHzxD0aGBlS0K0VIQBAV7Epy8t515h3dzYuEN1iamlG/bCVCAoLkDlnQEpWSNrPkR1Zdu3Zl+/bthISE8N133zFlyhSaNGnCvHnz6NGjR7Z9DBw4kPXr1+da8mPixIl069aN2NhYnJyc8PHxKVDJj0GDBnHs2DHF75lxR0ZGUqFCBYoXL87BgwcZPnw4Xl5elChRgnHjxjF27Nh8v5cm2Vlas+XjMTmu61OvOX3q5fwQutyKSm8nJ62VBdH1kh+q8PDwIOyTGXKHoRMSejTPsSyI/6US1Cshb1mQDx79UKjLgmj8jqiEhASePHkiSn4UJaKr1SiNP08rSn4ULXIPQhWF7wuNJ60o+VHEyH25RyStIAi6RiStoHZyd7RiuhlByA8dOEQV080ISnKbpFv4lw4kbWEnklZQK4miMYIrJ5G0+bS1xFy5Q9AJ3V+Oz3lFUbnuIiORtIJaiaNjzRNJK6iXyFqNE0krqJU4OtY8kbSC+omk1Shxc4Ug6BnR0wrqJc5pNS7PnjYoKIiOHTtqIxa1GD16NF5eXpibm+c4fUxiYiJBQUHUrl0bExMTvL29tR5jQbkPb8iHYZ8w4O1UWq7uInc4OZJ04L/CTi8Pj1NTU3OdWDyzwkC/fv1yXJ+Wloa5uTkjRoygQ4cOmgxT7RIeveb8zKPcWKPDD3jLfeNx4c/ZoldhwMrKih9//JHBgwdnm/9Y19379Sr3Q6+R9Pyt3KHkLh/PvYrnaQumyFUYEDTH2NiYqLcxsvayTxNjSU1N1cbHlU2RqzAgaM6kSZNovrgB4def0sSgttbf/4H0lDXsYti3Ywp1hYH3TtrDhw8za9Ysrl27RmxsLGlpaYoKA05OTkoVBiZPnvzOCgNZJSQk6EyFgczHvLRdYUDflClThkvxN2hu5ck/0gv8aYmhlqalvSTdYhfHOXzzz3xPu6tvilyFgfzSpQoD+sDS0pKzaVdIJ52V/EqylKLx9zwkneUPwjj//EqhT1goghUG9JmBkSFGZsYYGBlk+Vn3LgAYGhqyVzqFO66EsJk46Y1G3idNSmeTtI8onnAx6UaR+fdS5CoMAFy9epXk5GRiYmJ48+aNYpu6devm6720zWOKN55ft1L8XrlvXSKmHebctMMyRpW7ddIexhl8TAg/M0Dyx9mglNr2/VZKYgW/4kZZfkk/rFPVITStyFUYAGjfvj3379/Pto2uF5U+p8MJmpvvpZ8Ivn6dNtW96SL5UMOg0nvv87kUyzJ20IZGLJN+UUOU+kVUGMgHDw8Phl3UrxsyNKX7y/HZKgy8S0xMDE1LeuFFdbwNPAv8vpFSNBv4nY9pzyxpTYH3o89EhQFBKxwcHLiUeIOW5l48lV7QDV8MDfJ3Ph4hXWM/pzkR+ZdOVTjUNlFhQNAaMzMzTqdfwgpz/sd2EqXkvBuRcdqyVzrFn1zgwqtrRTphQVQYELTMwMCAUOkYXrizkE28lOLeuX2qlMY69vCCWC6k3KB48eJailR36d71AqFIWCnt5CN8WcwW7kuPc9wmXnrLIn7GhdIcSQ/H2Fg8SQoiaQUZfSet4+jtU/zE71yQbiit+0d6wQI24Ut9tkgHitQlnbyIpBVkValSJc6/uMoRIjggnUGSJG5JUfzIDvrRgSXSVrlD1DnieCOfcp3vVyiwEiVKcDH5Bq1MG7CYLSSQyJmH53B2dpY7NJ2kteu0hYGDg4NOjFw+e/aMkiVLyh0G9+7dIyYmRm37kySJvXv30rJlS9nuNdcHImn1kJeXl8o3NQiFjzinFQQ9I5JWEPSMSFo9NHjwYLlDEGQkzmkFQc+InlYQ9IxIWkHQMyJpBUHPiDuiCpGsM3ZkatmypQyRCJokBqIKESMjIyRJUtxcL0kS6enpMkclqJvoaQuRI0eOyB2CoAWipxUEPSN6Wj2S0zlrXsQ5beEjelo9knnOmh/inLbwET2tnmnXrh2tW7eWOwxBRiJp9UzTpk0ZO3as3GEIMhI3VwiCnhFJKwh6RiStIOgZkbSCoGdE0uqRqlWr4uDgIHcYgszEdVo9lJ6ezrlz57h79y4AFStWxNPTE0ND8R1cFIhLPnpmz549jBgxgqioKKXlZcuWZcmSJXTq1EmmyARtET2tHjlx4gS+vr7Y2NjQv39/atSoAWRUtl+1ahWvX7/m0KFDtGjRQuZIBU0SSatH2rZty7Vr1wgPD6dUqVJK654+fUr9+vWpWrUqBw8elClCQRvESZAe+euvvxg8eHC2hAUoXbo0Q4YM4ezZszJEJmiTSFo9kpycjI2NTa7rra2tSU5WrVCzoL/E4bEe8fT0xNTUlBMnTmSr1Zqamkrz5s1JTk4mIiJCpggFbRA9rR4ZOXIkYWFh+Pj4sGvXLu7cucOdO3fYtWsX3t7e/PXXXwwfPlzuMAUNEz2tnvnqq6/49ttvsz0na2RkxMSJE5kxY4ZMkQnaIpJWD925c4fQ0FDFzRWurq4EBARQqVIlmSMTtEEkrSDoGXFOKwh6RtzGqEd8fHzy3EaSJI4ePar5YATZiKTVI1FRUdkmdktLS+PRo0ekp6fj4OCAlZWVTNEJ2iLOaQuB5ORkFi9ezI8//sjx48cpU6aM3CEJGiSSthDp168fKSkpbN68We5QBA0SA1GFSNOmTdm/f7/cYQgaJpK2EImMjCQpKUnuMAQNEwNReuT+/fs5Ln/+/DmHDx9m4cKFeHt7azcoQevEOa0eeVdZEAMDA6pVq8bOnTupXLmyliMTtEkkrR6ZMWNGtqQ1MDDA3t6eypUr07p1a0VtWqHwEkkrCHpGDEQJgp4RSatnXr16xeTJk6lRowYWFhZYWFhQo0YNJk+eTGxsrNzhCVogDo/1yMOHD2nWrBkPHjygdOnSigGnW7du8fTpU8qVK8eJEydwcXGROVJBk0RPq0fGjh3Lo0ePWL16NdHR0Rw7doxjx44RHR3NmjVriI6OFmUwiwDR0+oRW1tb+vbty+LFi3NcP3r0aNatW8erV6+0G5igVaKn1TOZE5TnpHr16rlexxUKD5G0esTb2/udz8oeO3ZM3BFVBIik1SMLFizgzJkzjBkzhidPniiWP3nyhE8//ZRTp06xYMECGSMUtEGc0+oRV1dX3rx5Q0xMDJBxjgsozmFzeghekiTu3bunxSgFTRMPDOiRihUrinNWQfS0gqBvxDmtIOgZcXish27evMmuXbuUKsEHBARQpUoVmSMTtEEcHuuZiRMnMnfu3BzXffbZZ3z33XdajkjQNnF4rEdWrVrF3LlzadasGaGhody+fZvbt2+za9cumjVrxty5c1m5cqXcYQoaJnpaPeLp6YmZmRnHjx8XpS6LMNHT6pFr167Ro0ePbAkLYGxsTM+ePbl27ZoMkQnaJJJWj5iamhIXF5fr+tevX2NqaqrFiAQ5iKTVI40aNWLFihU8fvw427rHjx+zfPlyGjZsKENkgjaJc1o9curUKXx8fLCysiIoKEjxxM+VK1dYs2YN8fHxHDlyhKZNm8ocqaBJImn1zP79+xk2bFi2+4nLly/PDz/8QLt27eQJTNAakbR6SJIkzp8/r1QJ3sPDQ0yfWkSIpBUEPSMGogRBz4h7j/XIu8qCZJWenq6FaAS5iKTVI0FBQTlWgr979y6nT5+mVq1aeHh4yBSdoC3inLaQCA8P54MPPmD37t00atRI7nAEDRJJW4hMmDCBsLCwd07+Jug/MRBViFSuXJnw8HC5wxA0TCRtIXL06FHMzc3lDkPQMDEQpUfWrVuX4/LMSvC///47AwcO1HJUgraJc1o9knnJJ6c7n4yNjenfvz/z58/H0tJShugEbRFJq0eOHz+ebVlmJfgKFSqIZC0iRNIKgp4RA1GCoGfEQFQhMm3aNKXfJUli6tSp8gQjaIw4PC5EjI2NlW5zlCRJ3IdcCImkFQQ9I85pBUHPiKQVBD0jklYQ9IxIWkHQMyJpBUHPiKQVBD0jklZQCAoKomPHjnKHIeShyCdtdHQ0gwcPxsXFBVNTU5ydnQkODubhw4f53pe3tzcjRozQQJSwdu1arK2t37lNUFAQBgYG73wJ+q9IJ21kZCReXl5cvnyZdevWcfv2bX766SeuXLlC/fr1s83ir+tCQkJ4/Pix4mVpacnChQuVlgmFgFSEffDBB5KTk5MUHx+vtDw+Pl5ycnKS2rdvr1jWsmVLafjw4UrbBQYGSh06dFD8DCi9IiMjpSNHjkiAtHv3bqlOnTqSmZmZVK9ePSk8PFyxnzVr1khWVlZK+85s9+zZM8XPWV9ff/11np/PyspKWrNmjeL3S5cuSb6+vpK5ublUokQJKTAwUHr16lWOn0eSJOnChQuSo6OjNHnyZEmSJOnVq1dScHCwVLJkScna2lpq0aKFdPbs2Wyf448//pBq1KghWVpaSt7e3tLdu3fzjFVQXZHtaV+8eMG+ffsYPnx4tudQLS0tGTZsGHv37uXly5cq7S8kJITGjRvTv39/Ra9WtmxZxfrPPvuM7777jvDwcFxdXenYsSMJCQkq7btJkyYsXLgQS0tLxb4/++wz1T8sEB8fT9u2bbG2tiYsLIxff/2VU6dOMWDAgBy3P3HiBN7e3kyYMIGZM2ciSRIdOnQgOjqaPXv2cP78eVq0aEGrVq2UevCkpCRmzZrF6tWrOX36NK9evWLo0KH5ilV4tyL7lM+tW7eQJInq1avnuN7d3R1Jkrh16xYNGjTIc3/FixfH1NQUS0tLHB0ds63/8ssvadu2LQBr1qzBxcWFTZs2MWjQoDz3bWpqSvHixTEwMMhx36rYtGkT8fHxbNiwARsbGwCWL1+Oj48Pt2/fxs3NTbHtnj176N27N0uWLKFfv34AHDlyhAsXLvDs2TMsLCwAmDFjBrt372bDhg1MmDAByKhIv3TpUqpWrQpkfFkNGDAg1xk3hPwrskmrbY0bN1b8bG1tTa1atbh69arW3v/atWvUrl1bkbCQ0YMbGhpy9epVRdJGRETQuXNnNm3aRLdu3RTbRkREkJCQQMmSJZX2m5iYyJ07dxS/m5mZKRIWwMnJieTkZF6+fImdnZ2mPl6RUmST1s3NDQMDA65evUrnzp2zrb969SoGBgaKf8yGhobZZvdPSUlRSyya3LcqsvaAFStWpFSpUqxZswZ/f3/MzMyAjFIjpUuX5sSJE9naFytWTPGzsbHyP6nMfYtHBNWnyJ7T2tvb07ZtW3744Yds55YJCQksXbqUDz74QNE7lCxZMtvo68WLF5V+NzU1JS0tLcf3O3PmjOLn+Ph4Ll++rDg0L1myJAkJCcTFxSm2uXDhgsr7VkX16tX5+++/ef36tWLZqVOnSE9PVzpFsLOz49ChQ0RHR9O5c2eSkpIAqFevHk+fPsXQ0BA3NzelV6lSpQocl5B/RTZpAZYsWUJqaiqtW7fm8OHDPHjwgKNHj+Ln54ckSSxZskSxbatWrdi7dy+7du3ixo0bjB07lgcPHijtr0KFCoSFhXHv3j1iYmKUepdvvvmGgwcPcuXKFQYMGICpqSm9e/cGoGHDhlhZWTFp0iRu377Njh07+OGHH7LtOzExkYMHDxITE6PyIFamPn36YGlpSb9+/fj77785fvw4Q4YMoUuXLkrnswAODg4cOnSIhw8f0qVLF5KSkmjdujVNmzYlICCAvXv3EhkZyenTp/n6669z7H0FDZJx5FonREVFSYMGDZKcnJwkY2NjqUyZMtLAgQOlBw8eKG2XnJwsDRs2TLK3t5fs7e2lr776Ktslkhs3bkiNGjWSLCwssl3yCQ0NlWrVqiWZmppKHh4eUlhYmNL+d+7cKVWuXFkyNzeX2rRpI23YsEFxySfT0KFDJXt7+/e65NOqVSvJ3NxcsrW1zfOSz7Nnz6RatWpJHTp0kBITE6W4uDhp1KhRkrOzs2RiYiK5uLhIPXr0kG7fvi1JUt6XrgT1EDNXaNjRo0fx8fHh2bNnODg4yB2OUAgU6cNjQdBHImkFQc+Iw2NB0DOipxUEPSOSVhD0jEhaQdAzImkFQc+IpBUEPfN/B+Rr5zDrWVkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 136.8x417.6 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "output.rankings()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7fc1af57",
   "metadata": {},
   "source": [
    "\n",
    "The model chose London by making the highest probability token (ranking it #1) after the last layer in the model. How much did each layer contribute to increasing the ranking of *London*? This is a [logit lens](https://www.lesswrong.com/posts/AcKRB8wDpdaN6v6ru/interpreting-gpt-the-logit-lens) visualizations that helps explore the activity of different model layers.\n",
    "\n",
    "### What are the patterns in BERT neuron activation when it processes a piece of text? "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1f5e47ef",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertModel: ['vocab_transform.weight', 'vocab_layer_norm.weight', 'vocab_transform.bias', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",
      "- This IS expected if you are initializing DistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<html lang=\"en\">\n",
       "<script src=\"https://requirejs.org/docs/release/2.3.6/minified/require.js\"></script>\n",
       "<script>\n",
       "    var ecco_url = 'https://storage.googleapis.com/ml-intro/ecco/'\n",
       "    //var ecco_url = 'http://localhost:8000/'\n",
       "\n",
       "    if (window.ecco === undefined) window.ecco = {}\n",
       "\n",
       "    // Setup the paths of the script we'll be using\n",
       "    requirejs.config({\n",
       "        urlArgs: \"bust=\" + (new Date()).getTime(),\n",
       "        nodeRequire: require,\n",
       "        paths: {\n",
       "            d3: \"https://d3js.org/d3.v6.min\", // This is only for use in setup.html and basic.html\n",
       "            \"d3-array\": \"https://d3js.org/d3-array.v2.min\",\n",
       "            jquery: \"https://code.jquery.com/jquery-3.5.1.min\",\n",
       "            ecco: ecco_url + 'js/0.0.6/ecco-bundle.min',\n",
       "            xregexp: 'https://cdnjs.cloudflare.com/ajax/libs/xregexp/3.2.0/xregexp-all.min'\n",
       "        }\n",
       "    });\n",
       "\n",
       "    // Add the css file\n",
       "    //requirejs(['d3'],\n",
       "    //    function (d3) {\n",
       "    //        d3.select('#css').attr('href', ecco_url + 'html/styles.css')\n",
       "    //    })\n",
       "\n",
       "    console.log('Ecco initialize!!')\n",
       "\n",
       "    // returns a 'basic' object. basic.init() selects the html div we'll be\n",
       "    // rendering the html into, adds styles.css to the document.\n",
       "    define('basic', ['d3'],\n",
       "        function (d3) {\n",
       "            return {\n",
       "                init: function (viz_id = null) {\n",
       "                    if (viz_id == null) {\n",
       "                        viz_id = \"viz_\" + Math.round(Math.random() * 10000000)\n",
       "                    }\n",
       "                    // Select the div rendered below, change its id\n",
       "                    const div = d3.select('#basic').attr('id', viz_id),\n",
       "                        div_parent = d3.select('#' + viz_id).node().parentNode\n",
       "\n",
       "                    // Link to CSS file\n",
       "                    d3.select(div_parent).insert('link')\n",
       "                        .attr('rel', 'stylesheet')\n",
       "                        .attr('type', 'text/css')\n",
       "                        .attr('href', ecco_url + 'html/0.0.2/styles.css')\n",
       "\n",
       "                    return viz_id\n",
       "                }\n",
       "            }\n",
       "        }, function (err) {\n",
       "            console.log(err);\n",
       "        }\n",
       "    )\n",
       "</script>\n",
       "\n",
       "<head>\n",
       "    <link id='css' rel=\"stylesheet\" type=\"text/css\">\n",
       "</head>\n",
       "<div id=\"basic\"></div>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "         requirejs(['basic', 'ecco'], function(basic, ecco){\n",
       "            const viz_id = basic.init()\n",
       "            \n",
       "            ecco.interactiveTokensAndFactorSparklines(viz_id, {'tokens': [{'token': '[CLS]', 'token_id': 101, 'type': 'input', 'position': 0}, {'token': 'now', 'token_id': 2085, 'type': 'input', 'position': 1}, {'token': 'i', 'token_id': 1045, 'type': 'input', 'position': 2}, {'token': 'ask', 'token_id': 3198, 'type': 'input', 'position': 3}, {'token': 'you', 'token_id': 2017, 'type': 'input', 'position': 4}, {'token': ':', 'token_id': 1024, 'type': 'input', 'position': 5}, {'token': 'what', 'token_id': 2054, 'type': 'input', 'position': 6}, {'token': 'can', 'token_id': 2064, 'type': 'input', 'position': 7}, {'token': 'be', 'token_id': 2022, 'type': 'input', 'position': 8}, {'token': 'expected', 'token_id': 3517, 'type': 'input', 'position': 9}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 10}, {'token': 'man', 'token_id': 2158, 'type': 'input', 'position': 11}, {'token': 'since', 'token_id': 2144, 'type': 'input', 'position': 12}, {'token': 'he', 'token_id': 2002, 'type': 'input', 'position': 13}, {'token': 'is', 'token_id': 2003, 'type': 'input', 'position': 14}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 15}, {'token': 'being', 'token_id': 2108, 'type': 'input', 'position': 16}, {'token': 'endowed', 'token_id': 19038, 'type': 'input', 'position': 17}, {'token': 'with', 'token_id': 2007, 'type': 'input', 'position': 18}, {'token': 'strange', 'token_id': 4326, 'type': 'input', 'position': 19}, {'token': 'qualities', 'token_id': 11647, 'type': 'input', 'position': 20}, {'token': '?', 'token_id': 1029, 'type': 'input', 'position': 21}, {'token': 'shower', 'token_id': 6457, 'type': 'input', 'position': 22}, {'token': 'upon', 'token_id': 2588, 'type': 'input', 'position': 23}, {'token': 'him', 'token_id': 2032, 'type': 'input', 'position': 24}, {'token': 'every', 'token_id': 2296, 'type': 'input', 'position': 25}, {'token': 'earthly', 'token_id': 29520, 'type': 'input', 'position': 26}, {'token': 'blessing', 'token_id': 13301, 'type': 'input', 'position': 27}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 28}, {'token': 'drown', 'token_id': 19549, 'type': 'input', 'position': 29}, {'token': 'him', 'token_id': 2032, 'type': 'input', 'position': 30}, {'token': 'in', 'token_id': 1999, 'type': 'input', 'position': 31}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 32}, {'token': 'sea', 'token_id': 2712, 'type': 'input', 'position': 33}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 34}, {'token': 'happiness', 'token_id': 8404, 'type': 'input', 'position': 35}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 36}, {'token': 'so', 'token_id': 2061, 'type': 'input', 'position': 37}, {'token': 'that', 'token_id': 2008, 'type': 'input', 'position': 38}, {'token': 'nothing', 'token_id': 2498, 'type': 'input', 'position': 39}, {'token': 'but', 'token_id': 2021, 'type': 'input', 'position': 40}, {'token': 'bubbles', 'token_id': 17255, 'type': 'input', 'position': 41}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 42}, {'token': 'bliss', 'token_id': 13670, 'type': 'input', 'position': 43}, {'token': 'can', 'token_id': 2064, 'type': 'input', 'position': 44}, {'token': 'be', 'token_id': 2022, 'type': 'input', 'position': 45}, {'token': 'seen', 'token_id': 2464, 'type': 'input', 'position': 46}, {'token': 'on', 'token_id': 2006, 'type': 'input', 'position': 47}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 48}, {'token': 'surface', 'token_id': 3302, 'type': 'input', 'position': 49}, {'token': ';', 'token_id': 1025, 'type': 'input', 'position': 50}, {'token': 'give', 'token_id': 2507, 'type': 'input', 'position': 51}, {'token': 'him', 'token_id': 2032, 'type': 'input', 'position': 52}, {'token': 'economic', 'token_id': 3171, 'type': 'input', 'position': 53}, {'token': 'prosperity', 'token_id': 14165, 'type': 'input', 'position': 54}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 55}, {'token': 'such', 'token_id': 2107, 'type': 'input', 'position': 56}, {'token': 'that', 'token_id': 2008, 'type': 'input', 'position': 57}, {'token': 'he', 'token_id': 2002, 'type': 'input', 'position': 58}, {'token': 'should', 'token_id': 2323, 'type': 'input', 'position': 59}, {'token': 'have', 'token_id': 2031, 'type': 'input', 'position': 60}, {'token': 'nothing', 'token_id': 2498, 'type': 'input', 'position': 61}, {'token': 'else', 'token_id': 2842, 'type': 'input', 'position': 62}, {'token': 'to', 'token_id': 2000, 'type': 'input', 'position': 63}, {'token': 'do', 'token_id': 2079, 'type': 'input', 'position': 64}, {'token': 'but', 'token_id': 2021, 'type': 'input', 'position': 65}, {'token': 'sleep', 'token_id': 3637, 'type': 'input', 'position': 66}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 67}, {'token': 'eat', 'token_id': 4521, 'type': 'input', 'position': 68}, {'token': 'cakes', 'token_id': 22619, 'type': 'input', 'position': 69}, {'token': 'and', 'token_id': 1998, 'type': 'input', 'position': 70}, {'token': 'busy', 'token_id': 5697, 'type': 'input', 'position': 71}, {'token': 'himself', 'token_id': 2370, 'type': 'input', 'position': 72}, {'token': 'with', 'token_id': 2007, 'type': 'input', 'position': 73}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 74}, {'token': 'continuation', 'token_id': 13633, 'type': 'input', 'position': 75}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 76}, {'token': 'his', 'token_id': 2010, 'type': 'input', 'position': 77}, {'token': 'species', 'token_id': 2427, 'type': 'input', 'position': 78}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 79}, {'token': 'and', 'token_id': 1998, 'type': 'input', 'position': 80}, {'token': 'even', 'token_id': 2130, 'type': 'input', 'position': 81}, {'token': 'then', 'token_id': 2059, 'type': 'input', 'position': 82}, {'token': 'out', 'token_id': 2041, 'type': 'input', 'position': 83}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 84}, {'token': 'sheer', 'token_id': 11591, 'type': 'input', 'position': 85}, {'token': 'ing', 'token_id': 13749, 'type': 'input', 'position': 86}, {'token': '##rat', 'token_id': 8609, 'type': 'input', 'position': 87}, {'token': '##itude', 'token_id': 18679, 'type': 'input', 'position': 88}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 89}, {'token': 'sheer', 'token_id': 11591, 'type': 'input', 'position': 90}, {'token': 'spite', 'token_id': 8741, 'type': 'input', 'position': 91}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 92}, {'token': 'man', 'token_id': 2158, 'type': 'input', 'position': 93}, {'token': 'would', 'token_id': 2052, 'type': 'input', 'position': 94}, {'token': 'play', 'token_id': 2377, 'type': 'input', 'position': 95}, {'token': 'you', 'token_id': 2017, 'type': 'input', 'position': 96}, {'token': 'some', 'token_id': 2070, 'type': 'input', 'position': 97}, {'token': 'nasty', 'token_id': 11808, 'type': 'input', 'position': 98}, {'token': 'trick', 'token_id': 7577, 'type': 'input', 'position': 99}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 100}, {'token': 'he', 'token_id': 2002, 'type': 'input', 'position': 101}, {'token': 'would', 'token_id': 2052, 'type': 'input', 'position': 102}, {'token': 'even', 'token_id': 2130, 'type': 'input', 'position': 103}, {'token': 'risk', 'token_id': 3891, 'type': 'input', 'position': 104}, {'token': 'his', 'token_id': 2010, 'type': 'input', 'position': 105}, {'token': 'cakes', 'token_id': 22619, 'type': 'input', 'position': 106}, {'token': 'and', 'token_id': 1998, 'type': 'input', 'position': 107}, {'token': 'would', 'token_id': 2052, 'type': 'input', 'position': 108}, {'token': 'deliberately', 'token_id': 9969, 'type': 'input', 'position': 109}, {'token': 'desire', 'token_id': 4792, 'type': 'input', 'position': 110}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 111}, {'token': 'most', 'token_id': 2087, 'type': 'input', 'position': 112}, {'token': 'fatal', 'token_id': 10611, 'type': 'input', 'position': 113}, {'token': 'rubbish', 'token_id': 29132, 'type': 'input', 'position': 114}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 115}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 116}, {'token': 'most', 'token_id': 2087, 'type': 'input', 'position': 117}, {'token': 'une', 'token_id': 16655, 'type': 'input', 'position': 118}, {'token': '##con', 'token_id': 8663, 'type': 'input', 'position': 119}, {'token': '##omic', 'token_id': 22026, 'type': 'input', 'position': 120}, {'token': '##al', 'token_id': 2389, 'type': 'input', 'position': 121}, {'token': 'absurd', 'token_id': 18691, 'type': 'input', 'position': 122}, {'token': '##ity', 'token_id': 3012, 'type': 'input', 'position': 123}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 124}, {'token': 'simply', 'token_id': 3432, 'type': 'input', 'position': 125}, {'token': 'to', 'token_id': 2000, 'type': 'input', 'position': 126}, {'token': 'introduce', 'token_id': 8970, 'type': 'input', 'position': 127}, {'token': 'into', 'token_id': 2046, 'type': 'input', 'position': 128}, {'token': 'all', 'token_id': 2035, 'type': 'input', 'position': 129}, {'token': 'this', 'token_id': 2023, 'type': 'input', 'position': 130}, {'token': 'positive', 'token_id': 3893, 'type': 'input', 'position': 131}, {'token': 'good', 'token_id': 2204, 'type': 'input', 'position': 132}, {'token': 'sense', 'token_id': 3168, 'type': 'input', 'position': 133}, {'token': 'his', 'token_id': 2010, 'type': 'input', 'position': 134}, {'token': 'fatal', 'token_id': 10611, 'type': 'input', 'position': 135}, {'token': 'fantastic', 'token_id': 10392, 'type': 'input', 'position': 136}, {'token': 'element', 'token_id': 5783, 'type': 'input', 'position': 137}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 138}, {'token': 'it', 'token_id': 2009, 'type': 'input', 'position': 139}, {'token': 'is', 'token_id': 2003, 'type': 'input', 'position': 140}, {'token': 'just', 'token_id': 2074, 'type': 'input', 'position': 141}, {'token': 'his', 'token_id': 2010, 'type': 'input', 'position': 142}, {'token': 'fantastic', 'token_id': 10392, 'type': 'input', 'position': 143}, {'token': 'dreams', 'token_id': 5544, 'type': 'input', 'position': 144}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 145}, {'token': 'his', 'token_id': 2010, 'type': 'input', 'position': 146}, {'token': 'vulgar', 'token_id': 29364, 'type': 'input', 'position': 147}, {'token': 'folly', 'token_id': 26272, 'type': 'input', 'position': 148}, {'token': 'that', 'token_id': 2008, 'type': 'input', 'position': 149}, {'token': 'he', 'token_id': 2002, 'type': 'input', 'position': 150}, {'token': 'will', 'token_id': 2097, 'type': 'input', 'position': 151}, {'token': 'desire', 'token_id': 4792, 'type': 'input', 'position': 152}, {'token': 'to', 'token_id': 2000, 'type': 'input', 'position': 153}, {'token': 'retain', 'token_id': 9279, 'type': 'input', 'position': 154}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 155}, {'token': 'simply', 'token_id': 3432, 'type': 'input', 'position': 156}, {'token': 'in', 'token_id': 1999, 'type': 'input', 'position': 157}, {'token': 'order', 'token_id': 2344, 'type': 'input', 'position': 158}, {'token': 'to', 'token_id': 2000, 'type': 'input', 'position': 159}, {'token': 'prove', 'token_id': 6011, 'type': 'input', 'position': 160}, {'token': 'to', 'token_id': 2000, 'type': 'input', 'position': 161}, {'token': 'himself', 'token_id': 2370, 'type': 'input', 'position': 162}, {'token': '-', 'token_id': 1011, 'type': 'input', 'position': 163}, {'token': '-', 'token_id': 1011, 'type': 'input', 'position': 164}, {'token': 'as', 'token_id': 2004, 'type': 'input', 'position': 165}, {'token': 'though', 'token_id': 2295, 'type': 'input', 'position': 166}, {'token': 'that', 'token_id': 2008, 'type': 'input', 'position': 167}, {'token': 'were', 'token_id': 2020, 'type': 'input', 'position': 168}, {'token': 'so', 'token_id': 2061, 'type': 'input', 'position': 169}, {'token': 'necessary', 'token_id': 4072, 'type': 'input', 'position': 170}, {'token': '-', 'token_id': 1011, 'type': 'input', 'position': 171}, {'token': '-', 'token_id': 1011, 'type': 'input', 'position': 172}, {'token': 'that', 'token_id': 2008, 'type': 'input', 'position': 173}, {'token': 'men', 'token_id': 2273, 'type': 'input', 'position': 174}, {'token': 'still', 'token_id': 2145, 'type': 'input', 'position': 175}, {'token': 'are', 'token_id': 2024, 'type': 'input', 'position': 176}, {'token': 'men', 'token_id': 2273, 'type': 'input', 'position': 177}, {'token': 'and', 'token_id': 1998, 'type': 'input', 'position': 178}, {'token': 'not', 'token_id': 2025, 'type': 'input', 'position': 179}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 180}, {'token': 'keys', 'token_id': 6309, 'type': 'input', 'position': 181}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 182}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 183}, {'token': 'piano', 'token_id': 3682, 'type': 'input', 'position': 184}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 185}, {'token': 'which', 'token_id': 2029, 'type': 'input', 'position': 186}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 187}, {'token': 'laws', 'token_id': 4277, 'type': 'input', 'position': 188}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 189}, {'token': 'nature', 'token_id': 3267, 'type': 'input', 'position': 190}, {'token': 'threaten', 'token_id': 15686, 'type': 'input', 'position': 191}, {'token': 'to', 'token_id': 2000, 'type': 'input', 'position': 192}, {'token': 'control', 'token_id': 2491, 'type': 'input', 'position': 193}, {'token': 'so', 'token_id': 2061, 'type': 'input', 'position': 194}, {'token': 'completely', 'token_id': 3294, 'type': 'input', 'position': 195}, {'token': 'that', 'token_id': 2008, 'type': 'input', 'position': 196}, {'token': 'soon', 'token_id': 2574, 'type': 'input', 'position': 197}, {'token': 'one', 'token_id': 2028, 'type': 'input', 'position': 198}, {'token': 'will', 'token_id': 2097, 'type': 'input', 'position': 199}, {'token': 'be', 'token_id': 2022, 'type': 'input', 'position': 200}, {'token': 'able', 'token_id': 2583, 'type': 'input', 'position': 201}, {'token': 'to', 'token_id': 2000, 'type': 'input', 'position': 202}, {'token': 'desire', 'token_id': 4792, 'type': 'input', 'position': 203}, {'token': 'nothing', 'token_id': 2498, 'type': 'input', 'position': 204}, {'token': 'but', 'token_id': 2021, 'type': 'input', 'position': 205}, {'token': 'by', 'token_id': 2011, 'type': 'input', 'position': 206}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 207}, {'token': 'calendar', 'token_id': 8094, 'type': 'input', 'position': 208}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 209}, {'token': 'and', 'token_id': 1998, 'type': 'input', 'position': 210}, {'token': 'that', 'token_id': 2008, 'type': 'input', 'position': 211}, {'token': 'is', 'token_id': 2003, 'type': 'input', 'position': 212}, {'token': 'not', 'token_id': 2025, 'type': 'input', 'position': 213}, {'token': 'all', 'token_id': 2035, 'type': 'input', 'position': 214}, {'token': ':', 'token_id': 1024, 'type': 'input', 'position': 215}, {'token': 'even', 'token_id': 2130, 'type': 'input', 'position': 216}, {'token': 'if', 'token_id': 2065, 'type': 'input', 'position': 217}, {'token': 'man', 'token_id': 2158, 'type': 'input', 'position': 218}, {'token': 'really', 'token_id': 2428, 'type': 'input', 'position': 219}, {'token': 'were', 'token_id': 2020, 'type': 'input', 'position': 220}, {'token': 'nothing', 'token_id': 2498, 'type': 'input', 'position': 221}, {'token': 'but', 'token_id': 2021, 'type': 'input', 'position': 222}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 223}, {'token': 'piano', 'token_id': 3682, 'type': 'input', 'position': 224}, {'token': '-', 'token_id': 1011, 'type': 'input', 'position': 225}, {'token': 'key', 'token_id': 3145, 'type': 'input', 'position': 226}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 227}, {'token': 'even', 'token_id': 2130, 'type': 'input', 'position': 228}, {'token': 'if', 'token_id': 2065, 'type': 'input', 'position': 229}, {'token': 'this', 'token_id': 2023, 'type': 'input', 'position': 230}, {'token': 'were', 'token_id': 2020, 'type': 'input', 'position': 231}, {'token': 'proved', 'token_id': 4928, 'type': 'input', 'position': 232}, {'token': 'to', 'token_id': 2000, 'type': 'input', 'position': 233}, {'token': 'him', 'token_id': 2032, 'type': 'input', 'position': 234}, {'token': 'by', 'token_id': 2011, 'type': 'input', 'position': 235}, {'token': 'natural', 'token_id': 3019, 'type': 'input', 'position': 236}, {'token': 'science', 'token_id': 2671, 'type': 'input', 'position': 237}, {'token': 'and', 'token_id': 1998, 'type': 'input', 'position': 238}, {'token': 'mathematics', 'token_id': 5597, 'type': 'input', 'position': 239}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 240}, {'token': 'even', 'token_id': 2130, 'type': 'input', 'position': 241}, {'token': 'then', 'token_id': 2059, 'type': 'input', 'position': 242}, {'token': 'he', 'token_id': 2002, 'type': 'input', 'position': 243}, {'token': 'would', 'token_id': 2052, 'type': 'input', 'position': 244}, {'token': 'not', 'token_id': 2025, 'type': 'input', 'position': 245}, {'token': 'become', 'token_id': 2468, 'type': 'input', 'position': 246}, {'token': 'reasonable', 'token_id': 9608, 'type': 'input', 'position': 247}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 248}, {'token': 'but', 'token_id': 2021, 'type': 'input', 'position': 249}, {'token': 'would', 'token_id': 2052, 'type': 'input', 'position': 250}, {'token': 'purposely', 'token_id': 24680, 'type': 'input', 'position': 251}, {'token': 'do', 'token_id': 2079, 'type': 'input', 'position': 252}, {'token': 'something', 'token_id': 2242, 'type': 'input', 'position': 253}, {'token': 'per', 'token_id': 2566, 'type': 'input', 'position': 254}, {'token': '##verse', 'token_id': 16070, 'type': 'input', 'position': 255}, {'token': 'out', 'token_id': 2041, 'type': 'input', 'position': 256}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 257}, {'token': 'simple', 'token_id': 3722, 'type': 'input', 'position': 258}, {'token': 'ing', 'token_id': 13749, 'type': 'input', 'position': 259}, {'token': '##rat', 'token_id': 8609, 'type': 'input', 'position': 260}, {'token': '##itude', 'token_id': 18679, 'type': 'input', 'position': 261}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 262}, {'token': 'simply', 'token_id': 3432, 'type': 'input', 'position': 263}, {'token': 'to', 'token_id': 2000, 'type': 'input', 'position': 264}, {'token': 'gain', 'token_id': 5114, 'type': 'input', 'position': 265}, {'token': 'his', 'token_id': 2010, 'type': 'input', 'position': 266}, {'token': 'point', 'token_id': 2391, 'type': 'input', 'position': 267}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 268}, {'token': 'and', 'token_id': 1998, 'type': 'input', 'position': 269}, {'token': 'if', 'token_id': 2065, 'type': 'input', 'position': 270}, {'token': 'he', 'token_id': 2002, 'type': 'input', 'position': 271}, {'token': 'does', 'token_id': 2515, 'type': 'input', 'position': 272}, {'token': 'not', 'token_id': 2025, 'type': 'input', 'position': 273}, {'token': 'find', 'token_id': 2424, 'type': 'input', 'position': 274}, {'token': 'means', 'token_id': 2965, 'type': 'input', 'position': 275}, {'token': 'he', 'token_id': 2002, 'type': 'input', 'position': 276}, {'token': 'will', 'token_id': 2097, 'type': 'input', 'position': 277}, {'token': 'con', 'token_id': 9530, 'type': 'input', 'position': 278}, {'token': '##tri', 'token_id': 18886, 'type': 'input', 'position': 279}, {'token': '##ve', 'token_id': 3726, 'type': 'input', 'position': 280}, {'token': 'destruction', 'token_id': 6215, 'type': 'input', 'position': 281}, {'token': 'and', 'token_id': 1998, 'type': 'input', 'position': 282}, {'token': 'chaos', 'token_id': 8488, 'type': 'input', 'position': 283}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 284}, {'token': 'will', 'token_id': 2097, 'type': 'input', 'position': 285}, {'token': 'con', 'token_id': 9530, 'type': 'input', 'position': 286}, {'token': '##tri', 'token_id': 18886, 'type': 'input', 'position': 287}, {'token': '##ve', 'token_id': 3726, 'type': 'input', 'position': 288}, {'token': 'suffering', 'token_id': 6114, 'type': 'input', 'position': 289}, {'token': '##s', 'token_id': 2015, 'type': 'input', 'position': 290}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 291}, {'token': 'all', 'token_id': 2035, 'type': 'input', 'position': 292}, {'token': 'sorts', 'token_id': 11901, 'type': 'input', 'position': 293}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 294}, {'token': 'only', 'token_id': 2069, 'type': 'input', 'position': 295}, {'token': 'to', 'token_id': 2000, 'type': 'input', 'position': 296}, {'token': 'gain', 'token_id': 5114, 'type': 'input', 'position': 297}, {'token': 'his', 'token_id': 2010, 'type': 'input', 'position': 298}, {'token': 'point', 'token_id': 2391, 'type': 'input', 'position': 299}, {'token': '!', 'token_id': 999, 'type': 'input', 'position': 300}, {'token': 'he', 'token_id': 2002, 'type': 'input', 'position': 301}, {'token': 'will', 'token_id': 2097, 'type': 'input', 'position': 302}, {'token': 'launch', 'token_id': 4888, 'type': 'input', 'position': 303}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 304}, {'token': 'curse', 'token_id': 8364, 'type': 'input', 'position': 305}, {'token': 'upon', 'token_id': 2588, 'type': 'input', 'position': 306}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 307}, {'token': 'world', 'token_id': 2088, 'type': 'input', 'position': 308}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 309}, {'token': 'and', 'token_id': 1998, 'type': 'input', 'position': 310}, {'token': 'as', 'token_id': 2004, 'type': 'input', 'position': 311}, {'token': 'only', 'token_id': 2069, 'type': 'input', 'position': 312}, {'token': 'man', 'token_id': 2158, 'type': 'input', 'position': 313}, {'token': 'can', 'token_id': 2064, 'type': 'input', 'position': 314}, {'token': 'curse', 'token_id': 8364, 'type': 'input', 'position': 315}, {'token': '(', 'token_id': 1006, 'type': 'input', 'position': 316}, {'token': 'it', 'token_id': 2009, 'type': 'input', 'position': 317}, {'token': 'is', 'token_id': 2003, 'type': 'input', 'position': 318}, {'token': 'his', 'token_id': 2010, 'type': 'input', 'position': 319}, {'token': 'privilege', 'token_id': 14293, 'type': 'input', 'position': 320}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 321}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 322}, {'token': 'primary', 'token_id': 3078, 'type': 'input', 'position': 323}, {'token': 'distinction', 'token_id': 7835, 'type': 'input', 'position': 324}, {'token': 'between', 'token_id': 2090, 'type': 'input', 'position': 325}, {'token': 'him', 'token_id': 2032, 'type': 'input', 'position': 326}, {'token': 'and', 'token_id': 1998, 'type': 'input', 'position': 327}, {'token': 'other', 'token_id': 2060, 'type': 'input', 'position': 328}, {'token': 'animals', 'token_id': 4176, 'type': 'input', 'position': 329}, {'token': ')', 'token_id': 1007, 'type': 'input', 'position': 330}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 331}, {'token': 'may', 'token_id': 2089, 'type': 'input', 'position': 332}, {'token': 'be', 'token_id': 2022, 'type': 'input', 'position': 333}, {'token': 'by', 'token_id': 2011, 'type': 'input', 'position': 334}, {'token': 'his', 'token_id': 2010, 'type': 'input', 'position': 335}, {'token': 'curse', 'token_id': 8364, 'type': 'input', 'position': 336}, {'token': 'alone', 'token_id': 2894, 'type': 'input', 'position': 337}, {'token': 'he', 'token_id': 2002, 'type': 'input', 'position': 338}, {'token': 'will', 'token_id': 2097, 'type': 'input', 'position': 339}, {'token': 'attain', 'token_id': 18759, 'type': 'input', 'position': 340}, {'token': 'his', 'token_id': 2010, 'type': 'input', 'position': 341}, {'token': 'object', 'token_id': 4874, 'type': 'input', 'position': 342}, {'token': '-', 'token_id': 1011, 'type': 'input', 'position': 343}, {'token': '-', 'token_id': 1011, 'type': 'input', 'position': 344}, {'token': 'that', 'token_id': 2008, 'type': 'input', 'position': 345}, {'token': 'is', 'token_id': 2003, 'type': 'input', 'position': 346}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 347}, {'token': 'convince', 'token_id': 8054, 'type': 'input', 'position': 348}, {'token': 'himself', 'token_id': 2370, 'type': 'input', 'position': 349}, {'token': 'that', 'token_id': 2008, 'type': 'input', 'position': 350}, {'token': 'he', 'token_id': 2002, 'type': 'input', 'position': 351}, {'token': 'is', 'token_id': 2003, 'type': 'input', 'position': 352}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 353}, {'token': 'man', 'token_id': 2158, 'type': 'input', 'position': 354}, {'token': 'and', 'token_id': 1998, 'type': 'input', 'position': 355}, {'token': 'not', 'token_id': 2025, 'type': 'input', 'position': 356}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 357}, {'token': 'piano', 'token_id': 3682, 'type': 'input', 'position': 358}, {'token': '-', 'token_id': 1011, 'type': 'input', 'position': 359}, {'token': 'key', 'token_id': 3145, 'type': 'input', 'position': 360}, {'token': '!', 'token_id': 999, 'type': 'input', 'position': 361}, {'token': '[SEP]', 'token_id': 102, 'type': 'input', 'position': 362}], 'factors': [[[0.0006851888610981405, 0.0, 0.07580550760030746, 0.03562500327825546, 0.0033170832321047783, 0.0, 0.007548785302788019, 0.20885130763053894, 0.08036421984434128, 0.006420585792511702, 0.0, 0.0, 0.0, 0.0, 0.17210018634796143, 0.0, 0.014140808023512363, 0.0, 0.0, 0.0, 0.0, 0.048101797699928284, 0.0, 0.0006684949621558189, 0.0, 0.0, 0.0, 0.0, 0.043279971927404404, 0.0, 0.0, 0.0015145264333114028, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.287017822265625, 0.1237422451376915, 0.013725866563618183, 0.0, 0.0, 0.0, 0.0548601858317852, 0.05403843894600868, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0026552784256637096, 0.2459367960691452, 0.11796343326568604, 0.023050814867019653, 0.027025287970900536, 0.16413846611976624, 0.06331468373537064, 0.031140664592385292, 0.007080692797899246, 0.08568352460861206, 0.02344210259616375, 0.0, 0.02635231427848339, 0.008411149494349957, 0.0, 0.016717659309506416, 0.0, 0.0, 0.0, 0.0, 0.004009981639683247, 0.0, 0.0, 0.008656181395053864, 0.012266045436263084, 0.009533488191664219, 0.005685694515705109, 0.0, 0.004374144598841667, 0.004203957039862871, 0.00036147970240563154, 0.013247532770037651, 0.016458377242088318, 0.002209628699347377, 0.005305680446326733, 0.01982431299984455, 0.4244120121002197, 0.04968175292015076, 0.03824247047305107, 0.015209135599434376, 0.001440779073163867, 0.0, 0.021311085671186447, 0.03105463832616806, 0.4350323975086212, 0.10126891732215881, 0.041897788643836975, 0.00030985366902314126, 0.0, 0.0162531565874815, 0.40277567505836487, 0.046167951077222824, 0.04286368936300278, 0.008789265528321266, 0.01746201701462269, 0.0, 0.0, 0.0, 0.0013057072646915913, 0.02591923624277115, 0.0, 0.0007107364945113659, 0.0, 0.0, 0.0, 0.0026751093100756407, 0.0, 0.026250284165143967, 0.19733913242816925, 0.018603043630719185, 0.004667143803089857, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.013862485997378826, 0.20941290259361267, 0.020801670849323273, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0013116010231897235, 0.3730127513408661, 0.06199832260608673, 0.1602707952260971, 0.0, 0.0, 0.002355906879529357, 0.013393157161772251, 0.021706506609916687, 0.17224928736686707, 0.01746727153658867, 0.04057570919394493, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.1493723839521408, 0.0038136164657771587, 0.0, 0.0, 0.0, 0.0, 0.0, 0.05171244964003563, 0.18482095003128052, 0.0, 0.0, 0.02222966030240059, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0696704313158989, 0.14124254882335663, 0.004890574607998133, 0.0, 0.0, 0.0, 0.02484104409813881, 0.03405580297112465, 0.43178480863571167, 0.16869033873081207, 0.09141828864812851, 0.18042948842048645, 0.03311874344944954, 0.008234674111008644, 0.0, 0.013901982456445694, 0.0, 0.0, 0.0, 0.0, 0.002011215081438422, 0.21323318779468536, 0.051284126937389374, 0.0068155936896800995, 0.0, 0.0, 0.030330942943692207, 0.01217279490083456, 0.057644281536340714, 0.18501225113868713, 0.021999163553118706, 0.007126821205019951, 0.002859109081327915, 0.0, 0.0, 0.0, 3.8295172998914495e-05, 0.004774034023284912, 0.03503701463341713, 0.023921113461256027, 0.17750415205955505, 0.048059284687042236, 0.06638696789741516, 0.0029143409337848425, 0.01824241690337658, 0.000574432429857552, 0.0, 0.0, 0.0, 0.0, 0.0004898508195765316, 0.0212844330817461, 0.022699078544974327, 0.3938615322113037, 0.13460735976696014, 0.0883304551243782, 0.0, 0.0, 0.007964649237692356, 0.36014601588249207, 0.03888489678502083, 0.1036025881767273, 0.02240484207868576, 0.010997685603797436, 0.0, 0.011992834508419037, 0.011450008489191532, 0.0036907829344272614, 0.015495922416448593, 0.018645258620381355, 0.00879492238163948, 0.013400922529399395, 0.03550451621413231, 0.188999205827713, 0.044267769902944565, 0.0, 0.009460755623877048, 0.008965098299086094, 0.0, 0.03979659453034401, 0.022924166172742844, 0.22757534682750702, 0.1330370306968689, 0.06295503675937653, 0.042339976876974106, 0.043889567255973816, 0.4072994887828827, 0.05899534001946449, 0.03211979568004608, 0.061372309923172, 0.0, 0.012087673880159855, 0.0, 0.02591109462082386, 0.3284977376461029, 0.04636691138148308, 0.02169407531619072, 0.047434430569410324, 0.000695587310474366, 0.02756579779088497, 0.01303330808877945, 0.009764691814780235, 0.006554093211889267, 0.01087834034115076, 0.03579410910606384, 0.18936483561992645, 0.047208141535520554, 0.008559837006032467, 0.00281095365062356, 0.019291933625936508, 0.023034118115901947, 0.41957083344459534, 0.008033929392695427, 0.01928471215069294, 0.0, 0.002044059569016099, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0038731934037059546, 0.001933436025865376, 0.2653370499610901, 0.0, 0.0, 0.021148910745978355, 0.16416263580322266, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.20841750502586365, 0.08385711908340454, 0.005367591977119446, 0.0, 0.0, 0.0, 0.0, 0.35393771529197693, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.12540125846862793, 0.038400668650865555, 0.0, 0.0, 0.0055435942485928535, 0.0, 0.20584464073181152, 0.007388195022940636, 0.0, 0.0, 0.03883415088057518, 0.0, 0.0, 0.0, 0.0, 0.004770917817950249, 0.010399573482573032], [0.03385530784726143, 0.0, 0.007837343961000443, 0.0, 0.01383893471211195, 0.11993034929037094, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.008349654264748096, 0.0, 0.0, 0.12683936953544617, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.8046727776527405, 0.0, 0.0, 0.05939718335866928, 0.0, 0.0, 0.0, 0.0, 1.0256191492080688, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0005836720229126513, 0.0, 0.0, 0.0, 0.012843175791203976, 0.028463125228881836, 0.004429989494383335, 0.26796141266822815, 0.0, 0.0, 0.0, 0.0, 1.0053573846817017, 0.0, 0.0, 0.0, 0.0, 0.004631604067981243, 0.0, 0.0, 0.040063511580228806, 0.03215588629245758, 0.04681650921702385, 0.014031755737960339, 0.7064011096954346, 0.01225184090435505, 0.0, 0.16952581703662872, 0.024917617440223694, 0.02809605747461319, 0.07193707674741745, 0.10412139445543289, 0.025501582771539688, 0.02735370397567749, 0.0, 0.049136605113744736, 0.8862361907958984, 0.09442351013422012, 0.0, 0.03584849461913109, 0.013760385103523731, 0.02912406623363495, 0.0, 0.0, 0.0, 0.004839551169425249, 0.7648063898086548, 0.0, 0.03428787365555763, 0.9630887508392334, 0.029268575832247734, 0.02298750914633274, 0.026030467823147774, 0.045237477868795395, 0.022276459261775017, 0.014362012036144733, 0.03465091809630394, 0.06576009839773178, 0.006060272920876741, 0.006440557073801756, 0.012048741802573204, 0.016298949718475342, 0.03359268233180046, 0.0, 0.19941116869449615, 0.015472996048629284, 0.00042376696364954114, 0.0152174923568964, 0.05213102325797081, 0.017169401049613953, 0.0026029986329376698, 0.011187589727342129, 0.9296272993087769, 0.08137550950050354, 0.03133348748087883, 0.011891383677721024, 0.010164530016481876, 0.0035885917022824287, 0.036918170750141144, 0.0, 0.046601567417383194, 1.1260279417037964, 0.0014074624050408602, 0.046561967581510544, 0.003016567789018154, 0.020739344879984856, 0.0, 0.003489383263513446, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.7909152507781982, 0.0, 0.0, 0.0, 0.01967315375804901, 0.0, 0.0, 0.0, 0.0, 0.000647725653834641, 0.9713156223297119, 0.0, 0.024736318737268448, 0.0, 0.0005581448785960674, 0.0, 0.0, 0.0, 0.13611675798892975, 0.08322060853242874, 0.0, 0.0, 0.0, 0.004500408191233873, 0.0, 0.0, 0.1245846226811409, 0.08603982627391815, 0.0, 0.0, 0.0, 0.0, 0.0, 0.06550464034080505, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.974482536315918, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.006520227063447237, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0020846559200435877, 0.0, 0.0, 0.001036084839142859, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.10755541175603867, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.05827243998646736, 0.0, 0.9652978181838989, 0.0, 0.0, 0.0, 0.011966705322265625, 0.0, 0.0, 0.0, 0.019021181389689445, 0.0, 0.004025587346404791, 0.04535618796944618, 0.0, 1.0203243494033813, 0.0, 0.02280297875404358, 0.0, 0.0, 0.0, 0.0, 0.0, 0.9761419892311096, 0.0244744885712862, 0.0, 0.0, 0.0064225345849990845, 0.0, 0.0, 0.0, 0.01844724640250206, 0.0066315410658717155, 0.0, 0.0, 0.0, 0.0, 1.1036715507507324, 0.0, 0.0556207112967968, 0.0, 0.011194917373359203, 0.011409986764192581, 0.016362933441996574, 0.0, 0.0, 0.0, 0.0016681752167642117, 0.0, 0.012367984279990196, 0.03983496129512787, 0.01794230006635189, 0.0, 0.013064602389931679, 0.0, 0.007959835231304169, 0.01566561870276928, 0.11870968341827393, 0.025384049862623215, 0.9135971665382385, 0.014387045055627823, 0.00626341113820672, 0.0, 0.005676307715475559, 0.01180355530232191, 0.049956973642110825, 0.06528934836387634, 0.0, 0.02927147038280964, 1.1907196044921875, 0.02102924883365631, 0.07248926162719727, 0.002897653030231595, 0.02144731767475605, 0.021334176883101463, 0.094139963388443, 0.004981006029993296, 0.0, 0.0, 0.017619814723730087, 0.0, 0.009614523500204086, 0.0, 0.003188745817169547, 0.9458194375038147, 0.03479710966348648, 0.0, 0.0, 0.0, 0.0, 0.0, 0.15919840335845947, 0.0, 0.0, 0.0, 0.0, 0.7279790043830872, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.24542869627475739, 0.74189293384552, 0.0, 0.0, 0.0, 0.0, 0.0, 0.02744763344526291, 0.0, 0.0, 0.0, 0.0, 0.0, 0.10688786208629608, 0.020191878080368042, 0.0, 0.0, 0.7416996359825134, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.05374709516763687, 0.0, 0.0, 0.0, 0.06405626982450485, 0.0, 0.07405789941549301, 0.09764160960912704], [0.008299949578940868, 0.00010250315972371027, 0.0050315652042627335, 0.0009074204717762768, 0.00497415428981185, 0.005671922117471695, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.004321474116295576, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.026896623894572258, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0001215284937643446, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.3462361991405487, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.23004336655139923, 0.002679952420294285, 0.018867645412683487, 0.0, 0.0, 0.0, 0.0002574078389443457, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.000432170694693923, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.03668350353837013, 0.0, 0.0, 0.0, 0.0, 0.6279054880142212, 0.0024362923577427864, 0.0, 0.0, 0.0011467422591522336, 0.0, 0.0, 0.0, 0.0015467596240341663, 0.0023128341417759657, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0006639230414293706, 0.0018938810098916292, 0.00045372339081950486, 0.00035506277345120907, 0.001846248866058886, 1.2433271408081055, 0.0008962902938947082, 0.00038774023414589465, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0011282882187515497, 0.0, 0.0, 0.0017903333064168692, 0.0, 9.689034050097689e-05, 0.00021942464809399098, 0.0, 0.0, 0.0, 0.000418810173869133, 0.002928344998508692, 0.0, 0.0004371345567051321, 0.0, 0.0031597765628248453, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0002039393875747919, 0.0, 0.0, 0.0, 0.0013612605398520827, 1.4269825220108032, 0.006333955097943544, 0.00577945401892066, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0005830397712998092, 0.0, 0.0, 0.0, 0.0, 0.00025262843701057136, 0.0, 0.0, 0.005339573137462139, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0004171892942395061, 0.0, 0.0, 0.0, 0.0003381574060767889, 0.0, 0.0, 9.324716666014865e-05, 0.20706239342689514, 0.0, 0.0010923407971858978, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 7.65807053539902e-05, 0.0, 0.0009984108619391918, 0.0021897894330322742, 0.001981192035600543, 1.6809271574020386, 0.008651640266180038, 0.0005121549475006759, 0.002695014700293541, 0.0, 0.0, 0.008428909815847874, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0011154088424518704, 0.0, 6.466308695962653e-05, 0.08580517768859863, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.35010120272636414, 0.0, 0.0006896060658618808, 0.0, 0.0016803496982902288, 0.0, 0.0, 0.0005356424953788519, 0.000964330043643713, 0.0, 0.0, 0.0, 0.0010337859857827425, 0.0020237008575350046, 0.0, 0.0, 0.0, 0.0, 0.0022542241495102644, 0.001772907329723239, 1.480930209159851, 0.007603669539093971, 0.0015009200433269143, 0.0, 0.0, 0.0, 0.0, 0.002798778936266899, 0.0, 0.0, 0.0010518874041736126, 0.0023625295143574476, 0.0014283716445788741, 0.0, 0.0, 0.0001830731489462778, 0.0, 0.0006047951173968613, 0.0031878661829978228, 0.0033900542184710503, 0.0032347773667424917, 0.0, 0.0019357744604349136, 0.0, 0.0, 0.0011026026913896203, 0.11794690042734146, 0.0, 0.0, 0.0, 8.741537021705881e-05, 0.0, 0.025650370866060257, 0.0, 0.0, 0.0, 0.00022517149045597762, 0.0, 0.0, 0.0015584917273372412, 0.0011389543069526553, 0.2451850026845932, 6.025327093084343e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.004758964758366346, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0140494704246521, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.003573987865820527, 0.0, 0.0009520754683762789, 0.0052766334265470505, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.025372741743922234, 0.031204737722873688], [0.034592874348163605, 0.0077288043685257435, 0.1366715431213379, 0.0, 0.164694145321846, 0.019403131678700447, 0.03429598733782768, 0.0, 0.02443491481244564, 0.0, 0.027772624045610428, 0.25102728605270386, 0.0, 0.5662901401519775, 0.022501183673739433, 0.03893138840794563, 0.0618353970348835, 0.0, 0.0026145242154598236, 0.0, 0.0, 0.01621648296713829, 0.0, 0.0, 0.5419651865959167, 0.006799379829317331, 0.0, 0.0, 0.0, 0.0, 0.5581638813018799, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.006894335150718689, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.008062380366027355, 0.0, 0.5509172081947327, 0.0, 0.0, 0.0, 0.0022982563823461533, 0.015898501500487328, 0.6561359167098999, 0.0, 0.001243111095391214, 0.021957261487841606, 0.0, 0.0, 0.013771641068160534, 0.0, 0.007316731382161379, 0.005475130397826433, 0.0, 0.0, 0.009513230994343758, 0.0, 0.3958666920661926, 0.030056316405534744, 0.037142444401979446, 0.0, 0.011769304983317852, 0.5344699621200562, 0.0518997497856617, 0.0, 0.0, 0.0017668262589722872, 0.01168682612478733, 0.007941063493490219, 0.0, 0.0, 0.003644118783995509, 0.0, 0.005924387834966183, 0.02288452349603176, 0.0013045907253399491, 0.017308801412582397, 0.04480309784412384, 0.2815060317516327, 0.0, 0.0, 0.20397457480430603, 0.03911551460623741, 0.002990494016557932, 0.0034538127947598696, 0.037063587456941605, 0.6418619155883789, 0.010498277842998505, 0.0025970900896936655, 0.0, 0.456813782453537, 0.0011226270580664277, 0.06859227269887924, 0.02966625615954399, 0.0, 0.0, 0.030626779422163963, 0.00883773248642683, 0.0, 0.0, 0.0017771036364138126, 0.021646574139595032, 0.012903640046715736, 0.0, 0.0, 0.0, 0.0, 0.0, 0.01883438043296337, 0.0, 0.0, 0.0, 0.0, 0.0, 0.030687104910612106, 0.026408404111862183, 0.0, 0.013205623254179955, 0.0063377260230481625, 0.4080579876899719, 0.0, 0.0, 0.0, 0.0, 0.10881926864385605, 0.0, 0.0, 0.4598771333694458, 0.0033537852577865124, 0.0, 0.0, 0.430105596780777, 0.0, 0.0, 0.03092234395444393, 0.6846803426742554, 0.011734404601156712, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.005257159471511841, 0.3800491392612457, 0.0, 0.0, 0.0, 0.0, 0.056549884378910065, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.16345703601837158, 0.0, 0.0, 0.09617310762405396, 0.0, 0.0, 0.0, 0.0, 0.0, 0.010899116285145283, 0.0, 0.0, 0.010285094380378723, 0.014619878493249416, 0.0, 0.0, 0.03065100871026516, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.17001765966415405, 0.0, 0.026725050061941147, 0.0, 0.0038983013946563005, 0.0, 0.021801544353365898, 0.0, 0.0, 0.016666505485773087, 0.0, 0.0, 0.0, 0.05873516947031021, 0.0, 0.0, 0.002924208762124181, 0.01205015555024147, 0.0, 0.011825096793472767, 0.280292272567749, 0.025731762871146202, 0.0031952180434018373, 0.029689252376556396, 0.0, 0.04846223443746567, 0.018676917999982834, 0.03139374405145645, 0.034827928990125656, 0.004849020857363939, 0.0, 0.004170945379883051, 0.10854904353618622, 0.028177928179502487, 0.0, 0.010179245844483376, 0.5597630143165588, 0.008167464286088943, 0.01058824174106121, 0.016612522304058075, 0.0, 0.0, 0.0, 0.0, 0.017122888937592506, 0.6362881660461426, 0.0, 0.0016606524586677551, 0.0, 0.0, 0.0, 0.025197379291057587, 0.0013390918029472232, 0.0, 0.011742129921913147, 0.04430432990193367, 0.0, 0.0, 0.006574550643563271, 9.951891843229532e-05, 0.04118269681930542, 0.023196430876851082, 0.012889261357486248, 0.023683801293373108, 0.016974233090877533, 0.0, 0.0023707242216914892, 0.027002044022083282, 0.5186914801597595, 0.04548490047454834, 0.024971511214971542, 0.0015082907630130649, 0.06494346261024475, 0.6675393581390381, 0.037057191133499146, 0.031306248158216476, 0.045631494373083115, 0.04539718106389046, 0.6535138487815857, 0.031103795394301414, 0.02976739965379238, 0.0053362310864031315, 0.023641224950551987, 0.014498939737677574, 0.02380863018333912, 0.01804807037115097, 0.05946293845772743, 0.05488278344273567, 0.03309551253914833, 0.007904435507953167, 0.032097768038511276, 0.022099491208791733, 0.05397890508174896, 0.005794496275484562, 0.04741577431559563, 0.023329617455601692, 0.007587555795907974, 0.0, 0.021827712655067444, 0.03733045607805252, 0.5309420824050903, 0.05753076449036598, 0.028260577470064163, 0.6105363965034485, 0.0, 0.0, 0.05977598577737808, 0.0, 0.0, 0.03255479410290718, 0.04256419464945793, 0.0, 0.0, 0.0, 0.005727770738303661, 0.2374357432126999, 0.0, 0.0, 0.0, 0.09326718747615814, 0.0, 0.39674660563468933, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.3950403332710266, 0.0, 0.008809039369225502, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4512823224067688, 0.0, 0.0, 0.5240780115127563, 0.0, 0.0, 0.39730188250541687, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.3647157549858093, 0.025447508320212364, 0.5850504636764526, 0.01787405088543892, 0.04708464816212654, 0.17139039933681488, 0.0, 0.0, 0.02847827598452568, 0.0026527009904384613, 0.030353575944900513, 0.038961123675107956, 0.0092031704261899, 0.0], [0.005780109204351902, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.06811158359050751, 0.02071089670062065, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0034206355921924114, 0.004140390083193779, 0.0, 0.0, 0.0, 0.018540367484092712, 0.0, 0.0, 0.014524849131703377, 0.0, 0.0, 0.0, 0.0, 0.0032950183376669884, 0.0, 0.0, 0.0, 0.0, 0.0, 0.01274131890386343, 0.0, 0.0, 0.0, 0.012950996868312359, 0.04356997832655907, 0.008629558607935905, 0.0, 0.0, 0.0, 0.01721622794866562, 0.0, 0.026164276525378227, 0.01147777121514082, 0.010902014560997486, 0.0, 0.0, 0.057268206030130386, 0.03572569042444229, 0.05397937074303627, 0.0, 0.02077537029981613, 0.009941368363797665, 0.021471155807375908, 0.030938219279050827, 0.01704770140349865, 0.048668500036001205, 0.025732403621077538, 0.1008404791355133, 0.16380810737609863, 0.07718691974878311, 0.015501496382057667, 0.03006543405354023, 0.014462445862591267, 0.04004453495144844, 0.03972555696964264, 0.040453433990478516, 0.04805166274309158, 0.034261129796504974, 0.0, 0.0, 0.0, 0.0, 0.02605503238737583, 0.004888548515737057, 0.02507251873612404, 0.03739097714424133, 0.012521629221737385, 0.0, 0.0009271110175177455, 0.034364886581897736, 0.039844196289777756, 0.026433037593960762, 0.01614924520254135, 0.053929831832647324, 0.026257984340190887, 0.0, 0.02164694480597973, 0.0, 0.013950887136161327, 0.023562535643577576, 0.029413750395178795, 0.04429841786623001, 0.050455834716558456, 0.06233750283718109, 0.11227678507566452, 0.043170150369405746, 0.03300512209534645, 0.05565069615840912, 0.0984124168753624, 0.03431911766529083, 0.08339366316795349, 0.16206800937652588, 0.09173981845378876, 0.033373039215803146, 0.07828812301158905, 0.08696995675563812, 0.06663364917039871, 0.04597223177552223, 0.04308312386274338, 0.0102726174518466, 0.0, 0.031643401831388474, 0.02570083923637867, 0.031265050172805786, 0.013543874956667423, 0.04678744077682495, 0.0321553498506546, 0.0, 0.017778586596250534, 0.0, 0.0, 0.0008865204290486872, 0.00832868181169033, 0.0, 0.027475785464048386, 0.042448971420526505, 0.0, 0.0, 0.0, 0.028993850573897362, 0.10184502601623535, 0.002104830229654908, 0.00848104152828455, 0.005855083465576172, 0.0, 0.0, 0.025282468646764755, 0.01955690234899521, 0.044074613600969315, 0.016803620383143425, 0.01710672490298748, 0.06880933791399002, 0.013178042136132717, 0.03590880706906319, 0.0, 0.0, 0.0, 0.0, 0.012627853080630302, 0.02464943565428257, 0.0, 0.0042887842282652855, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0015311305178329349, 0.0, 0.022414935752749443, 0.0, 0.0, 0.0, 0.0, 0.0, 0.04355274885892868, 0.02277718111872673, 0.015258383005857468, 0.0, 0.02411598712205887, 0.0, 0.05708981305360794, 0.02678200602531433, 0.03643462434411049, 0.032050903886556625, 0.0001704920141492039, 0.0, 0.008890612982213497, 0.03271683305501938, 0.014307783916592598, 0.0025959054473787546, 0.04658769816160202, 0.022170735523104668, 0.06168872117996216, 0.03799235075712204, 0.06362709403038025, 0.09962723404169083, 0.08184085041284561, 0.11010672897100449, 0.11160732805728912, 0.12161970138549805, 0.1436251997947693, 0.11398489028215408, 0.18335004150867462, 0.17647475004196167, 0.20691224932670593, 0.22138047218322754, 0.2652553915977478, 0.2733689546585083, 0.31713205575942993, 0.3060809373855591, 0.27678412199020386, 0.2785351574420929, 0.43354958295822144, 0.31053876876831055, 0.37636157870292664, 0.5018923878669739, 0.4488031268119812, 0.49669402837753296, 0.5752740502357483, 0.6637929081916809, 0.626274824142456, 0.5411802530288696, 0.5977604985237122, 0.5073147416114807, 0.7525209188461304, 0.813549280166626, 0.8511605262756348, 0.6705968976020813, 0.4973396956920624, 0.7216330766677856, 0.8250121474266052, 0.69613116979599, 0.6373845338821411, 0.7761062979698181, 0.7124261856079102, 0.7771342992782593, 0.7991702556610107, 0.6168471574783325, 0.8360040783882141, 0.7476860284805298, 0.7399196624755859, 0.8193160891532898, 0.6960939168930054, 0.661250650882721, 0.5162659287452698, 0.6592879295349121, 0.6058831214904785, 0.5041603446006775, 0.4972987771034241, 0.6316506862640381, 0.4596908986568451, 0.4843297004699707, 0.478231281042099, 0.5990077257156372, 0.7403578758239746, 0.5963929295539856, 0.4618670344352722, 0.5037340521812439, 0.5607441067695618, 0.44780224561691284, 0.554725170135498, 0.5364442467689514, 0.39542728662490845, 0.4221869111061096, 0.4375688433647156, 0.46874865889549255, 0.3128781318664551, 0.3331538736820221, 0.2730380892753601, 0.3291301727294922, 0.007682332769036293], [0.033730391412973404, 0.1973084956407547, 0.23596131801605225, 0.18636348843574524, 0.25985631346702576, 0.24573324620723724, 0.28462764620780945, 0.23390065133571625, 0.37125179171562195, 0.14430180191993713, 0.6157160997390747, 0.1116834506392479, 0.2595561444759369, 0.16742070019245148, 0.3359021246433258, 0.533530056476593, 0.23560592532157898, 0.11581568419933319, 0.5183773040771484, 0.16550150513648987, 0.0602763369679451, 0.20114555954933167, 0.07087943702936172, 0.30936118960380554, 0.1490786224603653, 0.3254110813140869, 0.11427178233861923, 0.07922613620758057, 0.19929729402065277, 0.062438055872917175, 0.14713799953460693, 0.48846355080604553, 0.5519556403160095, 0.1376502513885498, 0.5843381881713867, 0.032596610486507416, 0.06941287964582443, 0.16374000906944275, 0.3091338872909546, 0.161500945687294, 0.25305160880088806, 0.0, 0.6173039674758911, 0.03411887586116791, 0.1902449131011963, 0.3168026804924011, 0.19916146993637085, 0.41259968280792236, 0.5677651762962341, 0.09909486770629883, 0.1456712782382965, 0.14129643142223358, 0.08245731890201569, 0.08026248216629028, 0.003960685804486275, 0.10866305977106094, 0.10142911970615387, 0.2712870240211487, 0.05595926567912102, 0.062347281724214554, 0.22384148836135864, 0.13450562953948975, 0.14582864940166473, 0.3649275004863739, 0.12252809852361679, 0.17851997911930084, 0.024320218712091446, 0.14398019015789032, 0.05694327875971794, 0.0, 0.18937784433364868, 0.03548266366124153, 0.048464585095644, 0.2922627031803131, 0.521373450756073, 0.004569989163428545, 0.4895499050617218, 0.1098424568772316, 0.0029887487180531025, 0.0, 0.0, 0.0, 0.0653052106499672, 0.1858758181333542, 0.48902663588523865, 0.06719361990690231, 0.06685702502727509, 0.09669274836778641, 0.017705678939819336, 0.1324865221977234, 0.053216662257909775, 0.0, 0.05321013182401657, 0.0, 0.0, 0.04261920973658562, 0.11087550222873688, 0.275717169046402, 0.003510611830279231, 0.0011044464772567153, 0.0471857413649559, 0.0, 0.0, 0.04692104086279869, 0.0, 0.12230419367551804, 0.0, 0.06116527318954468, 0.0, 0.0, 0.0, 0.45186537504196167, 0.2160753756761551, 0.010133913718163967, 0.0, 0.09786956757307053, 0.42602211236953735, 0.22347766160964966, 0.07747004926204681, 0.12017518281936646, 0.024133380502462387, 0.18847841024398804, 0.0, 0.09612825512886047, 0.07078320533037186, 0.04152616113424301, 0.33660051226615906, 0.022383254021406174, 0.3327256441116333, 0.26147255301475525, 0.3247626721858978, 0.10130535066127777, 0.17510183155536652, 0.12282238155603409, 0.21896962821483612, 0.06826724112033844, 0.03669043630361557, 0.0864177718758583, 0.11069554835557938, 0.23663915693759918, 0.26924291253089905, 0.23132941126823425, 0.1984531283378601, 0.07551760971546173, 0.07319392263889313, 0.2265879362821579, 0.22736860811710358, 0.049381744116544724, 0.017555518075823784, 0.3632342517375946, 0.09214357286691666, 0.05111365392804146, 0.06992001086473465, 0.39474907517433167, 0.03203878179192543, 0.16272743046283722, 0.11068686842918396, 0.4528835415840149, 0.2098897397518158, 0.44122281670570374, 0.09103568643331528, 0.5272004008293152, 0.07477529346942902, 0.05032402649521828, 0.028658874332904816, 0.19204682111740112, 0.15092700719833374, 0.28948476910591125, 0.2730105519294739, 0.24989186227321625, 0.08524223417043686, 0.02225424163043499, 0.0, 0.25566887855529785, 0.052726998925209045, 0.12204962223768234, 0.2708979845046997, 0.06376197189092636, 0.18603381514549255, 0.1928047388792038, 0.5606985092163086, 0.04014647379517555, 0.5591578483581543, 0.4943339228630066, 0.01701469160616398, 0.04754946753382683, 0.25764721632003784, 0.5411548018455505, 0.09929408878087997, 0.5831502079963684, 0.07919837534427643, 0.0007100652437657118, 0.4133942127227783, 0.08287705481052399, 0.22164902091026306, 0.0910978838801384, 0.23057208955287933, 0.07023078948259354, 0.18087159097194672, 0.033983707427978516, 0.21158374845981598, 0.08877723664045334, 0.33021655678749084, 0.010518482886254787, 0.06098386272788048, 0.07712879031896591, 0.23645968735218048, 0.4621025323867798, 0.0, 0.0, 0.0, 0.1510663777589798, 0.1461016833782196, 0.08734744042158127, 0.1027468591928482, 0.048080507665872574, 0.0, 0.07883985340595245, 0.0, 0.044226277619600296, 0.14374396204948425, 0.026696601882576942, 0.1458042562007904, 0.4039335250854492, 0.0, 0.10196667909622192, 0.0, 0.0, 0.0, 0.07824259996414185, 0.09978944808244705, 0.10567817836999893, 0.0, 0.379948228597641, 0.0, 0.26638513803482056, 0.04897284135222435, 0.0, 0.1924847960472107, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.02484472282230854, 0.036473069339990616, 0.0, 0.0, 0.0, 0.0, 0.0, 0.04626530036330223, 0.11135971546173096, 0.10483164340257645, 0.0, 0.15411296486854553, 0.4530754089355469, 0.0, 0.013645726256072521, 0.05372460559010506, 0.0, 0.0, 0.0, 0.22489669919013977, 0.0, 0.06953686475753784, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.01542856264859438, 0.0, 0.0, 0.0, 0.0, 0.02140982262790203, 0.007241898216307163, 0.05223330110311508, 0.0, 0.1275116354227066, 0.0, 0.0, 0.0, 0.0, 0.0, 0.04171799123287201, 0.0, 0.11090533435344696, 0.38317620754241943, 0.17114302515983582, 0.0, 0.0, 0.0, 0.1924714595079422, 0.0, 0.045474763959646225, 0.0, 0.0, 0.0, 0.0, 0.0, 0.3287542760372162, 0.0, 0.13490131497383118, 0.43105554580688477, 0.0, 0.0, 0.0, 0.011700586415827274, 0.07270429283380508, 0.0, 0.0, 0.0, 0.0, 0.05031796917319298, 0.09660445898771286, 0.0, 0.0, 0.0, 0.2831871509552002, 0.003920961171388626, 0.0, 0.1403045505285263, 0.0, 0.15641102194786072, 0.15364351868629456, 0.0, 0.0, 0.0, 0.0, 0.040286239236593246, 0.1790570765733719, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.055453959852457047, 0.0, 0.0, 0.0, 0.05248250439763069, 0.035639822483062744, 0.0, 0.0, 0.0, 0.173054501414299, 0.0, 0.12123335152864456, 0.3383525609970093, 0.0, 0.05220433697104454, 0.06319531798362732, 0.3782670497894287, 0.0, 0.08999831974506378, 0.0, 0.0, 0.22903886437416077], [0.029995810240507126, 0.04808514565229416, 0.0, 0.1974327713251114, 0.027895072475075722, 0.04312717542052269, 0.04751582443714142, 0.029963161796331406, 0.04833812639117241, 0.29948776960372925, 0.0, 0.1813119798898697, 0.06753478199243546, 0.0, 0.0, 0.0, 0.12355108559131622, 0.3285831809043884, 0.022983137518167496, 0.2973034679889679, 0.4012450575828552, 0.024698225781321526, 0.3124306797981262, 0.17140188813209534, 0.0, 0.0846443772315979, 0.28711122274398804, 0.31761306524276733, 0.0, 0.3509129285812378, 0.0, 0.006618903484195471, 0.0, 0.2463880181312561, 0.0, 0.35369330644607544, 0.0, 0.022723369300365448, 0.0, 0.18291884660720825, 0.03304453194141388, 0.4031003713607788, 0.0, 0.3785797655582428, 0.0, 0.047553665935993195, 0.1808428019285202, 0.03676044940948486, 0.0, 0.25517794489860535, 0.0077936891466379166, 0.18325187265872955, 0.0, 0.29377150535583496, 0.39769095182418823, 0.0, 0.09526362270116806, 0.0, 0.0, 0.060682427138090134, 0.11703341454267502, 0.17461448907852173, 0.16370940208435059, 0.0, 0.19028881192207336, 0.012326117604970932, 0.32486340403556824, 0.00043547109817154706, 0.32559043169021606, 0.46313613653182983, 0.0, 0.357503205537796, 0.10381760448217392, 0.05971578508615494, 0.0, 0.3429437577724457, 0.0, 0.0, 0.2715166211128235, 0.0, 0.0, 0.08087415248155594, 0.0844893753528595, 0.11736831068992615, 0.01534086000174284, 0.37190863490104675, 0.3064574599266052, 0.2275683581829071, 0.3289209008216858, 0.02413785271346569, 0.35265880823135376, 0.38896456360816956, 0.013179166242480278, 0.18411293625831604, 0.0, 0.29159170389175415, 0.06562675535678864, 0.10209480673074722, 0.4364232122898102, 0.37349212169647217, 0.0011486110743135214, 0.0, 0.0, 0.09570522606372833, 0.3696880638599396, 0.0437023900449276, 0.45551109313964844, 0.004299893509596586, 0.003743828507140279, 0.4051244854927063, 0.40620166063308716, 0.026320477947592735, 0.17368556559085846, 0.4457533359527588, 0.4003395736217499, 0.02793472446501255, 0.0, 0.1661592423915863, 0.33979135751724243, 0.23295824229717255, 0.3218156695365906, 0.18372367322444916, 0.41421055793762207, 0.2304990142583847, 0.008464673534035683, 0.20849932730197906, 0.0, 0.409006804227829, 0.07612010836601257, 0.09124338626861572, 0.06674037128686905, 0.3284169137477875, 0.2324965000152588, 0.23257531225681305, 0.0, 0.37258633971214294, 0.39880409836769104, 0.2708452045917511, 0.0, 0.0, 0.0, 0.07622434943914413, 0.0, 0.38677898049354553, 0.37323033809661865, 0.0, 0.0, 0.3787187337875366, 0.41590264439582825, 0.0, 0.0, 0.013700706884264946, 0.34068214893341064, 0.0, 0.44690507650375366, 0.0, 0.19318699836730957, 0.0, 0.146050825715065, 0.0, 0.36538732051849365, 0.0, 0.13338668644428253, 0.010318311862647533, 0.0, 0.03175235167145729, 0.08485487103462219, 0.0, 0.028547348454594612, 0.06434184312820435, 0.36386463046073914, 0.016088047996163368, 0.0, 0.0, 0.2433265745639801, 0.09683483093976974, 0.0081121064722538, 0.2993049621582031, 0.0, 0.03296453133225441, 0.0, 0.3576352298259735, 0.0, 0.0, 0.37097451090812683, 0.0, 0.0, 0.0, 0.30634453892707825, 0.0, 0.2909482419490814, 0.371548056602478, 0.0, 0.3038357198238373, 0.02602030336856842, 0.26665276288986206, 0.0, 0.18512453138828278, 0.05558723956346512, 0.0, 0.0668613612651825, 0.23620487749576569, 0.0, 0.3942877948284149, 0.19180457293987274, 0.05541394278407097, 0.05550873652100563, 0.0, 0.3338901102542877, 0.0, 0.0, 0.0, 0.0, 0.03912005200982094, 0.12046361714601517, 0.060883793979883194, 0.06886465847492218, 0.04556843638420105, 0.18050247430801392, 0.1663094162940979, 0.06997806578874588, 0.22036466002464294, 0.05002279952168465, 0.013514039106667042, 0.3770522177219391, 0.04358687996864319, 0.3273303210735321, 0.0, 0.04033203050494194, 0.04627319052815437, 0.054657187312841415, 0.03988303989171982, 0.333781361579895, 0.006897483486682177, 0.03126648813486099, 0.04271398484706879, 0.2580338418483734, 0.30791550874710083, 0.0, 0.39611902832984924, 0.0, 0.03388061374425888, 0.06830155849456787, 0.0, 0.0, 0.06008567661046982, 0.21919052302837372, 0.4143228232860565, 0.0, 0.0, 0.0, 0.3634015917778015, 0.18995505571365356, 0.12522529065608978, 0.2109210193157196, 0.3478885889053345, 0.11475735157728195, 0.00432844553142786, 0.3304064869880676, 0.29961055517196655, 0.2298605889081955, 0.3194379508495331, 0.00912780687212944, 0.1862918585538864, 0.0, 0.3791615962982178, 0.020761549472808838, 0.2999231517314911, 0.0009726751013658941, 0.0, 0.0427650548517704, 0.0, 0.08066022396087646, 0.06407929956912994, 0.22947454452514648, 0.22989995777606964, 0.0, 0.021820975467562675, 0.2553236782550812, 0.2802274227142334, 0.25539812445640564, 0.403150349855423, 0.036714956164360046, 0.3963521420955658, 0.013513409532606602, 0.014825217425823212, 0.24606257677078247, 0.27456724643707275, 0.26073914766311646, 0.41217276453971863, 0.13501112163066864, 0.010690336115658283, 0.08313446491956711, 0.31691932678222656, 0.0, 0.08853871375322342, 0.0, 0.3509698510169983, 0.025727529078722, 0.2792162299156189, 0.08775188773870468, 0.0, 0.0, 0.37038394808769226, 0.013625838793814182, 0.3480634093284607, 0.17582444846630096, 0.0, 0.1942547708749771, 0.0, 0.0, 0.0, 0.0551520474255085, 0.13449683785438538, 0.0, 0.32378876209259033, 0.0, 0.0, 0.0, 0.0, 0.32002153992652893, 0.0, 0.0, 0.21209442615509033, 0.2519865036010742, 0.05655962973833084, 0.0, 0.0, 0.022347476333379745, 0.21049176156520844, 0.0, 0.0, 0.0, 0.00942427571862936, 0.013928987085819244, 0.0, 0.279541939496994, 0.16428925096988678, 0.0, 0.0, 0.34115171432495117, 0.0, 0.27572759985923767, 0.0, 0.0, 0.0, 0.0, 0.0, 0.32874634861946106, 0.060932572931051254, 0.0, 0.0, 0.0, 0.0, 0.18762598931789398, 0.0, 0.0273696631193161, 0.0, 0.35129061341285706, 0.025826100260019302, 0.2768027186393738, 0.07430826872587204, 0.021264126524329185], [0.019907649606466293, 0.17296202480793, 0.0, 0.0, 0.005362444557249546, 0.13715916872024536, 0.06422200798988342, 0.0, 0.0, 0.0, 0.0, 0.0, 0.15939483046531677, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.10248720645904541, 0.0, 0.0002485568984411657, 0.0, 0.01851092278957367, 0.0, 0.0, 0.0, 0.0, 0.0, 0.013649959117174149, 0.0, 0.0, 0.0, 0.0, 0.0, 0.25165215134620667, 0.2758621871471405, 0.06483837962150574, 0.21984951198101044, 0.0, 0.0, 0.0, 0.0, 0.0, 0.016925888136029243, 0.01529675256460905, 0.0, 0.0, 0.1616477519273758, 0.06691098213195801, 0.0, 0.0, 0.0, 0.021687336266040802, 0.23468798398971558, 0.2977699637413025, 0.02551768533885479, 0.04008175805211067, 0.01973973959684372, 0.08407957851886749, 0.09154223650693893, 0.0, 0.03957812488079071, 0.23801007866859436, 0.0, 0.01527158822864294, 0.005785742308944464, 0.0, 0.2807658314704895, 0.0142303965985775, 0.0203761737793684, 0.036253396421670914, 0.0, 0.0, 0.009744424372911453, 0.0, 0.0, 0.0, 0.5951471924781799, 0.3526819050312042, 0.2815045714378357, 0.1314229667186737, 0.05361000820994377, 0.0076712048612535, 0.029941702261567116, 0.041738416999578476, 0.004739780444651842, 0.026555292308330536, 0.01241952646523714, 0.0, 0.0, 0.023315569385886192, 0.0, 0.0, 0.04630430042743683, 0.05936934053897858, 0.001228147535584867, 0.0, 0.02902648225426674, 0.038214266300201416, 0.0, 0.19374297559261322, 0.0, 0.0, 0.0, 0.3170386254787445, 0.0, 0.008726222440600395, 0.0, 0.030032822862267494, 0.06867717951536179, 0.0, 0.0, 0.010159875266253948, 0.09464658796787262, 0.07494107633829117, 0.008173326961696148, 0.034230221062898636, 0.0, 0.012707005254924297, 0.0, 0.0, 0.0, 0.19161243736743927, 0.0, 0.0, 0.020626921206712723, 0.09448426216840744, 0.039022721350193024, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.14666646718978882, 0.042252317070961, 0.1598772406578064, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.18288397789001465, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.17896804213523865, 0.11226610839366913, 0.10352714359760284, 0.0, 0.0, 0.013271626085042953, 0.0037047623191028833, 0.41949522495269775, 0.4564361870288849, 0.24019742012023926, 0.20249122381210327, 0.2030239701271057, 0.0374598503112793, 0.12929490208625793, 0.023174310103058815, 0.42203253507614136, 0.4504058361053467, 0.25905686616897583, 0.012909069657325745, 0.12224342674016953, 0.0, 0.0, 0.33962395787239075, 0.21252553164958954, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.20253969728946686, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.02314210683107376, 0.20250381529331207, 0.10820381343364716, 0.3360230028629303, 0.1543276458978653, 0.07857978343963623, 0.0, 0.02101985737681389, 0.04712895303964615, 0.027370871976017952, 0.011198659427464008, 0.13588182628154755, 0.3497011065483093, 0.1340664178133011, 0.048791926354169846, 0.026130465790629387, 0.021884847432374954, 0.6428056955337524, 0.40016838908195496, 0.19524729251861572, 0.30280929803848267, 0.24830231070518494, 0.340431809425354, 0.40889838337898254, 0.29051315784454346, 0.053409360349178314, 0.18351629376411438, 0.10055838525295258, 0.1493353247642517, 0.341407835483551, 0.09274922311306, 0.04211992025375366, 0.334307461977005, 0.07214488089084625, 0.1267448514699936, 0.432720422744751, 0.31434258818626404, 0.20992667973041534, 0.13010014593601227, 0.07098440825939178, 0.10230384021997452, 0.07940930128097534, 0.13003554940223694, 0.05792694166302681, 0.06440722942352295, 0.2916612923145294, 0.04814201965928078, 0.09900673478841782, 0.4077867567539215, 0.3644730746746063, 0.09518551081418991, 0.05055393651127815, 0.24709877371788025, 0.10032111406326294, 0.041623279452323914, 0.04325637221336365, 0.4670853018760681, 0.08989034593105316, 0.0834505707025528, 0.11907587945461273, 0.16386404633522034, 0.1281290203332901, 0.053474850952625275, 0.19903087615966797, 0.13519644737243652, 0.08521244674921036, 0.07750552892684937, 0.08646395802497864, 0.04084383323788643, 0.07111585885286331, 0.2718045115470886, 0.10658539831638336, 0.036448556929826736, 0.05722997337579727, 0.05864478275179863, 0.07287003844976425, 0.6892805695533752, 0.3498782813549042, 0.09123925119638443, 0.12194902449846268, 0.2504761815071106, 0.08159457892179489, 0.07902612537145615, 0.04577477276325226, 0.0, 0.05475151911377907, 0.05033421888947487, 0.03136696293950081, 0.0, 0.27659279108047485, 0.0, 0.06504635512828827, 0.06709643453359604, 0.0848773792386055, 0.057883620262145996, 0.0494454950094223, 0.003000367432832718, 0.09613648802042007, 0.09761794656515121, 0.16105803847312927, 0.05123426392674446, 0.009462482295930386, 0.32269486784935, 0.04519620165228844, 0.0, 0.0, 0.0, 0.16545747220516205, 0.015130935236811638, 0.0, 0.0, 0.0, 0.0, 0.03464516997337341, 0.0, 0.02266468107700348, 0.0, 0.5037193298339844, 0.2788721024990082, 0.14653243124485016, 0.0, 0.0, 0.0, 0.14764618873596191, 0.07105933129787445, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.10817701369524002, 0.015621698461472988, 0.0, 0.07324862480163574, 0.0, 0.02337624877691269, 0.0434599369764328, 0.025297610089182854, 0.0, 0.0, 0.015626078471541405, 0.0, 0.0, 0.0, 0.0, 0.0, 0.36900094151496887, 0.44368040561676025, 0.34778323769569397, 0.21740074455738068, 0.016966164112091064, 0.0, 0.007479475345462561, 0.2034652680158615, 0.0, 0.0, 0.0, 0.0, 0.3349291682243347, 0.20582252740859985, 0.0046554915606975555, 0.0, 0.25581857562065125, 0.013669251464307308, 0.19325436651706696, 0.0374503992497921]]]},\n",
       "            {\n",
       "            'hltrCFG': {'tokenization_config': {\"token_prefix\": \"\", \"partial_token_prefix\": \"##\"}\n",
       "                }\n",
       "            })\n",
       "         }, function (err) {\n",
       "            console.log(err);\n",
       "        })"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import ecco\n",
    "lm = ecco.from_pretrained('distilbert-base-uncased', activations=True)\n",
    "text = ''' Now I ask you: \\n what can be expected of man since he is a being endowed with strange qualities? Shower upon him every earthly blessing, drown him in a sea of happiness, so that nothing but bubbles of bliss can be seen on the surface; give him economic prosperity, such that he should have nothing else to do but sleep, eat cakes and busy himself with the continuation of his species, and even then out of sheer ingratitude, sheer spite, man would play you some nasty trick. He would even risk his cakes and would deliberately desire the most fatal rubbish, the most uneconomical absurdity, simply to introduce into all this positive good sense his fatal fantastic element. It is just his fantastic dreams, his vulgar folly that he will desire to retain, simply in order to prove to himself--as though that were so necessary-- that men still are men and not the keys of a piano, which the laws of nature threaten to control so completely that soon one will be able to desire nothing but by the calendar. And that is not all: even if man really were nothing but a piano-key, even if this were proved to him by natural science and mathematics, even then he would not become reasonable, but would purposely do something perverse out of simple ingratitude, simply to gain his point. And if he does not find means he will contrive destruction and chaos, will contrive sufferings of all sorts, only to gain his point! He will launch a curse upon the world, and as only man can curse (it is his privilege, the primary distinction between him and other animals), may be by his curse alone he will attain his object--that is, convince himself that he is a man and not a piano-key!\n",
    "'''\n",
    "inputs = lm.tokenizer([text], return_tensors=\"pt\")\n",
    "output = lm(inputs)\n",
    "nmf_1 = output.run_nmf(n_components=8) \n",
    "nmf_1.explore()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1567ef2e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}