examples/ruGPT3XL_generation.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r-XkQgf-ufgE"
      },
      "source": [
        "## Install lib"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "gH2kLmcXmL3e"
      },
      "outputs": [],
      "source": [
        "%%bash\n",
        "export LD_LIBRARY_PATH=/usr/lib/"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NRkBHiuOFYB-"
      },
      "outputs": [],
      "source": [
        "!apt-get install clang-9 llvm-9 llvm-9-dev llvm-9-tools"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "GfdFeAVylpIQ"
      },
      "outputs": [],
      "source": [
        "!rm -rf apex"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zFqiiOnRnh9p"
      },
      "outputs": [],
      "source": [
        "%%bash\n",
        "git clone https://github.com/qywu/apex\n",
        "cd apex\n",
        "pip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" ./"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7t-l-Gq6mGYg"
      },
      "outputs": [],
      "source": [
        "!pip install triton"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LEyrQ-spCdhR"
      },
      "outputs": [],
      "source": [
        "!DS_BUILD_CPU_ADAM=1 DS_BUILD_SPARSE_ATTN=1 pip install deepspeed"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "---7k9osCliv",
        "outputId": "7e4d0400-028c-4edd-a7b1-0a51ef53bcb5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--------------------------------------------------\n",
            "DeepSpeed C++/CUDA extension op report\n",
            "--------------------------------------------------\n",
            "NOTE: Ops not installed will be just-in-time (JIT) compiled at\n",
            "      runtime if needed. Op compatibility means that your system\n",
            "      meet the required dependencies to JIT install the op.\n",
            "--------------------------------------------------\n",
            "JIT compiled ops requires ninja\n",
            "ninja .................. \u001b[92m[OKAY]\u001b[0m\n",
            "--------------------------------------------------\n",
            "op name ................ installed .. compatible\n",
            "--------------------------------------------------\n",
            "cpu_adam ............... \u001b[92m[YES]\u001b[0m ...... \u001b[92m[OKAY]\u001b[0m\n",
            "cpu_adagrad ............ \u001b[93m[NO]\u001b[0m ....... \u001b[92m[OKAY]\u001b[0m\n",
            "fused_adam ............. \u001b[93m[NO]\u001b[0m ....... \u001b[92m[OKAY]\u001b[0m\n",
            "fused_lamb ............. \u001b[93m[NO]\u001b[0m ....... \u001b[92m[OKAY]\u001b[0m\n",
            "sparse_attn ............ \u001b[92m[YES]\u001b[0m ...... \u001b[92m[OKAY]\u001b[0m\n",
            "transformer ............ \u001b[93m[NO]\u001b[0m ....... \u001b[92m[OKAY]\u001b[0m\n",
            "stochastic_transformer . \u001b[93m[NO]\u001b[0m ....... \u001b[92m[OKAY]\u001b[0m\n",
            "\u001b[93m [WARNING] \u001b[0m async_io requires the dev libaio .so object and headers but these were not found.\n",
            "\u001b[93m [WARNING] \u001b[0m async_io: please install the libaio-dev package with apt\n",
            "\u001b[93m [WARNING] \u001b[0m If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found.\n",
            "async_io ............... \u001b[93m[NO]\u001b[0m ....... \u001b[93m[NO]\u001b[0m\n",
            "transformer_inference .. \u001b[93m[NO]\u001b[0m ....... \u001b[92m[OKAY]\u001b[0m\n",
            "utils .................. \u001b[93m[NO]\u001b[0m ....... \u001b[92m[OKAY]\u001b[0m\n",
            "quantizer .............. \u001b[93m[NO]\u001b[0m ....... \u001b[92m[OKAY]\u001b[0m\n",
            "--------------------------------------------------\n",
            "DeepSpeed general environment info:\n",
            "torch install path ............... ['/usr/local/lib/python3.8/dist-packages/torch']\n",
            "torch version .................... 1.12.1+cu113\n",
            "torch cuda version ............... 11.3\n",
            "nvcc version ..................... 11.2\n",
            "deepspeed install path ........... ['/usr/local/lib/python3.8/dist-packages/deepspeed']\n",
            "deepspeed info ................... 0.5.10, unknown, unknown\n",
            "deepspeed wheel compiled w. ...... torch 1.12, cuda 11.3\n"
          ]
        }
      ],
      "source": [
        "!ds_report"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rl1JIPsK8viz"
      },
      "source": [
        "Test, that deepspeed was installed correctly."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "DqPIkEYjCmLH"
      },
      "outputs": [],
      "source": [
        "import deepspeed.ops.sparse_attention.sparse_attn_op"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "gw4lJHfX4QnB"
      },
      "outputs": [],
      "source": [
        "!rm -rf ru-gpts"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "t43yH5k1jtZZ"
      },
      "outputs": [],
      "source": [
        "!git clone  https://github.com/sberbank-ai/ru-gpts"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "H2XiJvm_tQgL"
      },
      "outputs": [],
      "source": [
        "!pip install transformers"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install huggingface_hub"
      ],
      "metadata": {
        "id": "mnbodJMXCjij"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HN9FJ3rKnO5Q"
      },
      "outputs": [],
      "source": [
        "!pip install timm==0.3.2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "1RCujWwJUwm-"
      },
      "outputs": [],
      "source": [
        "!cp ru-gpts/src_utils/trainer_pt_utils.py /usr/local/lib/python3.8/dist-packages/transformers/trainer_pt_utils.py"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "GHhCIrDQAAdJ"
      },
      "outputs": [],
      "source": [
        "!cp ru-gpts/src_utils/_amp_state.py /usr/local/lib/python3.8/dist-packages/apex/amp/_amp_state.py"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iWnXPnNR8nW4"
      },
      "source": [
        "Here restart colab env!!!"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7TbybJfIpBVa"
      },
      "source": [
        "# Test model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "P5P4MjOkwTtN"
      },
      "outputs": [],
      "source": [
        "import deepspeed"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rU8lvJHAjpPQ"
      },
      "source": [
        "### Load model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "EnTy1SEajpPV"
      },
      "outputs": [],
      "source": [
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "Z-DSEz0ljpPV"
      },
      "outputs": [],
      "source": [
        "import sys\n",
        "sys.path.append(\"ru-gpts/\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "_9GABoxNVpH4"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "os.environ[\"USE_DEEPSPEED\"] = \"1\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "_HeCFnJEjpPV"
      },
      "outputs": [],
      "source": [
        "from src.xl_wrapper import RuGPT3XL"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OdyughHDjpPV"
      },
      "source": [
        "Note! seq_len is max sequence length for generation used in generation process. Max avialable seq_len is 2048 (in tokens).\n",
        "Also inference takes around 10 Gb GPU memory."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ik1FP4Jcq8tg",
        "outputId": "10cfa305-5501-42e3-ba0b-1ecd99f9d814"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ],
      "source": [
        "import torch\n",
        "torch.cuda.is_available()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "BKpw5ik2uWrw"
      },
      "outputs": [],
      "source": [
        "from src import mpu"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "id": "T0hDbxdvvUiO"
      },
      "outputs": [],
      "source": [
        "os.environ[\"MASTER_ADDR\"] = \"127.0.0.1\"\n",
        "os.environ[\"MASTER_PORT\"] = \"5000\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 222,
          "referenced_widgets": [
            "a2b82782bb154ebeafd301e2ad9127fa",
            "eab7c921d1404991b91483198c70357f",
            "1be1ed8832bb42ceb2f6ae4786fc0597",
            "661386adcd334f6c868360c4796cbb62",
            "c1cb7cd3e8794ac8ade560564b46d6f8",
            "4a3c58d4fb6f45549682bc6199c7a68e",
            "c23b3ffb0350444cb829d393532c8f73",
            "2ed8ef395f2b4d8bbdbf873397f9ddfb",
            "78517fe47e804d06b92a1defed591aed",
            "47b894ea77334c7ba9495046c5366338",
            "0131ffe4872d4dfbbb8a1d3ad12a5ac7",
            "dd62eb1fd87e49eb8893e21141a0efa9",
            "86f6e7ac7414499cada6f7c68266a564",
            "03dcd4ffe444421388a2bc4183f1c5d0",
            "be97ef7753f9467d85c513f8770eb1b9",
            "b3751931d73542cc8ad388e7322b48a4",
            "8a3943d9f2f74f3f9a3ea911b7f96da3",
            "24189f7e8c2149408af9266d0052dd65",
            "bd57cb6eac35491ea699158d4bb3187c",
            "6795288096004aae9aa0d9cceb45603c",
            "a50eeeb97b1d4fcf9a6ff1f3e34cdecf",
            "78d92f6ff39a4c39b8e9f7449e0313b4",
            "0c4711135e1f498eac79fdf56e14af07",
            "ae1772f0b2044a02a55c48630e495a3f",
            "4f249d1e5be24048b393b7fe209e15bb",
            "0e3debfa8c5449daae0ba3c5371c3fdd",
            "2133e266f1d74666bf260f74bd9c06da",
            "ce3acbf249524fd3a18d18662108a504",
            "a35cbebc740a4d10afb5898280966a1b",
            "7fe10ce044254824be356a23598ea50b",
            "4959b00d6bbb441594383593ad83bf3d",
            "885b8760c603467b8223a49aedc72a62",
            "16e09195cf1c467ba8b91176365dd729",
            "63d2f7943db04de5b0b5dd72d53230e4"
          ]
        },
        "id": "56aNJNPYjpPW",
        "outputId": "58ba12a9-6e50-42e9-e71e-4c9b01fe0aeb",
        "scrolled": true
      },
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "> initializing model parallel with size 1\n",
            "> initializing model parallel cuda seeds on global rank 0, model parallel rank 0, and data parallel rank 0 with model parallel seed: 3952 and data parallel seed: 1234\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a2b82782bb154ebeafd301e2ad9127fa",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.57M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "dd62eb1fd87e49eb8893e21141a0efa9",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.23M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0c4711135e1f498eac79fdf56e14af07",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/2.63G [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "63d2f7943db04de5b0b5dd72d53230e4",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/653 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Use alternating sparse & dense attention layers\n",
            "Load checkpoint from /root/.cache/huggingface/hub/models--sberbank-ai--rugpt3xl/snapshots/c3c28881c0bfd644772ecf42e062e100f06626eb/mp_rank_00_model_states.pt\n",
            "Model Loaded\n"
          ]
        }
      ],
      "source": [
        "# gpt = RuGPT3XL.from_pretrained(\"sberbank-ai/rugpt3xl\", seq_len=512)\n",
        "gpt = RuGPT3XL.from_pretrained(\"sberbank-ai/rugpt3xl\", seq_len=512)\n",
        "# model parallel group is not initialized - если не подключена gpu"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7qR_3287jpPW"
      },
      "source": [
        "### Get logits"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "id": "9yfRylnnjpPW"
      },
      "outputs": [],
      "source": [
        "logits = gpt(\"Кто был президентом США в 2020? \").logits"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BHcwcMwHjpPX",
        "outputId": "13caa13e-747b-44a3-9c9f-e8fe0773d76f"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(torch.Tensor, torch.Size([1, 8, 50264]))"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ],
      "source": [
        "type(logits), logits.shape"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J7a29eADjpPX"
      },
      "source": [
        "### Get loss"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "id": "q44D1VlojpPX"
      },
      "outputs": [],
      "source": [
        "input_ids = [gpt.tokenizer(\"Кто был президентом США в 2020? \")['input_ids']]\n",
        "labels = input_ids"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "id": "9zIpNDegjpPX"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "\n",
        "\n",
        "with torch.no_grad():\n",
        "    loss = gpt(input_ids=input_ids, labels=labels).loss"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WcYvXFsBjpPY",
        "outputId": "d16f03f8-c418-4491-a1d1-826229157d7e"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[tensor(4.3874, device='cuda:0')]"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ],
      "source": [
        "loss"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Lgi-JUNijpPY"
      },
      "source": [
        "### Simple generation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "id": "Z0AIfKPPjpPY"
      },
      "outputs": [],
      "source": [
        "def filter_resuls(nr):\n",
        "    return [x[:x.find(\"<|endoftext|>\")] for x in nr]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aHWixkU3jpPY"
      },
      "source": [
        "Greedy decoding"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wla741VxjpPY",
        "outputId": "55f138d9-0ed3-4081-d0a1-41304fb78f35"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Кто был президентом США в 2020? \\nВ этом году выборы президента Соединенных Штатов Америки пройдут уже через несколько дней. И, как и всегда на протяжении последних лет (а это не первый раз), кандидаты будут бороться за право стать главой государства с помощь']"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ],
      "source": [
        "filter_resuls(gpt.generate(\n",
        "    \"Кто был президентом США в 2020? \",\n",
        "    max_length=50,\n",
        "    no_repeat_ngram_size=3,\n",
        "    repetition_penalty=2.,\n",
        "))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I_ltaza-jpPZ"
      },
      "source": [
        "sample"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3oPE1lP2jpPZ",
        "outputId": "4975fec2-c3d8-47d0-c39e-c94b7172a1f4"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Кто был президентом США в 2020? \\nНовым, на первый взгляд странным и непонятным образом определилось будущее Америки. За несколько часов до конца таймера ожидания выборов президента уже ясно было - кто займет президентский пост после окончания голосования – Дональд Трам',\n",
              " 'Кто был президентом США в 2020? \\n\"Я никогда не выйду живым из этого леса\",- говорит главный герой фильма \"Зеленая миля\". Услышав это, начинаешь задумываться: а так ли хорош твой опыт жизни на другой стороне земного шара',\n",
              " 'Кто был президентом США в 2020? \\nВы хотите знать, как будет выглядеть ваш мир через сорок лет после того срока своего президентства уходящего следующего президента? Посмотрите на эти четыре изображения и попробуйте угадать кто бы это мог быть. \\n Вот первый предполаг',\n",
              " 'Кто был президентом США в 2020? \\nВ интернете появился новый опрос на тему, которую так трудно назвать \"демократичной\". Народ хочет знать о том. какие президенты стояли у руля их страны и за сколько часов до собственной смерти они успевали подписат',\n",
              " 'Кто был президентом США в 2020? \\nВот, мы знаем. Кто президент России с 2018 — точно: Владимир Владимирович Путин наш дорогой и любимый (пока не отлучим его по суду). У остальных тоже есть свои любимчики... или там ненав']"
            ]
          },
          "metadata": {},
          "execution_count": 32
        }
      ],
      "source": [
        "filter_resuls(gpt.generate(\n",
        "    \"Кто был президентом США в 2020? \", do_sample=True, num_return_sequences=5,\n",
        "    max_length=50,\n",
        "    no_repeat_ngram_size=3,\n",
        "    repetition_penalty=2.,\n",
        "))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9FbokoJDjpPZ"
      },
      "source": [
        "### Top_k top_p filtering"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8L-CF6uLjpPZ",
        "outputId": "4fa855d5-7ab8-440f-8d4f-5ae6382c09a3"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['Александр Сергеевич Пушкин родился в \\nМоскве. В 1799 г., после того как его отец, отставной поручик лейб-гвардии Преображенского полка Александр Иванович (17451816), женился на вдове капитана Екатерине Петро',\n",
              " 'Александр Сергеевич Пушкин родился в \\xa01899 году. Он был сыном великого поэта Александра Сергеевича Пушкина и Натальи Гончаровой - сестры прапрабабушки Льва Толстого Александры Николаевны, которая была замужем за князем Николаем Васильевичем Трубецки',\n",
              " 'Александр Сергеевич Пушкин родился в \\n1817 году. Его отец – Александр Иванович, служил чиновником при министерстве внутренних дел Российской империи и умер рано; его мать Мария Алексеевна Ганнибал (урожденная Пушкина), урождённая Энгельга',\n",
              " 'Александр Сергеевич Пушкин родился в \\n1817 г. (по другим сведениям 1820). В детстве и ранней юности жил с родителями за границей, учился у лучших педагогов Франции - Лагарпа, Жозефа Мари Ашара. С ранних лет увле',\n",
              " 'Александр Сергеевич Пушкин родился в \\n1799 г. Его отец, Александр Матвеевич Ганнибал (ум.-1837), был родом из деревни Слепушкино Тверской губернии; дед - отставной майор Иван Петрович Гамильтон-Пушки']"
            ]
          },
          "execution_count": 20,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "filter_resuls(gpt.generate(\n",
        "    \"Александр Сергеевич Пушкин родился в \",\n",
        "    top_k=5,\n",
        "    top_p=0.95,\n",
        "    temperature=1.2,\n",
        "    num_return_sequences=5,\n",
        "    do_sample=True,\n",
        "    max_length=50,\n",
        "    no_repeat_ngram_size=3,\n",
        "    repetition_penalty=2.,\n",
        "))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "scs7xKdhjpPZ"
      },
      "source": [
        "### Beamsearch"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7Qw65CVzjpPZ",
        "outputId": "55950661-61c4-4887-8c89-4d58d391668f"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['Александр Сергеевич Пушкин родился в \\n1799 году в селе Михайловском Псковской губернии. Его отец, \\nАлександр Львович Пушкин, происходил из старинного \\nдворянского рода. Мать, урожденная Ганнибал, был',\n",
              " 'Александр Сергеевич Пушкин родился в \\n1799 году в селе Михайловском Псковской губернии. Его отец, \\nАлександр Львович Пушкин, происходил из старинного \\nдворянского рода. Мать поэта, Мария Алексеевна Ганнибал',\n",
              " 'Александр Сергеевич Пушкин родился в \\n1799 году в селе Михайловском Псковской губернии. Его отец, \\nАлександр Львович Пушкин, происходил из старинного \\nдворянского рода. Мать поэта, Наталья Николаевна \\nПушкин',\n",
              " 'Александр Сергеевич Пушкин родился в \\n1799 году в селе Михайловском Псковской губернии. Его отец, \\nАлександр Львович Пушкин, происходил из старинного \\nдворянского рода. Мать, урождённая Ганнибал, был',\n",
              " 'Александр Сергеевич Пушкин родился в \\n1799 году в селе Михайловском Псковской губернии. Его отец, \\nАлександр Львович Пушкин, происходил из старинного \\nдворянского рода. Мать, урожденная Ганнибал,']"
            ]
          },
          "execution_count": 21,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "filter_resuls(gpt.generate(\n",
        "    text=\"Александр Сергеевич Пушкин родился в \",\n",
        "    max_length=50,\n",
        "    num_beams=10,\n",
        "    no_repeat_ngram_size=3,\n",
        "    repetition_penalty=2.,\n",
        "    num_return_sequences=5,\n",
        "))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sw4Iv-Dl3q05"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "ruGPT3XL_generation.ipynb",
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.12"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "a2b82782bb154ebeafd301e2ad9127fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_eab7c921d1404991b91483198c70357f",
              "IPY_MODEL_1be1ed8832bb42ceb2f6ae4786fc0597",
              "IPY_MODEL_661386adcd334f6c868360c4796cbb62"
            ],
            "layout": "IPY_MODEL_c1cb7cd3e8794ac8ade560564b46d6f8"
          }
        },
        "eab7c921d1404991b91483198c70357f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4a3c58d4fb6f45549682bc6199c7a68e",
            "placeholder": "​",
            "style": "IPY_MODEL_c23b3ffb0350444cb829d393532c8f73",
            "value": "Downloading: 100%"
          }
        },
        "1be1ed8832bb42ceb2f6ae4786fc0597": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2ed8ef395f2b4d8bbdbf873397f9ddfb",
            "max": 1572428,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_78517fe47e804d06b92a1defed591aed",
            "value": 1572428
          }
        },
        "661386adcd334f6c868360c4796cbb62": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_47b894ea77334c7ba9495046c5366338",
            "placeholder": "​",
            "style": "IPY_MODEL_0131ffe4872d4dfbbb8a1d3ad12a5ac7",
            "value": " 1.57M/1.57M [00:00&lt;00:00, 1.47MB/s]"
          }
        },
        "c1cb7cd3e8794ac8ade560564b46d6f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4a3c58d4fb6f45549682bc6199c7a68e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c23b3ffb0350444cb829d393532c8f73": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2ed8ef395f2b4d8bbdbf873397f9ddfb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "78517fe47e804d06b92a1defed591aed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "47b894ea77334c7ba9495046c5366338": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0131ffe4872d4dfbbb8a1d3ad12a5ac7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "dd62eb1fd87e49eb8893e21141a0efa9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_86f6e7ac7414499cada6f7c68266a564",
              "IPY_MODEL_03dcd4ffe444421388a2bc4183f1c5d0",
              "IPY_MODEL_be97ef7753f9467d85c513f8770eb1b9"
            ],
            "layout": "IPY_MODEL_b3751931d73542cc8ad388e7322b48a4"
          }
        },
        "86f6e7ac7414499cada6f7c68266a564": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8a3943d9f2f74f3f9a3ea911b7f96da3",
            "placeholder": "​",
            "style": "IPY_MODEL_24189f7e8c2149408af9266d0052dd65",
            "value": "Downloading: 100%"
          }
        },
        "03dcd4ffe444421388a2bc4183f1c5d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bd57cb6eac35491ea699158d4bb3187c",
            "max": 1230754,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6795288096004aae9aa0d9cceb45603c",
            "value": 1230754
          }
        },
        "be97ef7753f9467d85c513f8770eb1b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a50eeeb97b1d4fcf9a6ff1f3e34cdecf",
            "placeholder": "​",
            "style": "IPY_MODEL_78d92f6ff39a4c39b8e9f7449e0313b4",
            "value": " 1.23M/1.23M [00:00&lt;00:00, 2.22MB/s]"
          }
        },
        "b3751931d73542cc8ad388e7322b48a4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8a3943d9f2f74f3f9a3ea911b7f96da3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "24189f7e8c2149408af9266d0052dd65": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bd57cb6eac35491ea699158d4bb3187c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6795288096004aae9aa0d9cceb45603c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a50eeeb97b1d4fcf9a6ff1f3e34cdecf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "78d92f6ff39a4c39b8e9f7449e0313b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0c4711135e1f498eac79fdf56e14af07": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ae1772f0b2044a02a55c48630e495a3f",
              "IPY_MODEL_4f249d1e5be24048b393b7fe209e15bb",
              "IPY_MODEL_0e3debfa8c5449daae0ba3c5371c3fdd"
            ],
            "layout": "IPY_MODEL_2133e266f1d74666bf260f74bd9c06da"
          }
        },
        "ae1772f0b2044a02a55c48630e495a3f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ce3acbf249524fd3a18d18662108a504",
            "placeholder": "​",
            "style": "IPY_MODEL_a35cbebc740a4d10afb5898280966a1b",
            "value": "Downloading:  76%"
          }
        },
        "4f249d1e5be24048b393b7fe209e15bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7fe10ce044254824be356a23598ea50b",
            "max": 2631541505,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4959b00d6bbb441594383593ad83bf3d",
            "value": 1992434688
          }
        },
        "0e3debfa8c5449daae0ba3c5371c3fdd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_885b8760c603467b8223a49aedc72a62",
            "placeholder": "​",
            "style": "IPY_MODEL_16e09195cf1c467ba8b91176365dd729",
            "value": " 1.99G/2.63G [00:26&lt;00:08, 74.7MB/s]"
          }
        },
        "2133e266f1d74666bf260f74bd9c06da": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ce3acbf249524fd3a18d18662108a504": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a35cbebc740a4d10afb5898280966a1b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7fe10ce044254824be356a23598ea50b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4959b00d6bbb441594383593ad83bf3d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "885b8760c603467b8223a49aedc72a62": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "16e09195cf1c467ba8b91176365dd729": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}