PBT_MARL_water_down.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "PBT_MARL_water_down.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WwD3_kI2HDbA",
        "colab_type": "text"
      },
      "source": [
        "#Setup Colab"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IyAKAl49kg7I",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "39ffeff3-d02e-49a5-9ee4-d309d475ca45"
      },
      "source": [
        "from google.colab import drive \n",
        "\n",
        "drive.mount('/content/gdrive')\n",
        "%cd \"/content/gdrive/My Drive/Colab Notebooks/PBT_MARL_watered_down/\"\n",
        "!pwd\n",
        "!ls -l\n",
        "\n",
        "# Install if you haven't done so.\n",
        "#!pip install tensorflow==2.3.0\n",
        "#!pip install lz4\n",
        "#!pip install 'ray[tune]'\n",
        "#!pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.9.0.dev0-cp36-cp36m-manylinux1_x86_64.whl"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n",
            "/content/gdrive/My Drive/Colab Notebooks/PBT_MARL_watered_down\n",
            "/content/gdrive/My Drive/Colab Notebooks/PBT_MARL_watered_down\n",
            "total 75\n",
            "drwx------ 6 root root  4096 Aug 24 09:01 chkpt\n",
            "-rw------- 1 root root  3139 Jun 11 03:59 Helper.py\n",
            "-rw------- 1 root root     1 Jun 11 03:59 __init__.py\n",
            "-rw------- 1 root root  1072 Jun 10 04:55 LICENSE\n",
            "-rw------- 1 root root  6543 Aug 24 15:06 PBT_MARL.py\n",
            "-rw------- 1 root root  9014 Jun 10 17:23 pbt_marl_water_down_cpu_only.py\n",
            "-rw------- 1 root root 35785 Aug 25 07:57 PBT_MARL_water_down.ipynb\n",
            "drwx------ 2 root root  4096 Aug 24 06:35 __pycache__\n",
            "drwx------ 2 root root  4096 Aug 25 07:58 ray_results\n",
            "-rw------- 1 root root  3710 Aug  4 08:18 README.md\n",
            "-rw------- 1 root root  2056 Aug  4 08:04 RockPaperScissorsEnv.py\n",
            "Requirement already satisfied: lz4 in /usr/local/lib/python3.6/dist-packages (3.1.0)\n",
            "Requirement already satisfied: ray[tune] in /usr/local/lib/python3.6/dist-packages (0.9.0.dev0)\n",
            "Requirement already satisfied: gpustat in /usr/local/lib/python3.6/dist-packages (from ray[tune]) (0.6.0)\n",
            "Requirement already satisfied: numpy>=1.16 in /usr/local/lib/python3.6/dist-packages (from ray[tune]) (1.18.5)\n",
            "Requirement already satisfied: colorama in /usr/local/lib/python3.6/dist-packages (from ray[tune]) (0.4.3)\n",
            "Requirement already satisfied: aioredis in /usr/local/lib/python3.6/dist-packages (from ray[tune]) (1.3.1)\n",
            "Requirement already satisfied: opencensus in /usr/local/lib/python3.6/dist-packages (from ray[tune]) (0.7.10)\n",
            "Requirement already satisfied: colorful in /usr/local/lib/python3.6/dist-packages (from ray[tune]) (0.5.4)\n",
            "Requirement already satisfied: google in /usr/local/lib/python3.6/dist-packages (from ray[tune]) (2.0.3)\n",
            "Requirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.6/dist-packages (from ray[tune]) (3.12.4)\n",
            "Requirement already satisfied: aiohttp-cors in /usr/local/lib/python3.6/dist-packages (from ray[tune]) (0.7.0)\n",
            "Requirement already satisfied: py-spy>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from ray[tune]) (0.3.3)\n",
            "Requirement already satisfied: prometheus-client>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from ray[tune]) (0.8.0)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from ray[tune]) (3.13)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from ray[tune]) (2.23.0)\n",
            "Requirement already satisfied: jsonschema in /usr/local/lib/python3.6/dist-packages (from ray[tune]) (2.6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.6/dist-packages (from ray[tune]) (3.6.2)\n",
            "Requirement already satisfied: grpcio>=1.28.1 in /usr/local/lib/python3.6/dist-packages (from ray[tune]) (1.31.0)\n",
            "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.6/dist-packages (from ray[tune]) (7.1.2)\n",
            "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from ray[tune]) (1.0.0)\n",
            "Requirement already satisfied: redis<3.5.0,>=3.3.2 in /usr/local/lib/python3.6/dist-packages (from ray[tune]) (3.4.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from ray[tune]) (3.0.12)\n",
            "Requirement already satisfied: tabulate; extra == \"tune\" in /usr/local/lib/python3.6/dist-packages (from ray[tune]) (0.8.7)\n",
            "Requirement already satisfied: pandas; extra == \"tune\" in /usr/local/lib/python3.6/dist-packages (from ray[tune]) (1.0.5)\n",
            "Requirement already satisfied: tensorboardX; extra == \"tune\" in /usr/local/lib/python3.6/dist-packages (from ray[tune]) (2.1)\n",
            "Requirement already satisfied: six>=1.7 in /usr/local/lib/python3.6/dist-packages (from gpustat->ray[tune]) (1.15.0)\n",
            "Requirement already satisfied: nvidia-ml-py3>=7.352.0 in /usr/local/lib/python3.6/dist-packages (from gpustat->ray[tune]) (7.352.0)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.6/dist-packages (from gpustat->ray[tune]) (5.4.8)\n",
            "Requirement already satisfied: blessings>=1.6 in /usr/local/lib/python3.6/dist-packages (from gpustat->ray[tune]) (1.7)\n",
            "Requirement already satisfied: hiredis in /usr/local/lib/python3.6/dist-packages (from aioredis->ray[tune]) (1.1.0)\n",
            "Requirement already satisfied: async-timeout in /usr/local/lib/python3.6/dist-packages (from aioredis->ray[tune]) (3.0.1)\n",
            "Requirement already satisfied: google-api-core<2.0.0,>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from opencensus->ray[tune]) (1.16.0)\n",
            "Requirement already satisfied: opencensus-context==0.1.1 in /usr/local/lib/python3.6/dist-packages (from opencensus->ray[tune]) (0.1.1)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.6/dist-packages (from google->ray[tune]) (4.6.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.8.0->ray[tune]) (49.2.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->ray[tune]) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->ray[tune]) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->ray[tune]) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->ray[tune]) (2020.6.20)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray[tune]) (19.3.0)\n",
            "Requirement already satisfied: idna-ssl>=1.0; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray[tune]) (1.1.0)\n",
            "Requirement already satisfied: multidict<5.0,>=4.5 in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray[tune]) (4.7.6)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray[tune]) (1.5.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.5; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray[tune]) (3.7.4.2)\n",
            "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from pandas; extra == \"tune\"->ray[tune]) (2.8.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas; extra == \"tune\"->ray[tune]) (2018.9)\n",
            "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.6.0 in /usr/local/lib/python3.6/dist-packages (from google-api-core<2.0.0,>=1.0.0->opencensus->ray[tune]) (1.52.0)\n",
            "Requirement already satisfied: google-auth<2.0dev,>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from google-api-core<2.0.0,>=1.0.0->opencensus->ray[tune]) (1.17.2)\n",
            "Requirement already satisfied: contextvars; python_version >= \"3.6\" and python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from opencensus-context==0.1.1->opencensus->ray[tune]) (2.4)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.6/dist-packages (from google-auth<2.0dev,>=0.4.0->google-api-core<2.0.0,>=1.0.0->opencensus->ray[tune]) (0.2.8)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from google-auth<2.0dev,>=0.4.0->google-api-core<2.0.0,>=1.0.0->opencensus->ray[tune]) (4.6)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth<2.0dev,>=0.4.0->google-api-core<2.0.0,>=1.0.0->opencensus->ray[tune]) (4.1.1)\n",
            "Requirement already satisfied: immutables>=0.9 in /usr/local/lib/python3.6/dist-packages (from contextvars; python_version >= \"3.6\" and python_version < \"3.7\"->opencensus-context==0.1.1->opencensus->ray[tune]) (0.14)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.6/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2.0dev,>=0.4.0->google-api-core<2.0.0,>=1.0.0->opencensus->ray[tune]) (0.4.8)\n",
            "Collecting ray==0.9.0.dev0\n",
            "  Using cached https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.9.0.dev0-cp36-cp36m-manylinux1_x86_64.whl\n",
            "Requirement already satisfied, skipping upgrade: filelock in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (3.0.12)\n",
            "Requirement already satisfied, skipping upgrade: py-spy>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (0.3.3)\n",
            "Requirement already satisfied, skipping upgrade: requests in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (2.23.0)\n",
            "Requirement already satisfied, skipping upgrade: numpy>=1.16 in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (1.18.5)\n",
            "Requirement already satisfied, skipping upgrade: colorama in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (0.4.3)\n",
            "Requirement already satisfied, skipping upgrade: click>=7.0 in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (7.1.2)\n",
            "Requirement already satisfied, skipping upgrade: aiohttp in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (3.6.2)\n",
            "Requirement already satisfied, skipping upgrade: jsonschema in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (2.6.0)\n",
            "Requirement already satisfied, skipping upgrade: msgpack<2.0.0,>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (1.0.0)\n",
            "Requirement already satisfied, skipping upgrade: gpustat in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (0.6.0)\n",
            "Requirement already satisfied, skipping upgrade: pyyaml in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (3.13)\n",
            "Requirement already satisfied, skipping upgrade: colorful in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (0.5.4)\n",
            "Requirement already satisfied, skipping upgrade: opencensus in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (0.7.10)\n",
            "Requirement already satisfied, skipping upgrade: aioredis in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (1.3.1)\n",
            "Requirement already satisfied, skipping upgrade: google in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (2.0.3)\n",
            "Requirement already satisfied, skipping upgrade: prometheus-client>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (0.8.0)\n",
            "Requirement already satisfied, skipping upgrade: protobuf>=3.8.0 in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (3.12.4)\n",
            "Requirement already satisfied, skipping upgrade: grpcio>=1.28.1 in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (1.31.0)\n",
            "Requirement already satisfied, skipping upgrade: redis<3.5.0,>=3.3.2 in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (3.4.1)\n",
            "Requirement already satisfied, skipping upgrade: aiohttp-cors in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (0.7.0)\n",
            "Requirement already satisfied, skipping upgrade: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->ray==0.9.0.dev0) (3.0.4)\n",
            "Requirement already satisfied, skipping upgrade: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->ray==0.9.0.dev0) (2.10)\n",
            "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->ray==0.9.0.dev0) (2020.6.20)\n",
            "Requirement already satisfied, skipping upgrade: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->ray==0.9.0.dev0) (1.24.3)\n",
            "Requirement already satisfied, skipping upgrade: async-timeout<4.0,>=3.0 in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray==0.9.0.dev0) (3.0.1)\n",
            "Requirement already satisfied, skipping upgrade: idna-ssl>=1.0; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray==0.9.0.dev0) (1.1.0)\n",
            "Requirement already satisfied, skipping upgrade: typing-extensions>=3.6.5; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray==0.9.0.dev0) (3.7.4.2)\n",
            "Requirement already satisfied, skipping upgrade: attrs>=17.3.0 in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray==0.9.0.dev0) (19.3.0)\n",
            "Requirement already satisfied, skipping upgrade: yarl<2.0,>=1.0 in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray==0.9.0.dev0) (1.5.1)\n",
            "Requirement already satisfied, skipping upgrade: multidict<5.0,>=4.5 in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray==0.9.0.dev0) (4.7.6)\n",
            "Requirement already satisfied, skipping upgrade: nvidia-ml-py3>=7.352.0 in /usr/local/lib/python3.6/dist-packages (from gpustat->ray==0.9.0.dev0) (7.352.0)\n",
            "Requirement already satisfied, skipping upgrade: blessings>=1.6 in /usr/local/lib/python3.6/dist-packages (from gpustat->ray==0.9.0.dev0) (1.7)\n",
            "Requirement already satisfied, skipping upgrade: psutil in /usr/local/lib/python3.6/dist-packages (from gpustat->ray==0.9.0.dev0) (5.4.8)\n",
            "Requirement already satisfied, skipping upgrade: six>=1.7 in /usr/local/lib/python3.6/dist-packages (from gpustat->ray==0.9.0.dev0) (1.15.0)\n",
            "Requirement already satisfied, skipping upgrade: opencensus-context==0.1.1 in /usr/local/lib/python3.6/dist-packages (from opencensus->ray==0.9.0.dev0) (0.1.1)\n",
            "Requirement already satisfied, skipping upgrade: google-api-core<2.0.0,>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from opencensus->ray==0.9.0.dev0) (1.16.0)\n",
            "Requirement already satisfied, skipping upgrade: hiredis in /usr/local/lib/python3.6/dist-packages (from aioredis->ray==0.9.0.dev0) (1.1.0)\n",
            "Requirement already satisfied, skipping upgrade: beautifulsoup4 in /usr/local/lib/python3.6/dist-packages (from google->ray==0.9.0.dev0) (4.6.3)\n",
            "Requirement already satisfied, skipping upgrade: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.8.0->ray==0.9.0.dev0) (49.2.0)\n",
            "Requirement already satisfied, skipping upgrade: contextvars; python_version >= \"3.6\" and python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from opencensus-context==0.1.1->opencensus->ray==0.9.0.dev0) (2.4)\n",
            "Requirement already satisfied, skipping upgrade: googleapis-common-protos<2.0dev,>=1.6.0 in /usr/local/lib/python3.6/dist-packages (from google-api-core<2.0.0,>=1.0.0->opencensus->ray==0.9.0.dev0) (1.52.0)\n",
            "Requirement already satisfied, skipping upgrade: pytz in /usr/local/lib/python3.6/dist-packages (from google-api-core<2.0.0,>=1.0.0->opencensus->ray==0.9.0.dev0) (2018.9)\n",
            "Requirement already satisfied, skipping upgrade: google-auth<2.0dev,>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from google-api-core<2.0.0,>=1.0.0->opencensus->ray==0.9.0.dev0) (1.17.2)\n",
            "Requirement already satisfied, skipping upgrade: immutables>=0.9 in /usr/local/lib/python3.6/dist-packages (from contextvars; python_version >= \"3.6\" and python_version < \"3.7\"->opencensus-context==0.1.1->opencensus->ray==0.9.0.dev0) (0.14)\n",
            "Requirement already satisfied, skipping upgrade: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth<2.0dev,>=0.4.0->google-api-core<2.0.0,>=1.0.0->opencensus->ray==0.9.0.dev0) (4.1.1)\n",
            "Requirement already satisfied, skipping upgrade: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.6/dist-packages (from google-auth<2.0dev,>=0.4.0->google-api-core<2.0.0,>=1.0.0->opencensus->ray==0.9.0.dev0) (0.2.8)\n",
            "Requirement already satisfied, skipping upgrade: rsa<5,>=3.1.4; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from google-auth<2.0dev,>=0.4.0->google-api-core<2.0.0,>=1.0.0->opencensus->ray==0.9.0.dev0) (4.6)\n",
            "Requirement already satisfied, skipping upgrade: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.6/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2.0dev,>=0.4.0->google-api-core<2.0.0,>=1.0.0->opencensus->ray==0.9.0.dev0) (0.4.8)\n",
            "Installing collected packages: ray\n",
            "  Found existing installation: ray 0.9.0.dev0\n",
            "    Uninstalling ray-0.9.0.dev0:\n",
            "      Successfully uninstalled ray-0.9.0.dev0\n",
            "Successfully installed ray-0.9.0.dev0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2BMjPt0fbNSf",
        "colab_type": "text"
      },
      "source": [
        "#Chkpt/restore & log path"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zQMyQcPpbIai",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "g_drive_path = \"/content/gdrive/My Drive/Colab Notebooks/PBT_MARL_watered_down/\"\n",
        "\n",
        "local_dir = g_drive_path + \"chkpt/\"\n",
        "chkpt_freq = 10\n",
        "chkpt = 150\n",
        "restore_path = \"{}checkpoint_{}/checkpoint-{}\".format(local_dir, chkpt, chkpt)\n",
        "is_restore = False\n",
        "\n",
        "log_dir = g_drive_path + \"ray_results/\""
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I-GBqoxsHBZV",
        "colab_type": "text"
      },
      "source": [
        "#Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p8DRdL7tgKBr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        },
        "outputId": "4a358b5a-34d5-48b7-8c14-8c22687534b2"
      },
      "source": [
        "from collections import defaultdict\n",
        "from typing import Dict\n",
        "import random\n",
        "import numpy as np\n",
        "\n",
        "from gym.spaces import Discrete\n",
        "\n",
        "import ray\n",
        "from ray import tune\n",
        "from ray.tune.logger import pretty_print\n",
        "\n",
        "from ray.tune.registry import register_env\n",
        "from ray.rllib.models import ModelCatalog\n",
        "\n",
        "from ray.rllib.policy import Policy\n",
        "from ray.rllib.policy.torch_policy import LearningRateSchedule, EntropyCoeffSchedule\n",
        "\n",
        "from ray.rllib.agents.ppo.ppo_torch_policy import PPOTorchPolicy, KLCoeffMixin, ValueNetworkMixin\n",
        "from ray.rllib.agents.ppo import ddppo\n",
        "from ray.rllib.agents.ppo.ddppo import DDPPOTrainer\n",
        "\n",
        "from ray.rllib.env import BaseEnv\n",
        "from ray.rllib.env.multi_agent_env import MultiAgentEnv\n",
        "\n",
        "from ray.rllib.policy.sample_batch import SampleBatch\n",
        "from ray.rllib.evaluation import MultiAgentEpisode, RolloutWorker\n",
        "from ray.rllib.agents.callbacks import DefaultCallbacks\n",
        "\n",
        "from ray.rllib.utils.schedules import ConstantSchedule\n",
        "from ray.rllib.utils import try_import_tf\n",
        "tf = try_import_tf()\n",
        "\n",
        "from RockPaperScissorsEnv import RockPaperScissorsEnv\n",
        "from Helper import Helper\n",
        "from PBT_MARL import PBT_MARL"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/compat/v2_compat.py:96: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "non-resource variables are not supported in the long term\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bFRqbhqAunwM",
        "colab_type": "text"
      },
      "source": [
        "#Callbacks"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CBp3zwiEuqM7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\"\"\"#Callbacks\"\"\"\n",
        "\n",
        "class MyCallbacks(DefaultCallbacks):\n",
        "    def on_episode_start(self, worker: RolloutWorker, base_env: BaseEnv,\n",
        "                         policies: Dict[str, Policy],\n",
        "                         episode: MultiAgentEpisode, **kwargs):        \n",
        "        for k, v in policies.items():\n",
        "            key = k + \"_gamma\"\n",
        "            episode.user_data[key] = []\n",
        "            episode.hist_data[key] = []\n",
        "\n",
        "    def on_episode_step(self, worker: RolloutWorker, base_env: BaseEnv,\n",
        "                        episode: MultiAgentEpisode, **kwargs):\n",
        "        for k, v in policies.items():\n",
        "            key = k + \"_gamma\"\n",
        "            episode.user_data[key].append(worker.get_policy(k).config[\"gamma\"])\n",
        "\n",
        "    def on_episode_end(self, worker: RolloutWorker, base_env: BaseEnv,\n",
        "                       policies: Dict[str, Policy], episode: MultiAgentEpisode,\n",
        "                       **kwargs):\n",
        "        for k, v in policies.items():\n",
        "            key = k + \"_gamma\"        \n",
        "            episode.custom_metrics[key] = np.mean(episode.user_data[key])\n",
        "            episode.hist_data[key] = episode.user_data[key]\n",
        "\n",
        "\n",
        "        player_policy = []\n",
        "        score = []\n",
        "        for k,v in episode.agent_rewards.items():\n",
        "            player_policy.append(k)\n",
        "            score.append(v)\n",
        "\n",
        "        pol_i_key = player_policy[0][1]\n",
        "        pol_j_key = player_policy[1][1]\n",
        "        _, str_i = pol_i_key.split(\"_\")\n",
        "        _, str_j = pol_j_key.split(\"_\")\n",
        "        agt_i_key = \"agt_\" + str_i\n",
        "        agt_j_key = \"agt_\" + str_j\n",
        "\n",
        "        g_helper = ray.get_actor(\"g_helper\")     \n",
        "        prev_rating_i = ray.get(g_helper.get_rating.remote(agt_i_key))\n",
        "        prev_rating_j = ray.get(g_helper.get_rating.remote(agt_j_key))\n",
        "        score_i = score[0]\n",
        "        score_j = score[1]\n",
        "        rating_i, rating_j = l_PBT_MARL.compute_rating(prev_rating_i, prev_rating_j, score_i, score_j)\n",
        "        ray.get(g_helper.update_rating.remote(agt_i_key, agt_j_key, rating_i, rating_j, score_i, score_j))\n",
        "        #print(\"on_episode_end ray.get(g_helper.get_agt_store.remote())\", ray.get(g_helper.get_agt_store.remote()))\n",
        "\n",
        "    def on_sample_end(self, worker: RolloutWorker, samples: SampleBatch,\n",
        "                      **kwargs):\n",
        "        #print(\"on_sample_end returned sample batch of size {}\".format(samples.count))\n",
        "        pass\n",
        "\n",
        "    def on_train_result(self, trainer, result: dict, **kwargs):\n",
        "        print(\"trainer.train() result: {} -> {} episodes\".format(trainer, result[\"episodes_this_iter\"]))\n",
        "        # you can mutate the result dict to add new fields to return\n",
        "        result[\"callback_ok\"] = True\n",
        "        #print(\"on_train_result result\", result)\n",
        "\n",
        "        l_PBT_MARL.PBT(trainer)     # perform PBT\n",
        "\n",
        "        g_helper = ray.get_actor(\"g_helper\")     \n",
        "        ray.get(g_helper.set_pair.remote())     # set the lastest pair\n",
        "        #print(\"on_train_result g_helper.get_pair.remote()\", ray.get(g_helper.get_pair.remote())\n",
        "\n",
        "    def on_postprocess_trajectory(\n",
        "            self, worker: RolloutWorker, episode: MultiAgentEpisode,\n",
        "            agent_id: str, policy_id: str, policies: Dict[str, Policy],\n",
        "            postprocessed_batch: SampleBatch,\n",
        "            original_batches: Dict[str, SampleBatch], **kwargs):\n",
        "        #\u0010print(\"postprocessed {}, {}, {}, {} steps\".format(episode, agent_id, policy_id, postprocessed_batch.count))              \n",
        "        \"\"\"\n",
        "        if \"num_batches\" not in episode.custom_metrics:\n",
        "            episode.custom_metrics[\"num_batches\"] = 0\n",
        "        episode.custom_metrics[\"num_batches\"] += 1\n",
        "        \"\"\"\n",
        "        pass"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i492GjpyrGNy",
        "colab_type": "text"
      },
      "source": [
        "#Mixin for extending policy & trainer."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n6Ww51NXrBEX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class My_Mixin:\n",
        "    def __init__(self, config):\n",
        "        self.config = config\n",
        "                \n",
        "    def update_lr_schedule(self, lr):        \n",
        "        self.lr_schedule = ConstantSchedule(lr, framework=None)  \n",
        "        print(\"update_lr_schedule, lr={}\".format(lr))\n",
        "\n",
        "    def update_gamma(self, gamma):        \n",
        "        self.config[\"gamma\"] = gamma     \n",
        "        print(\"update_gamma, gamma={}\".format(gamma))\n",
        "\n",
        "def setup_mixins(policy, obs_space, action_space, config):\n",
        "    ValueNetworkMixin.__init__(policy, obs_space, action_space, config)\n",
        "    KLCoeffMixin.__init__(policy, config)\n",
        "    EntropyCoeffSchedule.__init__(policy, config[\"entropy_coeff\"],\n",
        "                                  config[\"entropy_coeff_schedule\"])\n",
        "    LearningRateSchedule.__init__(policy, config[\"lr\"], config[\"lr_schedule\"])  \n",
        "\n",
        "CustomPolicy = PPOTorchPolicy.with_updates(\n",
        "    name=\"Custom_Policy\",\n",
        "    before_init=setup_mixins,\n",
        "    mixins=[\n",
        "        LearningRateSchedule, EntropyCoeffSchedule, KLCoeffMixin,\n",
        "        ValueNetworkMixin, \n",
        "        My_Mixin\n",
        "    ])\n",
        "\n",
        "def get_policy_class(config):\n",
        "    return CustomPolicy\n",
        "\n",
        "CustomTrainer = DDPPOTrainer.with_updates(name=\"Custom_Trainer\",\n",
        "                                          default_policy=CustomPolicy,\n",
        "                                          get_policy_class=get_policy_class,\n",
        "                                          )"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DpcJGyAaBbc2",
        "colab_type": "text"
      },
      "source": [
        "#Policy"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iMZ20pVCzxUN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def init_policies(population_size, obs_space, act_space, use_lstm, hyperparameters_range):\n",
        "    \"\"\"\n",
        "    Sample hyper-parameter from the hyper-parameter distribution.\n",
        "    \"\"\"\n",
        "    policies = {}\n",
        "    for i in range(population_size):\n",
        "        pol_key = \"p_\" + str(i)\n",
        "        lr = np.random.uniform(low=hyperparameters_range[\"lr\"][0], high=hyperparameters_range[\"lr\"][1], size=None)\n",
        "        gamma = np.random.uniform(low=hyperparameters_range[\"gamma\"][0], high=hyperparameters_range[\"gamma\"][1], size=None)\n",
        "        policies[pol_key] = (None, obs_space, act_space, {\"model\": {\"use_lstm\": use_lstm},\n",
        "                                                          \"lr\": lr,\n",
        "                                                          \"gamma\": gamma})\n",
        "    return policies\n",
        "\n",
        "def train_policies(population_size):    \n",
        "    train_policies = []\n",
        "    for i in range(population_size):\n",
        "        pol_key = \"p_\" + str(i)\n",
        "        train_policies.append(pol_key)\n",
        "\n",
        "    return policies\n",
        "\n",
        "def select_policy(agent_id):\n",
        "    _, i = agent_id.split(\"_\")\n",
        "    policy = \"p_\" + str(i)\n",
        "    #print(\"select_policy {} {}\".format(agent_id , policy))\n",
        "    return policy     "
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oAEySBSfBS5u",
        "colab_type": "text"
      },
      "source": [
        "#Variables"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "trdlnMoHwbfT",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "outputId": "1012bc2a-cd4b-45df-f41d-06f910f5ae9b"
      },
      "source": [
        "population_size = 6\n",
        "K = 0.1     \n",
        "T_select = 0.77 #0.47\n",
        "binomial_n = 1\n",
        "inherit_prob = 0.5\n",
        "perturb_prob = 0.1\n",
        "perturb_val = [0.8, 1.2]\n",
        "hyperparameters_range = {\"lr\": [1e-7, 1e-1], \n",
        "                         \"gamma\": [0.9, 0.999]}\n",
        "\n",
        "register_env(\"RockPaperScissorsEnv\", lambda _: RockPaperScissorsEnv(_, population_size))     # register RockPaperScissorsEnv with RLlib     \n",
        "# get obs & act spaces from dummy CDA env\n",
        "dummy_env = RockPaperScissorsEnv(_, population_size=0)\n",
        "obs_space = dummy_env.observation_space\n",
        "act_space = dummy_env.action_space\n",
        "\n",
        "use_lstm=False\n",
        "policies = init_policies(population_size, obs_space, act_space, use_lstm, hyperparameters_range)\n",
        "train_policies = train_policies(population_size)\n",
        "\n",
        "l_PBT_MARL = PBT_MARL(population_size, \n",
        "                      K, T_select, \n",
        "                      binomial_n, inherit_prob,\n",
        "                      perturb_prob, perturb_val)\n",
        "\n",
        "ray.init(ignore_reinit_error=True, log_to_driver=True, num_cpus=2, num_gpus=1)     \n",
        "#print(\"ray.nodes()\", ray.nodes())\n",
        "\n",
        "g_helper = Helper.options(name=\"g_helper\").remote(population_size, policies)      # this object runs on a different ray actor process\n",
        "ray.get(g_helper.set_pair.remote())\n",
        "\n",
        "num_iters = 30     # num of main training loop"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2020-08-25 07:58:28,101\tINFO resource_spec.py:250 -- Starting Ray with 7.13 GiB memory available for workers and up to 3.59 GiB for objects. You can adjust these settings with ray.init(memory=<bytes>, object_store_memory=<bytes>).\n",
            "2020-08-25 07:58:28,691\tINFO services.py:1213 -- View the Ray dashboard at \u001b[1m\u001b[32mlocalhost:8265\u001b[39m\u001b[22m\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xyucHLoqBe5G",
        "colab_type": "text"
      },
      "source": [
        "#Config"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KNWNnavQt9y0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_config():\n",
        "    config = ddppo.DEFAULT_CONFIG.copy()\n",
        "\n",
        "    config[\"env\"] = RockPaperScissorsEnv\n",
        "    config[\"multiagent\"] = {\"policies_to_train\": train_policies,\n",
        "                            \"policies\": policies,\n",
        "                            \"policy_mapping_fn\": select_policy}        \n",
        "    config[\"num_cpus_per_worker\"] = 0.25                                \n",
        "    config[\"num_gpus_per_worker\"] = 0.125\n",
        "    config[\"num_workers\"] = 2      \n",
        "    config[\"num_envs_per_worker\"] = 3\n",
        "    config[\"rollout_fragment_length\"] = 30                  \n",
        "    #config[\"train_batch_size\"] = -1     # must be -1 for DDPPO trainer \n",
        "    config[\"sgd_minibatch_size\"] = 10                       \n",
        "    config[\"num_sgd_iter\"] = 3      # number of epochs to execute per train batch.\n",
        "    config[\"callbacks\"] = MyCallbacks\n",
        "    config[\"log_level\"] = \"WARN\"      # WARN/INFO/DEBUG \n",
        "    config[\"output\"] = log_dir\n",
        "\n",
        "    return config"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vb_4cEGdBlqf",
        "colab_type": "text"
      },
      "source": [
        "#Go train"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kUB40TYSuDAn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "7e70e8e1-3ac8-4f3a-8322-c62d2ea982a5"
      },
      "source": [
        "def go_train(config):     \n",
        "    trainer = CustomTrainer(config=get_config(), env=\"RockPaperScissorsEnv\")         \n",
        "\n",
        "    if is_restore == True:\n",
        "        trainer.restore(restore_path) \n",
        "    \n",
        "    result = None\n",
        "    for i in range(num_iters):\n",
        "        result = trainer.train()       \n",
        "        print(\"training loop = {} of {}\".format(i + 1, num_iters))            \n",
        "        print(pretty_print(result))     \n",
        "\n",
        "        if i % chkpt_freq == 0:\n",
        "            checkpoint = trainer.save(local_dir)\n",
        "            print(\"checkpoint saved at\", checkpoint)\n",
        "    \n",
        "    checkpoint = trainer.save(local_dir)\n",
        "    print(\"checkpoint saved at\", checkpoint)\n",
        "    \n",
        "\n",
        "# run everything\n",
        "go_train(get_config())    \n",
        "\n",
        "ray.shutdown()"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2020-08-25 07:58:32,773\tINFO trainer.py:637 -- Current log_level is WARN. For more information, set 'log_level': 'INFO' / 'DEBUG' or use the -v and -vv flags.\n",
            "2020-08-25 07:58:33,197\tWARNING worker.py:413 -- ray.get_gpu_ids() will return a list of strings by default in a future version of Ray for compatibility with CUDA. To enable the forward-compatible behavior, use `ray.get_gpu_ids(as_str=True)`.\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/compat/v2_compat.py:96: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m non-resource variables are not supported in the long term\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/compat/v2_compat.py:96: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m non-resource variables are not supported in the long term\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m 2020-08-25 07:58:38,922\tWARNING worker.py:413 -- ray.get_gpu_ids() will return a list of strings by default in a future version of Ray for compatibility with CUDA. To enable the forward-compatible behavior, use `ray.get_gpu_ids(as_str=True)`.\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m /usr/local/lib/python3.6/dist-packages/ray/rllib/utils/torch_ops.py:149: UserWarning: The given NumPy array is not writeable, and PyTorch does not support non-writeable tensors. This means you can write to the underlying (supposedly non-writeable) NumPy array using the tensor. You may want to copy the array to protect its data or make it writeable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at  /pytorch/torch/csrc/utils/tensor_numpy.cpp:141.)\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m   tensor = torch.from_numpy(np.asarray(item))\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m /usr/local/lib/python3.6/dist-packages/ray/rllib/utils/torch_ops.py:149: UserWarning: The given NumPy array is not writeable, and PyTorch does not support non-writeable tensors. This means you can write to the underlying (supposedly non-writeable) NumPy array using the tensor. You may want to copy the array to protect its data or make it writeable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at  /pytorch/torch/csrc/utils/tensor_numpy.cpp:141.)\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m   tensor = torch.from_numpy(np.asarray(item))\n",
            "2020-08-25 07:58:43,871\tINFO trainable.py:256 -- Trainable.setup took 11.103 seconds. If your trainable is slow to initialize, consider setting reuse_actors=True to reduce actor creation overheads.\n",
            "2020-08-25 07:58:43,872\tWARNING util.py:38 -- Install gputil for GPU system monitoring.\n",
            "/usr/local/lib/python3.6/dist-packages/ray/rllib/utils/torch_ops.py:149: UserWarning: The given NumPy array is not writeable, and PyTorch does not support non-writeable tensors. This means you can write to the underlying (supposedly non-writeable) NumPy array using the tensor. You may want to copy the array to protect its data or make it writeable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at  /pytorch/torch/csrc/utils/tensor_numpy.cpp:141.)\n",
            "  tensor = torch.from_numpy(np.asarray(item))\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.05250648179450637\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9128771915578374\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.05250648179450637\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9128771915578374\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.02562047833341632\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.02562047833341632\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.020496382666733058\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.020496382666733058\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.020496382666733058\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.020496382666733058\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.05676311129414549\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.923716349014329\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.05676311129414549\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.923716349014329\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.03843071750012448\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.03843071750012448\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "training loop = 1 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9128771915578375\n",
            "  p_0_gamma_mean: 0.9128771915578375\n",
            "  p_0_gamma_min: 0.9128771915578374\n",
            "  p_1_gamma_max: 0.9111350797026094\n",
            "  p_1_gamma_mean: 0.9111350797026093\n",
            "  p_1_gamma_min: 0.9111350797026094\n",
            "  p_2_gamma_max: 0.9029134342610622\n",
            "  p_2_gamma_mean: 0.9029134342610621\n",
            "  p_2_gamma_min: 0.9029134342610622\n",
            "  p_3_gamma_max: 0.936873684187527\n",
            "  p_3_gamma_mean: 0.9368736841875269\n",
            "  p_3_gamma_min: 0.9368736841875269\n",
            "  p_4_gamma_max: 0.9237163490143291\n",
            "  p_4_gamma_mean: 0.923716349014329\n",
            "  p_4_gamma_min: 0.923716349014329\n",
            "  p_5_gamma_max: 0.9549629876590181\n",
            "  p_5_gamma_mean: 0.9549629876590182\n",
            "  p_5_gamma_min: 0.9549629876590181\n",
            "date: 2020-08-25_07-58-45\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 18\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_2:\n",
            "      allreduce_latency: 0.006484640969170464\n",
            "      cur_kl_coeff: 0.2\n",
            "      cur_lr: 0.033963435480469394\n",
            "      entropy: 0.6902632084157732\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.784281720717748\n",
            "      model: {}\n",
            "      policy_loss: 0.16711185810466608\n",
            "      total_loss: 4.636524544821845\n",
            "      vf_explained_var: -0.03355904296040535\n",
            "      vf_loss: 4.312556121084425\n",
            "    p_4:\n",
            "      allreduce_latency: 0.00793562995062934\n",
            "      cur_kl_coeff: 0.2\n",
            "      cur_lr: 0.07095388911768186\n",
            "      entropy: 0.24866609275341034\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 5.137387898233202\n",
            "      model: {}\n",
            "      policy_loss: 0.3900202421678437\n",
            "      total_loss: 8.966990881496006\n",
            "      vf_explained_var: -0.16216322779655457\n",
            "      vf_loss: 7.549493153889974\n",
            "  num_steps_sampled: 180\n",
            "  num_steps_trained: 180\n",
            "iterations_since_restore: 1\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 88.53333333333335\n",
            "  ram_util_percent: 31.46666666666667\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_2: 8.0\n",
            "  p_4: 7.0\n",
            "policy_reward_mean:\n",
            "  p_2: -0.2777777777777778\n",
            "  p_4: 0.2777777777777778\n",
            "policy_reward_min:\n",
            "  p_2: -7.0\n",
            "  p_4: -8.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.36731458479358303\n",
            "  mean_env_wait_ms: 0.49951384144444627\n",
            "  mean_inference_ms: 6.050775128026163\n",
            "  mean_raw_obs_processing_ms: 3.9720535278320312\n",
            "time_since_restore: 1.7439560890197754\n",
            "time_this_iter_s: 1.7439560890197754\n",
            "time_total_s: 1.7439560890197754\n",
            "timers:\n",
            "  learn_time_ms: 1717.819\n",
            "timestamp: 1598342325\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 180\n",
            "training_iteration: 1\n",
            "\n",
            "checkpoint saved at /content/gdrive/My Drive/Colab Notebooks/PBT_MARL_watered_down/chkpt/checkpoint_1/checkpoint-1\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.02459565920007967\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.02459565920007967\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.01967652736006374\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.01967652736006374\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.02459565920007967\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.02459565920007967\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.029514791040095602\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.029514791040095602\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.029514791040095602\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.029514791040095602\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.046116861000149376\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.046116861000149376\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "training loop = 2 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9128771915578375\n",
            "  p_0_gamma_mean: 0.9128771915578374\n",
            "  p_0_gamma_min: 0.9128771915578374\n",
            "  p_1_gamma_max: 0.9549629876590181\n",
            "  p_1_gamma_mean: 0.9330490336808138\n",
            "  p_1_gamma_min: 0.9111350797026094\n",
            "  p_2_gamma_max: 0.9549629876590181\n",
            "  p_2_gamma_mean: 0.92893821096004\n",
            "  p_2_gamma_min: 0.9029134342610622\n",
            "  p_3_gamma_max: 0.9549629876590181\n",
            "  p_3_gamma_mean: 0.9459183359232726\n",
            "  p_3_gamma_min: 0.9368736841875269\n",
            "  p_4_gamma_max: 0.9237163490143291\n",
            "  p_4_gamma_mean: 0.9237163490143288\n",
            "  p_4_gamma_min: 0.923716349014329\n",
            "  p_5_gamma_max: 0.9549629876590181\n",
            "  p_5_gamma_mean: 0.954962987659018\n",
            "  p_5_gamma_min: 0.9549629876590181\n",
            "date: 2020-08-25_07-58-48\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 36\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_1:\n",
            "      allreduce_latency: 0.008314013481140137\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.03746819904075166\n",
            "      entropy: 0.5271001805861791\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 1.9522093335787456\n",
            "      model: {}\n",
            "      policy_loss: 0.2206338040608292\n",
            "      total_loss: 7.671166102091472\n",
            "      vf_explained_var: -0.5025754570960999\n",
            "      vf_loss: 7.060090065002441\n",
            "    p_2:\n",
            "      allreduce_latency: 0.0056773026784261065\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.033963435480469394\n",
            "      entropy: 0.6174374222755432\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.20279239738980928\n",
            "      model: {}\n",
            "      policy_loss: 0.14127543568611145\n",
            "      total_loss: 4.84935720761617\n",
            "      vf_explained_var: -0.001712540746666491\n",
            "      vf_loss: 4.667523543039958\n",
            "    p_4:\n",
            "      allreduce_latency: 0.006322542826334636\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.07095388911768186\n",
            "      entropy: 0.06603473704308271\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.24840701619784036\n",
            "      model: {}\n",
            "      policy_loss: 0.011005356907844543\n",
            "      total_loss: 3.6179326375325522\n",
            "      vf_explained_var: -0.019017120823264122\n",
            "      vf_loss: 3.557245969772339\n",
            "    p_5:\n",
            "      allreduce_latency: 0.009319782257080078\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.0320255979167704\n",
            "      entropy: 0.47175006568431854\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 1.470048427581787\n",
            "      model: {}\n",
            "      policy_loss: 0.30867989423374337\n",
            "      total_loss: 4.131033420562744\n",
            "      vf_explained_var: -0.07022739201784134\n",
            "      vf_loss: 3.5283437172571817\n",
            "  num_steps_sampled: 360\n",
            "  num_steps_trained: 360\n",
            "iterations_since_restore: 2\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 82.725\n",
            "  ram_util_percent: 31.575000000000003\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_1: 3.0\n",
            "  p_2: 8.0\n",
            "  p_4: 7.0\n",
            "  p_5: 5.0\n",
            "policy_reward_mean:\n",
            "  p_1: -1.1666666666666667\n",
            "  p_2: -0.3333333333333333\n",
            "  p_4: 0.3333333333333333\n",
            "  p_5: 1.1666666666666667\n",
            "policy_reward_min:\n",
            "  p_1: -5.0\n",
            "  p_2: -7.0\n",
            "  p_4: -8.0\n",
            "  p_5: -3.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.33900556206261906\n",
            "  mean_env_wait_ms: 0.3936650070924724\n",
            "  mean_inference_ms: 6.109650427547478\n",
            "  mean_raw_obs_processing_ms: 4.0350439118557295\n",
            "time_since_restore: 3.3316166400909424\n",
            "time_this_iter_s: 1.587660551071167\n",
            "time_total_s: 3.3316166400909424\n",
            "timers:\n",
            "  learn_time_ms: 1644.031\n",
            "timestamp: 1598342328\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 360\n",
            "training_iteration: 2\n",
            "\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.029514791040095602\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.029514791040095602\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.01967652736006374\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.01967652736006374\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.01967652736006374\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.01967652736006374\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.023611832832076483\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.023611832832076483\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.03541774924811472\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.03541774924811472\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.01981372889281496\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9109535103642415\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.01981372889281496\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9109535103642415\n",
            "training loop = 3 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9549629876590181\n",
            "  p_0_gamma_mean: 0.9269057902582312\n",
            "  p_0_gamma_min: 0.9128771915578374\n",
            "  p_1_gamma_max: 0.9549629876590181\n",
            "  p_1_gamma_mean: 0.9403536850068817\n",
            "  p_1_gamma_min: 0.9111350797026094\n",
            "  p_2_gamma_max: 0.9549629876590181\n",
            "  p_2_gamma_mean: 0.9376131365263658\n",
            "  p_2_gamma_min: 0.9029134342610622\n",
            "  p_3_gamma_max: 0.9549629876590181\n",
            "  p_3_gamma_mean: 0.9489332198351874\n",
            "  p_3_gamma_min: 0.9368736841875269\n",
            "  p_4_gamma_max: 0.9549629876590181\n",
            "  p_4_gamma_mean: 0.9341318952292252\n",
            "  p_4_gamma_min: 0.923716349014329\n",
            "  p_5_gamma_max: 0.9549629876590181\n",
            "  p_5_gamma_mean: 0.9549629876590179\n",
            "  p_5_gamma_min: 0.9549629876590181\n",
            "date: 2020-08-25_07-58-51\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 54\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_0:\n",
            "      allreduce_latency: 0.007349252700805664\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.052506481794506364\n",
            "      entropy: 0.1622253037057817\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 4.612603823343913\n",
            "      model: {}\n",
            "      policy_loss: 0.37466136490305263\n",
            "      total_loss: 19.16854101419449\n",
            "      vf_explained_var: -0.3677062690258026\n",
            "      vf_loss: 17.871358513832092\n",
            "    p_1:\n",
            "      allreduce_latency: 0.006592565112643772\n",
            "      cur_kl_coeff: 0.2\n",
            "      cur_lr: 0.02562047833341632\n",
            "      entropy: 0.4038828859726588\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.31496424310737187\n",
            "      model: {}\n",
            "      policy_loss: -0.10344352738724814\n",
            "      total_loss: 1.3700248334142897\n",
            "      vf_explained_var: -0.1490042358636856\n",
            "      vf_loss: 1.4104755057228937\n",
            "    p_5:\n",
            "      allreduce_latency: 0.008336782455444336\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.03843071750012448\n",
            "      entropy: 0.005453842226415873\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 3.2803122997283936\n",
            "      model: {}\n",
            "      policy_loss: 0.08285722012321155\n",
            "      total_loss: 1.1297557950019836\n",
            "      vf_explained_var: -0.008090376853942871\n",
            "      vf_loss: 0.39083608984947205\n",
            "  num_steps_sampled: 540\n",
            "  num_steps_trained: 540\n",
            "iterations_since_restore: 3\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 83.67500000000001\n",
            "  ram_util_percent: 31.6\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_0: 3.0\n",
            "  p_1: 3.0\n",
            "  p_2: 8.0\n",
            "  p_4: 7.0\n",
            "  p_5: 5.0\n",
            "policy_reward_mean:\n",
            "  p_0: -0.25\n",
            "  p_1: -0.4666666666666667\n",
            "  p_2: -0.3333333333333333\n",
            "  p_4: 0.3333333333333333\n",
            "  p_5: 0.9444444444444444\n",
            "policy_reward_min:\n",
            "  p_0: -3.0\n",
            "  p_1: -5.0\n",
            "  p_2: -7.0\n",
            "  p_4: -8.0\n",
            "  p_5: -3.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.32507754426005236\n",
            "  mean_env_wait_ms: 0.33412742084745384\n",
            "  mean_inference_ms: 6.101897857380668\n",
            "  mean_raw_obs_processing_ms: 4.111514029642889\n",
            "time_since_restore: 4.930767774581909\n",
            "time_this_iter_s: 1.5991511344909668\n",
            "time_total_s: 4.930767774581909\n",
            "timers:\n",
            "  learn_time_ms: 1623.113\n",
            "timestamp: 1598342331\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 540\n",
            "training_iteration: 3\n",
            "\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.03541774924811472\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.03541774924811472\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.02833419939849178\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.02833419939849178\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.04424926408523843\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9774519661959785\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.04424926408523843\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9774519661959785\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.018889466265661188\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.018889466265661188\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.019101303516522082\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9214855724021866\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.019101303516522082\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9214855724021866\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.022667359518793426\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.022667359518793426\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "training loop = 4 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9549629876590181\n",
            "  p_0_gamma_mean: 0.9339200896084277\n",
            "  p_0_gamma_min: 0.9128771915578374\n",
            "  p_1_gamma_max: 0.9549629876590181\n",
            "  p_1_gamma_mean: 0.9440060106699159\n",
            "  p_1_gamma_min: 0.9111350797026094\n",
            "  p_2_gamma_max: 0.9549629876590181\n",
            "  p_2_gamma_mean: 0.941950599309529\n",
            "  p_2_gamma_min: 0.9029134342610622\n",
            "  p_3_gamma_max: 0.9549629876590181\n",
            "  p_3_gamma_mean: 0.9504406617911452\n",
            "  p_3_gamma_min: 0.9368736841875269\n",
            "  p_4_gamma_max: 0.9549629876590181\n",
            "  p_4_gamma_mean: 0.9393396683366737\n",
            "  p_4_gamma_min: 0.923716349014329\n",
            "  p_5_gamma_max: 0.9549629876590181\n",
            "  p_5_gamma_mean: 0.9439606183353241\n",
            "  p_5_gamma_min: 0.9109535103642414\n",
            "date: 2020-08-25_07-58-54\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 72\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_0:\n",
            "      allreduce_latency: 0.006415764490763347\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.02459565920007967\n",
            "      entropy: 2.9582499008288615e-10\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.013343256810912862\n",
            "      model: {}\n",
            "      policy_loss: 0.001982255761201183\n",
            "      total_loss: 4.400432765483856\n",
            "      vf_explained_var: -1.0\n",
            "      vf_loss: 4.395782073338826\n",
            "    p_1:\n",
            "      allreduce_latency: 0.005997657775878906\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.01967652736006374\n",
            "      entropy: 0.006637462181970477\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.27679437820916064\n",
            "      model: {}\n",
            "      policy_loss: -0.06757795065641403\n",
            "      total_loss: 0.20360582073529562\n",
            "      vf_explained_var: 'null'\n",
            "      vf_loss: 0.21582490149497366\n",
            "    p_2:\n",
            "      allreduce_latency: 0.006917357444763184\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.02459565920007967\n",
            "      entropy: 0.2351283592482408\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.43937327961126965\n",
            "      model: {}\n",
            "      policy_loss: 0.15295764214048782\n",
            "      total_loss: 2.85217422246933\n",
            "      vf_explained_var: -0.3800372779369354\n",
            "      vf_loss: 2.611341973145803\n",
            "    p_3:\n",
            "      allreduce_latency: 0.0066585540771484375\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.029514791040095602\n",
            "      entropy: 0.07974256125224362\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 12.357601563135782\n",
            "      model: {}\n",
            "      policy_loss: 0.137313362210989\n",
            "      total_loss: 13.369280179341635\n",
            "      vf_explained_var: -0.25516167283058167\n",
            "      vf_loss: 10.760446747144064\n",
            "  num_steps_sampled: 720\n",
            "  num_steps_trained: 720\n",
            "iterations_since_restore: 4\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 84.1\n",
            "  ram_util_percent: 31.6\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_0: 3.0\n",
            "  p_1: 3.0\n",
            "  p_2: 8.0\n",
            "  p_3: 6.0\n",
            "  p_4: 7.0\n",
            "  p_5: 5.0\n",
            "policy_reward_mean:\n",
            "  p_0: -0.5\n",
            "  p_1: -0.2222222222222222\n",
            "  p_2: -0.8611111111111112\n",
            "  p_3: 1.9166666666666667\n",
            "  p_4: 0.3333333333333333\n",
            "  p_5: 0.9444444444444444\n",
            "policy_reward_min:\n",
            "  p_0: -3.0\n",
            "  p_1: -5.0\n",
            "  p_2: -7.0\n",
            "  p_3: -4.0\n",
            "  p_4: -8.0\n",
            "  p_5: -3.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.3145174931123442\n",
            "  mean_env_wait_ms: 0.29513501329390845\n",
            "  mean_inference_ms: 6.051229246619097\n",
            "  mean_raw_obs_processing_ms: 4.182956743749303\n",
            "time_since_restore: 6.459977388381958\n",
            "time_this_iter_s: 1.5292096138000488\n",
            "time_total_s: 6.459977388381958\n",
            "timers:\n",
            "  learn_time_ms: 1594.458\n",
            "timestamp: 1598342334\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 720\n",
            "training_iteration: 4\n",
            "\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.02833419939849178\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.02833419939849178\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.034001039278190134\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.053099116902286116\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9774519661959785\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.034001039278190134\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.053099116902286116\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9774519661959785\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.022667359518793426\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.022667359518793426\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.0229215642198265\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9214855724021866\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.0229215642198265\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9214855724021866\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.018133887615034743\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.018133887615034743\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "training loop = 5 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9549629876590181\n",
            "  p_0_gamma_mean: 0.9381286692185457\n",
            "  p_0_gamma_min: 0.9128771915578374\n",
            "  p_1_gamma_max: 0.9549629876590181\n",
            "  p_1_gamma_mean: 0.9461974060677362\n",
            "  p_1_gamma_min: 0.9111350797026094\n",
            "  p_2_gamma_max: 0.9774519661959784\n",
            "  p_2_gamma_mean: 0.9490508726868186\n",
            "  p_2_gamma_min: 0.9029134342610622\n",
            "  p_3_gamma_max: 0.9549629876590181\n",
            "  p_3_gamma_mean: 0.9513451269647196\n",
            "  p_3_gamma_min: 0.9368736841875269\n",
            "  p_4_gamma_max: 0.9549629876590181\n",
            "  p_4_gamma_mean: 0.935768849149776\n",
            "  p_4_gamma_min: 0.9214855724021866\n",
            "  p_5_gamma_max: 0.9549629876590181\n",
            "  p_5_gamma_mean: 0.9461610922000625\n",
            "  p_5_gamma_min: 0.9109535103642414\n",
            "date: 2020-08-25_07-58-56\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 90\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_2:\n",
            "      allreduce_latency: 0.006166696548461914\n",
            "      cur_kl_coeff: 0.2\n",
            "      cur_lr: 0.01967652736006374\n",
            "      entropy: 0.19680261529154247\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.07727121375501156\n",
            "      model: {}\n",
            "      policy_loss: 0.018376324894941516\n",
            "      total_loss: 11.115781943003336\n",
            "      vf_explained_var: 0.5910215377807617\n",
            "      vf_loss: 11.081951035393608\n",
            "    p_3:\n",
            "      allreduce_latency: 0.007314205169677734\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.02361183283207648\n",
            "      entropy: 1.9894601193589747e-12\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 4.280917485555013\n",
            "      model: {}\n",
            "      policy_loss: 0.05784336477518082\n",
            "      total_loss: 8.360580285390219\n",
            "      vf_explained_var: 0.07123015075922012\n",
            "      vf_loss: 7.446553071339925\n",
            "    p_5:\n",
            "      allreduce_latency: 0.0068817536036173505\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.01981372889281496\n",
            "      entropy: 0.030598731711506844\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.023244955111294985\n",
            "      model: {}\n",
            "      policy_loss: -0.008221368926266829\n",
            "      total_loss: 4.058287084102631\n",
            "      vf_explained_var: 0.1590019315481186\n",
            "      vf_loss: 4.061859687169393\n",
            "  num_steps_sampled: 900\n",
            "  num_steps_trained: 900\n",
            "iterations_since_restore: 5\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 86.125\n",
            "  ram_util_percent: 31.6\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_0: 3.0\n",
            "  p_1: 3.0\n",
            "  p_2: 9.0\n",
            "  p_3: 6.0\n",
            "  p_4: 7.0\n",
            "  p_5: 8.0\n",
            "policy_reward_mean:\n",
            "  p_0: -0.5\n",
            "  p_1: -0.2222222222222222\n",
            "  p_2: -1.1851851851851851\n",
            "  p_3: -1.5\n",
            "  p_4: 0.3333333333333333\n",
            "  p_5: 3.3333333333333335\n",
            "policy_reward_min:\n",
            "  p_0: -3.0\n",
            "  p_1: -5.0\n",
            "  p_2: -8.0\n",
            "  p_3: -9.0\n",
            "  p_4: -8.0\n",
            "  p_5: -3.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.3068732632947666\n",
            "  mean_env_wait_ms: 0.2672238457112631\n",
            "  mean_inference_ms: 6.0046579034258745\n",
            "  mean_raw_obs_processing_ms: 4.227692777775563\n",
            "time_since_restore: 7.993426561355591\n",
            "time_this_iter_s: 1.5334491729736328\n",
            "time_total_s: 7.993426561355591\n",
            "timers:\n",
            "  learn_time_ms: 1578.781\n",
            "timestamp: 1598342336\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 900\n",
            "training_iteration: 5\n",
            "\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.034001039278190134\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.034001039278190134\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.02720083142255211\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.02720083142255211\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.03264099770706253\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.03264099770706253\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.09542297512492572\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9594542998975585\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.09542297512492572\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9594542998975585\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.01971195997833793\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9319299152885292\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.01971195997833793\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9319299152885292\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.02176066513804169\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.02176066513804169\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "training loop = 6 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9549629876590181\n",
            "  p_0_gamma_mean: 0.9431789647506872\n",
            "  p_0_gamma_min: 0.9128771915578374\n",
            "  p_1_gamma_max: 0.9549629876590181\n",
            "  p_1_gamma_mean: 0.950580196863377\n",
            "  p_1_gamma_min: 0.9111350797026094\n",
            "  p_2_gamma_max: 0.9774519661959784\n",
            "  p_2_gamma_mean: 0.9578540645925279\n",
            "  p_2_gamma_min: 0.9029134342610622\n",
            "  p_3_gamma_max: 0.9549629876590181\n",
            "  p_3_gamma_mean: 0.9531540573118688\n",
            "  p_3_gamma_min: 0.9368736841875269\n",
            "  p_4_gamma_max: 0.9549629876590181\n",
            "  p_4_gamma_mean: 0.9341620593460459\n",
            "  p_4_gamma_min: 0.9214855724021866\n",
            "  p_5_gamma_max: 0.9549629876590181\n",
            "  p_5_gamma_mean: 0.9470412817459581\n",
            "  p_5_gamma_min: 0.9109535103642414\n",
            "date: 2020-08-25_07-58-59\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 108\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_0:\n",
            "      allreduce_latency: 0.009015957514444986\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.03541774924811472\n",
            "      entropy: 8.52101662996749e-16\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 3.335790736518074e-12\n",
            "      model: {}\n",
            "      policy_loss: 9.623666604359945e-09\n",
            "      total_loss: 4.053852041562398\n",
            "      vf_explained_var: 0.02034037373960018\n",
            "      vf_loss: 4.053851922353108\n",
            "    p_2:\n",
            "      allreduce_latency: 0.006608247756958008\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.04424926408523843\n",
            "      entropy: 0.006477684248238802\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 2.26239283879598\n",
            "      model: {}\n",
            "      policy_loss: 0.15801305820544562\n",
            "      total_loss: 4.731376727422078\n",
            "      vf_explained_var: -0.3518557846546173\n",
            "      vf_loss: 4.120885054270427\n",
            "    p_5:\n",
            "      allreduce_latency: 0.00806774033440484\n",
            "      cur_kl_coeff: 0.2\n",
            "      cur_lr: 0.022667359518793426\n",
            "      entropy: 0.19078115042712954\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.3704329834630092\n",
            "      model: {}\n",
            "      policy_loss: 0.027960074444611866\n",
            "      total_loss: 3.8250758515463934\n",
            "      vf_explained_var: -0.29046887159347534\n",
            "      vf_loss: 3.723029308848911\n",
            "  num_steps_sampled: 1080\n",
            "  num_steps_trained: 1080\n",
            "iterations_since_restore: 6\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 84.45\n",
            "  ram_util_percent: 31.6\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_0: 3.0\n",
            "  p_1: 3.0\n",
            "  p_2: 9.0\n",
            "  p_3: 6.0\n",
            "  p_4: 7.0\n",
            "  p_5: 8.0\n",
            "policy_reward_mean:\n",
            "  p_0: -2.2333333333333334\n",
            "  p_1: -0.2222222222222222\n",
            "  p_2: -1.4423076923076923\n",
            "  p_3: -1.5\n",
            "  p_4: 0.0\n",
            "  p_5: 3.6875\n",
            "policy_reward_min:\n",
            "  p_0: -7.0\n",
            "  p_1: -5.0\n",
            "  p_2: -8.0\n",
            "  p_3: -9.0\n",
            "  p_4: -8.0\n",
            "  p_5: -3.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.296560686913278\n",
            "  mean_env_wait_ms: 0.2273618699162971\n",
            "  mean_inference_ms: 5.947369576177631\n",
            "  mean_raw_obs_processing_ms: 4.261985462356164\n",
            "time_since_restore: 9.633419036865234\n",
            "time_this_iter_s: 1.6399924755096436\n",
            "time_total_s: 9.633419036865234\n",
            "timers:\n",
            "  learn_time_ms: 1585.301\n",
            "timestamp: 1598342339\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 1080\n",
            "training_iteration: 6\n",
            "\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.03264099770706253\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.03264099770706253\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.021760665138041688\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.021760665138041688\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.023654351974005516\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9319299152885292\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.023654351974005516\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9319299152885292\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.026112798165650028\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.026112798165650028\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.01740853211043335\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.01740853211043335\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.039169197248475035\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.039169197248475035\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "training loop = 7 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9549629876590181\n",
            "  p_0_gamma_mean: 0.9507544080488998\n",
            "  p_0_gamma_min: 0.9128771915578374\n",
            "  p_1_gamma_max: 0.9549629876590181\n",
            "  p_1_gamma_mean: 0.9549629876590179\n",
            "  p_1_gamma_min: 0.9549629876590181\n",
            "  p_2_gamma_max: 0.9774519661959784\n",
            "  p_2_gamma_mean: 0.9630590199323236\n",
            "  p_2_gamma_min: 0.9549629876590181\n",
            "  p_3_gamma_max: 0.9594542998975586\n",
            "  p_3_gamma_mean: 0.955771423861955\n",
            "  p_3_gamma_min: 0.9549629876590181\n",
            "  p_4_gamma_max: 0.9549629876590181\n",
            "  p_4_gamma_mean: 0.935640501275402\n",
            "  p_4_gamma_min: 0.9214855724021866\n",
            "  p_5_gamma_max: 0.9549629876590181\n",
            "  p_5_gamma_mean: 0.9470412817459581\n",
            "  p_5_gamma_min: 0.9109535103642414\n",
            "date: 2020-08-25_07-59-02\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 126\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_0:\n",
            "      allreduce_latency: 0.0064690907796223955\n",
            "      cur_kl_coeff: 0.2\n",
            "      cur_lr: 0.028334199398491782\n",
            "      entropy: 2.6288010246537678e-17\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 5.521750408984408e-18\n",
            "      model: {}\n",
            "      policy_loss: -4.537610544098748e-08\n",
            "      total_loss: 3.038767655690511\n",
            "      vf_explained_var: 0.3605605363845825\n",
            "      vf_loss: 3.038767615954081\n",
            "    p_5:\n",
            "      allreduce_latency: 0.005489455329047309\n",
            "      cur_kl_coeff: 0.2\n",
            "      cur_lr: 0.018133887615034743\n",
            "      entropy: 5.115265033674001e-06\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.0847731480995814\n",
            "      model: {}\n",
            "      policy_loss: 0.0010185281021727456\n",
            "      total_loss: 4.6123965051439075\n",
            "      vf_explained_var: 0.13079439103603363\n",
            "      vf_loss: 4.594423426522149\n",
            "  num_steps_sampled: 1260\n",
            "  num_steps_trained: 1260\n",
            "iterations_since_restore: 7\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 82.85\n",
            "  ram_util_percent: 31.625\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_0: 3.0\n",
            "  p_1: 3.0\n",
            "  p_2: 9.0\n",
            "  p_3: 6.0\n",
            "  p_4: 2.0\n",
            "  p_5: 9.0\n",
            "policy_reward_mean:\n",
            "  p_0: -4.770833333333333\n",
            "  p_1: 0.22580645161290322\n",
            "  p_2: -1.8717948717948718\n",
            "  p_3: -1.5\n",
            "  p_4: -0.6666666666666666\n",
            "  p_5: 5.311475409836065\n",
            "policy_reward_min:\n",
            "  p_0: -9.0\n",
            "  p_1: -4.0\n",
            "  p_2: -8.0\n",
            "  p_3: -9.0\n",
            "  p_4: -3.0\n",
            "  p_5: -3.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.28434759962545697\n",
            "  mean_env_wait_ms: 0.17833992939990004\n",
            "  mean_inference_ms: 5.908259671130006\n",
            "  mean_raw_obs_processing_ms: 4.3284859082126\n",
            "time_since_restore: 11.238818645477295\n",
            "time_this_iter_s: 1.6053996086120605\n",
            "time_total_s: 11.238818645477295\n",
            "timers:\n",
            "  learn_time_ms: 1582.743\n",
            "timestamp: 1598342342\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 1260\n",
            "training_iteration: 7\n",
            "\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.026112798165650025\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.026112798165650025\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.01740853211043335\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.01740853211043335\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.06485621197473602\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9304069342801448\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.06485621197473602\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9304069342801448\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.08414381211361253\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9451726820545104\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.08414381211361253\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9451726820545104\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.013926825688346682\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.013926825688346682\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.04700303669817004\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.04700303669817004\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "training loop = 8 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9549629876590181\n",
            "  p_0_gamma_mean: 0.9549629876590179\n",
            "  p_0_gamma_min: 0.9549629876590181\n",
            "  p_1_gamma_max: 0.9549629876590181\n",
            "  p_1_gamma_mean: 0.9549629876590179\n",
            "  p_1_gamma_min: 0.9549629876590181\n",
            "  p_2_gamma_max: 0.9774519661959784\n",
            "  p_2_gamma_mean: 0.9589130669056358\n",
            "  p_2_gamma_min: 0.9319299152885293\n",
            "  p_3_gamma_max: 0.9594542998975586\n",
            "  p_3_gamma_mean: 0.9557714238619553\n",
            "  p_3_gamma_min: 0.9549629876590181\n",
            "  p_4_gamma_max: 0.9549629876590181\n",
            "  p_4_gamma_mean: 0.9387651651398712\n",
            "  p_4_gamma_min: 0.9214855724021866\n",
            "  p_5_gamma_max: 0.9549629876590181\n",
            "  p_5_gamma_mean: 0.9470412817459581\n",
            "  p_5_gamma_min: 0.9109535103642414\n",
            "date: 2020-08-25_07-59-05\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 144\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_0:\n",
            "      allreduce_latency: 0.006867302788628472\n",
            "      cur_kl_coeff: 0.2\n",
            "      cur_lr: 0.034001039278190134\n",
            "      entropy: 2.0130704506089737e-17\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 2.108243625394843e-18\n",
            "      model: {}\n",
            "      policy_loss: 4.470348358154297e-08\n",
            "      total_loss: 5.9147267209159\n",
            "      vf_explained_var: 0.16257773339748383\n",
            "      vf_loss: 5.914726747406854\n",
            "    p_1:\n",
            "      allreduce_latency: 0.006648143132527669\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.02720083142255211\n",
            "      entropy: 0.010836011245070646\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.12295849248766899\n",
            "      model: {}\n",
            "      policy_loss: -0.003721139238526424\n",
            "      total_loss: 0.16719182246985534\n",
            "      vf_explained_var: 'null'\n",
            "      vf_loss: 0.14632125128991902\n",
            "    p_5:\n",
            "      allreduce_latency: 0.0067365169525146484\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.02176066513804169\n",
            "      entropy: 7.163884523227655e-06\n",
            "      entropy_coeff: 0.0\n",
            "      kl: -2.2833564372604087e-08\n",
            "      model: {}\n",
            "      policy_loss: 7.450580596923828e-08\n",
            "      total_loss: 4.068109750747681\n",
            "      vf_explained_var: 0.11310050636529922\n",
            "      vf_loss: 4.068109750747681\n",
            "  num_steps_sampled: 1440\n",
            "  num_steps_trained: 1440\n",
            "iterations_since_restore: 8\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 83.19999999999999\n",
            "  ram_util_percent: 31.625000000000004\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_0: 1.0\n",
            "  p_1: 3.0\n",
            "  p_2: 9.0\n",
            "  p_3: 6.0\n",
            "  p_5: 9.0\n",
            "policy_reward_mean:\n",
            "  p_0: -4.819672131147541\n",
            "  p_1: 0.6428571428571429\n",
            "  p_2: -2.0833333333333335\n",
            "  p_3: -1.5\n",
            "  p_5: 6.631578947368421\n",
            "policy_reward_min:\n",
            "  p_0: -9.0\n",
            "  p_1: -1.0\n",
            "  p_2: -8.0\n",
            "  p_3: -9.0\n",
            "  p_5: -2.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.2780979532352022\n",
            "  mean_env_wait_ms: 0.15510808285704658\n",
            "  mean_inference_ms: 5.856477611477033\n",
            "  mean_raw_obs_processing_ms: 4.376217393976\n",
            "time_since_restore: 12.838958740234375\n",
            "time_this_iter_s: 1.60014009475708\n",
            "time_total_s: 12.838958740234375\n",
            "timers:\n",
            "  learn_time_ms: 1582.322\n",
            "timestamp: 1598342345\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 1440\n",
            "training_iteration: 8\n",
            "\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.07708448252816569\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.972552746800975\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.07708448252816569\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.972552746800975\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.08518098239085593\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9736806821996851\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.08518098239085593\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9736806821996851\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.06814478591268475\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9736806821996851\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.06814478591268475\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9736806821996851\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.061667586022532556\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.972552746800975\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.061667586022532556\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.972552746800975\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.016712190826016018\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.016712190826016018\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9549629876590182\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.09250137903379882\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.972552746800975\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.09250137903379882\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:root:NaN or Inf found in input tensor.\n",
            "WARNING:root:NaN or Inf found in input tensor.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.972552746800975\n",
            "training loop = 9 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9549629876590181\n",
            "  p_0_gamma_mean: 0.9549629876590179\n",
            "  p_0_gamma_min: 0.9549629876590181\n",
            "  p_1_gamma_max: 0.9549629876590181\n",
            "  p_1_gamma_mean: 0.9549629876590179\n",
            "  p_1_gamma_min: 0.9549629876590181\n",
            "  p_2_gamma_max: 0.9774519661959784\n",
            "  p_2_gamma_mean: 0.9544929772974391\n",
            "  p_2_gamma_min: 0.9304069342801448\n",
            "  p_3_gamma_max: 0.9594542998975586\n",
            "  p_3_gamma_mean: 0.9540091688531439\n",
            "  p_3_gamma_min: 0.9451726820545104\n",
            "  p_4_gamma_max: 0.9549629876590181\n",
            "  p_4_gamma_mean: 0.9387651651398707\n",
            "  p_4_gamma_min: 0.9214855724021866\n",
            "  p_5_gamma_max: 0.9549629876590181\n",
            "  p_5_gamma_mean: 0.9505620399295401\n",
            "  p_5_gamma_min: 0.9109535103642414\n",
            "date: 2020-08-25_07-59-08\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 162\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_0:\n",
            "      allreduce_latency: 0.007499535878499349\n",
            "      cur_kl_coeff: 0.2\n",
            "      cur_lr: 0.03264099770706253\n",
            "      entropy: 9.698930726382751e-18\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 2.4890299763188666e-19\n",
            "      model: {}\n",
            "      policy_loss: -1.655684577094184e-09\n",
            "      total_loss: 6.537638160917494\n",
            "      vf_explained_var: 0.3495005965232849\n",
            "      vf_loss: 6.537638028462728\n",
            "    p_1:\n",
            "      allreduce_latency: 0.004954814910888672\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.021760665138041688\n",
            "      entropy: 8.658092080319572e-08\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 1.595058711245656e-05\n",
            "      model: {}\n",
            "      policy_loss: 8.64267349243164e-06\n",
            "      total_loss: 0.025175482034683228\n",
            "      vf_explained_var: 'null'\n",
            "      vf_loss: 0.025163666034738224\n",
            "    p_3:\n",
            "      allreduce_latency: 0.0060164133707682295\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.02611279816565003\n",
            "      entropy: 9.708882461011148e-13\n",
            "      entropy_coeff: 0.0\n",
            "      kl: .inf\n",
            "      model: {}\n",
            "      policy_loss: 4.495028396907704e-07\n",
            "      total_loss: .inf\n",
            "      vf_explained_var: -0.0006495912675745785\n",
            "      vf_loss: 14.204764445622763\n",
            "  num_steps_sampled: 1620\n",
            "  num_steps_trained: 1620\n",
            "iterations_since_restore: 9\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 82.95\n",
            "  ram_util_percent: 31.7\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_0: 0.0\n",
            "  p_1: 1.0\n",
            "  p_2: 9.0\n",
            "  p_3: 10.0\n",
            "  p_5: 9.0\n",
            "policy_reward_mean:\n",
            "  p_0: -5.9855072463768115\n",
            "  p_1: 0.9047619047619048\n",
            "  p_2: -2.129032258064516\n",
            "  p_3: 3.36\n",
            "  p_5: 6.962962962962963\n",
            "policy_reward_min:\n",
            "  p_0: -10.0\n",
            "  p_1: 0.0\n",
            "  p_2: -8.0\n",
            "  p_3: -9.0\n",
            "  p_5: -2.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.2755165642452665\n",
            "  mean_env_wait_ms: 0.14149510698719156\n",
            "  mean_inference_ms: 5.828133433778403\n",
            "  mean_raw_obs_processing_ms: 4.391839861596138\n",
            "time_since_restore: 14.459671258926392\n",
            "time_this_iter_s: 1.6207125186920166\n",
            "time_total_s: 14.459671258926392\n",
            "timers:\n",
            "  learn_time_ms: 1584.334\n",
            "timestamp: 1598342348\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 1620\n",
            "training_iteration: 9\n",
            "\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.08813803394567385\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9547732208611494\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.08813803394567385\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9547732208611494\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.11100165484055859\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.972552746800975\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.11100165484055859\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.972552746800975\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.08880132387244688\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.972552746800975\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.08880132387244688\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.972552746800975\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.07051042715653909\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9547732208611494\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.07051042715653909\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9547732208611494\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.1332019858086703\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.972552746800975\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.1332019858086703\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.972552746800975\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.029508842059114776\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.029508842059114776\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "training loop = 10 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9725527468009749\n",
            "  p_0_gamma_mean: 0.9581291443045701\n",
            "  p_0_gamma_min: 0.9549629876590181\n",
            "  p_1_gamma_max: 0.9736806821996851\n",
            "  p_1_gamma_mean: 0.9583321726763379\n",
            "  p_1_gamma_min: 0.9549629876590181\n",
            "  p_2_gamma_max: 0.9774519661959784\n",
            "  p_2_gamma_mean: 0.9560630440318018\n",
            "  p_2_gamma_min: 0.9304069342801448\n",
            "  p_3_gamma_max: 0.9725527468009749\n",
            "  p_3_gamma_mean: 0.9571753254986959\n",
            "  p_3_gamma_min: 0.9451726820545104\n",
            "  p_4_gamma_max: 0.9549629876590181\n",
            "  p_4_gamma_mean: 0.9414433583604171\n",
            "  p_4_gamma_min: 0.9214855724021866\n",
            "  p_5_gamma_max: 0.9725527468009749\n",
            "  p_5_gamma_mean: 0.9581291443045701\n",
            "  p_5_gamma_min: 0.9549629876590181\n",
            "date: 2020-08-25_07-59-11\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 180\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_0:\n",
            "      allreduce_latency: 0.007646481196085612\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.026112798165650025\n",
            "      entropy: 8.797002121594283e-18\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 6.515955736157004e-23\n",
            "      model: {}\n",
            "      policy_loss: 3.476937611897787e-08\n",
            "      total_loss: 9.082413673400879\n",
            "      vf_explained_var: -0.0033207933884114027\n",
            "      vf_loss: 9.082413832346598\n",
            "    p_2:\n",
            "      allreduce_latency: 0.0059564510981241865\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.06485621197473602\n",
            "      entropy: 0.00016435849602203234\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.0004571412719087675\n",
            "      model: {}\n",
            "      policy_loss: 3.0110590159893036e-05\n",
            "      total_loss: 8.125590880711874\n",
            "      vf_explained_var: -0.05745207145810127\n",
            "      vf_loss: 8.125469128290812\n",
            "    p_3:\n",
            "      allreduce_latency: 0.008108960257636176\n",
            "      cur_kl_coeff: 0.2\n",
            "      cur_lr: 0.08414381211361253\n",
            "      entropy: 7.88421068159915e-13\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 1.2258251930582636e-15\n",
            "      model: {}\n",
            "      policy_loss: 4.553132587009006e-08\n",
            "      total_loss: 18.790921370188396\n",
            "      vf_explained_var: 0.5725722908973694\n",
            "      vf_loss: 18.79092068142361\n",
            "  num_steps_sampled: 1800\n",
            "  num_steps_trained: 1800\n",
            "iterations_since_restore: 10\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 83.775\n",
            "  ram_util_percent: 31.7\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_0: 0.0\n",
            "  p_1: 1.0\n",
            "  p_2: 9.0\n",
            "  p_3: 10.0\n",
            "  p_5: 9.0\n",
            "policy_reward_mean:\n",
            "  p_0: -6.527777777777778\n",
            "  p_1: 0.8888888888888888\n",
            "  p_2: 2.4285714285714284\n",
            "  p_3: 1.3636363636363635\n",
            "  p_5: 6.959183673469388\n",
            "policy_reward_min:\n",
            "  p_0: -10.0\n",
            "  p_1: 0.0\n",
            "  p_2: -8.0\n",
            "  p_3: -9.0\n",
            "  p_5: -2.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.2743807646899989\n",
            "  mean_env_wait_ms: 0.1325221295511901\n",
            "  mean_inference_ms: 5.830618726308337\n",
            "  mean_raw_obs_processing_ms: 4.39224476580896\n",
            "time_since_restore: 16.059836626052856\n",
            "time_this_iter_s: 1.6001653671264648\n",
            "time_total_s: 16.059836626052856\n",
            "timers:\n",
            "  learn_time_ms: 1583.128\n",
            "timestamp: 1598342351\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 1800\n",
            "training_iteration: 10\n",
            "\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.07051042715653909\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9547732208611494\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.07051042715653909\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9547732208611494\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.08541171801203147\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.90765878306066\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.08541171801203147\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.90765878306066\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.05640834172523127\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9547732208611494\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.05640834172523127\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9547732208611494\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.06832937440962518\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.90765878306066\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.06832937440962518\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.90765878306066\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.10249406161443776\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.90765878306066\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.10249406161443776\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.90765878306066\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.03541061047093773\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.03541061047093773\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "training loop = 11 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9725527468009749\n",
            "  p_0_gamma_mean: 0.9580949862809538\n",
            "  p_0_gamma_min: 0.9547732208611495\n",
            "  p_1_gamma_max: 0.9736806821996851\n",
            "  p_1_gamma_mean: 0.9614983293218904\n",
            "  p_1_gamma_min: 0.9549629876590181\n",
            "  p_2_gamma_max: 0.9774519661959784\n",
            "  p_2_gamma_mean: 0.9551811845407013\n",
            "  p_2_gamma_min: 0.9304069342801448\n",
            "  p_3_gamma_max: 0.9725527468009749\n",
            "  p_3_gamma_mean: 0.9571411674750797\n",
            "  p_3_gamma_min: 0.9451726820545104\n",
            "  p_4_gamma_max: 0.9725527468009749\n",
            "  p_4_gamma_mean: 0.9506354497521992\n",
            "  p_4_gamma_min: 0.9214855724021866\n",
            "  p_5_gamma_max: 0.9887892005419217\n",
            "  p_5_gamma_mean: 0.9642178626234928\n",
            "  p_5_gamma_min: 0.9549629876590181\n",
            "date: 2020-08-25_07-59-14\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 198\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_1:\n",
            "      allreduce_latency: 0.005965113639831543\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.08518098239085593\n",
            "      entropy: 1.907028566468701e-06\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 9.48232804868591e-08\n",
            "      model: {}\n",
            "      policy_loss: 0.0\n",
            "      total_loss: 0.04950840429713329\n",
            "      vf_explained_var: -1.0\n",
            "      vf_loss: 0.04950837713355819\n",
            "    p_2:\n",
            "      allreduce_latency: 0.007094913058810764\n",
            "      cur_kl_coeff: 0.2\n",
            "      cur_lr: 0.06814478591268475\n",
            "      entropy: 0.0014573566214595404\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 2.0305253706788386e-05\n",
            "      model: {}\n",
            "      policy_loss: -6.382871005270217e-05\n",
            "      total_loss: 10.539189338684082\n",
            "      vf_explained_var: -0.25440850853919983\n",
            "      vf_loss: 10.53924920823839\n",
            "    p_3:\n",
            "      allreduce_latency: 0.00909733772277832\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.061667586022532556\n",
            "      entropy: 7.783837774275512e-13\n",
            "      entropy_coeff: 0.0\n",
            "      kl: -2.52739659491596e-18\n",
            "      model: {}\n",
            "      policy_loss: -1.800556977589925e-08\n",
            "      total_loss: 4.698645830154419\n",
            "      vf_explained_var: 0.0\n",
            "      vf_loss: 4.698645909627278\n",
            "  num_steps_sampled: 1980\n",
            "  num_steps_trained: 1980\n",
            "iterations_since_restore: 11\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 84.125\n",
            "  ram_util_percent: 31.7\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_0: 0.0\n",
            "  p_1: 1.0\n",
            "  p_2: 9.0\n",
            "  p_3: 10.0\n",
            "  p_5: 9.0\n",
            "policy_reward_mean:\n",
            "  p_0: -6.6716417910447765\n",
            "  p_1: 0.5333333333333333\n",
            "  p_2: 4.515151515151516\n",
            "  p_3: 0.5\n",
            "  p_5: 7.764705882352941\n",
            "policy_reward_min:\n",
            "  p_0: -10.0\n",
            "  p_1: 0.0\n",
            "  p_2: -6.0\n",
            "  p_3: -9.0\n",
            "  p_5: 2.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.27398562186568753\n",
            "  mean_env_wait_ms: 0.12602879255814006\n",
            "  mean_inference_ms: 5.838378950066056\n",
            "  mean_raw_obs_processing_ms: 4.393566545949354\n",
            "time_since_restore: 17.671802520751953\n",
            "time_this_iter_s: 1.6119658946990967\n",
            "time_total_s: 17.671802520751953\n",
            "timers:\n",
            "  learn_time_ms: 1570.585\n",
            "timestamp: 1598342354\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 1980\n",
            "training_iteration: 11\n",
            "\n",
            "checkpoint saved at /content/gdrive/My Drive/Colab Notebooks/PBT_MARL_watered_down/chkpt/checkpoint_11/checkpoint-11\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.08199524929155022\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.90765878306066\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.08199524929155022\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.90765878306066\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.06559619943324017\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.90765878306066\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.06559619943324017\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.90765878306066\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.04512667338018502\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9547732208611494\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.04512667338018502\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9547732208611494\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.07871543931988821\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.90765878306066\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.07871543931988821\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.90765878306066\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.08199524929155022\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.90765878306066\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.08199524929155022\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.90765878306066\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.04249273256512527\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.04249273256512527\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "training loop = 12 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9725527468009749\n",
            "  p_0_gamma_mean: 0.958060828257338\n",
            "  p_0_gamma_min: 0.9547732208611495\n",
            "  p_1_gamma_max: 0.9736806821996851\n",
            "  p_1_gamma_mean: 0.9529835724941857\n",
            "  p_1_gamma_min: 0.9076587830606598\n",
            "  p_2_gamma_max: 0.9736806821996851\n",
            "  p_2_gamma_mean: 0.9528981286633887\n",
            "  p_2_gamma_min: 0.9304069342801448\n",
            "  p_3_gamma_max: 0.9725527468009749\n",
            "  p_3_gamma_mean: 0.9482671056682923\n",
            "  p_3_gamma_min: 0.9076587830606598\n",
            "  p_4_gamma_max: 0.9725527468009749\n",
            "  p_4_gamma_mean: 0.9473110802398168\n",
            "  p_4_gamma_min: 0.9076587830606598\n",
            "  p_5_gamma_max: 0.9887892005419217\n",
            "  p_5_gamma_mean: 0.9703065809424154\n",
            "  p_5_gamma_min: 0.9549629876590181\n",
            "date: 2020-08-25_07-59-16\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 216\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_1:\n",
            "      allreduce_latency: 0.007981538772583008\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.11100165484055859\n",
            "      entropy: 1.7624405851771978e-06\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 4.018457270641799e-09\n",
            "      model: {}\n",
            "      policy_loss: 1.5397866566975912e-07\n",
            "      total_loss: 0.01354306936264038\n",
            "      vf_explained_var: -1.0\n",
            "      vf_loss: 0.013542921127130588\n",
            "    p_2:\n",
            "      allreduce_latency: 0.006714158587985569\n",
            "      cur_kl_coeff: 0.2\n",
            "      cur_lr: 0.08880132387244688\n",
            "      entropy: 0.03296116827469733\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.0053400192300614435\n",
            "      model: {}\n",
            "      policy_loss: -0.0049772850341267055\n",
            "      total_loss: 4.669153213500977\n",
            "      vf_explained_var: 0.15128931403160095\n",
            "      vf_loss: 4.673062456978692\n",
            "    p_4:\n",
            "      allreduce_latency: 0.007116635640462239\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.1332019858086703\n",
            "      entropy: 0.0002574654404108857\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.49549781779448193\n",
            "      model: {}\n",
            "      policy_loss: -0.005567895869414012\n",
            "      total_loss: 2.875406046708425\n",
            "      vf_explained_var: -0.05947205424308777\n",
            "      vf_loss: 2.7818743288517\n",
            "  num_steps_sampled: 2160\n",
            "  num_steps_trained: 2160\n",
            "iterations_since_restore: 12\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 82.9\n",
            "  ram_util_percent: 31.7\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_0: 0.0\n",
            "  p_1: 1.0\n",
            "  p_2: 9.0\n",
            "  p_3: 10.0\n",
            "  p_4: 8.0\n",
            "  p_5: 9.0\n",
            "policy_reward_mean:\n",
            "  p_0: -6.538461538461538\n",
            "  p_1: 0.4444444444444444\n",
            "  p_2: 1.6875\n",
            "  p_3: 0.5\n",
            "  p_4: 6.75\n",
            "  p_5: 9.0\n",
            "policy_reward_min:\n",
            "  p_0: -10.0\n",
            "  p_1: 0.0\n",
            "  p_2: -8.0\n",
            "  p_3: -9.0\n",
            "  p_4: 4.0\n",
            "  p_5: 9.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.27394852339170106\n",
            "  mean_env_wait_ms: 0.12066720488375586\n",
            "  mean_inference_ms: 5.841112872762613\n",
            "  mean_raw_obs_processing_ms: 4.394517852686494\n",
            "time_since_restore: 19.2247052192688\n",
            "time_this_iter_s: 1.5529026985168457\n",
            "time_total_s: 19.2247052192688\n",
            "timers:\n",
            "  learn_time_ms: 1566.865\n",
            "timestamp: 1598342356\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 2160\n",
            "training_iteration: 12\n",
            "\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.09839429914986027\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.90765878306066\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.09839429914986027\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.90765878306066\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.06297235145591057\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.90765878306066\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.06297235145591057\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.90765878306066\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.06297235145591057\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.90765878306066\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.06297235145591057\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.90765878306066\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.033994186052100216\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.033994186052100216\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.027195348841680175\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.027195348841680175\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.05099127907815033\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.05099127907815033\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "training loop = 13 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9725527468009749\n",
            "  p_0_gamma_mean: 0.9495460714296334\n",
            "  p_0_gamma_min: 0.9076587830606598\n",
            "  p_1_gamma_max: 0.9736806821996851\n",
            "  p_1_gamma_mean: 0.9444688156664811\n",
            "  p_1_gamma_min: 0.9076587830606598\n",
            "  p_2_gamma_max: 0.9736806821996851\n",
            "  p_2_gamma_mean: 0.9547066164294118\n",
            "  p_2_gamma_min: 0.9304069342801448\n",
            "  p_3_gamma_max: 0.9725527468009749\n",
            "  p_3_gamma_mean: 0.9393032176167333\n",
            "  p_3_gamma_min: 0.9076587830606598\n",
            "  p_4_gamma_max: 0.9725527468009749\n",
            "  p_4_gamma_mean: 0.9410996306491611\n",
            "  p_4_gamma_min: 0.9076587830606598\n",
            "  p_5_gamma_max: 0.9887892005419217\n",
            "  p_5_gamma_mean: 0.9763952992613381\n",
            "  p_5_gamma_min: 0.9549629876590181\n",
            "date: 2020-08-25_07-59-19\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 234\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_2:\n",
            "      allreduce_latency: 0.006234645843505859\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.05640834172523127\n",
            "      entropy: 0.010309093476583561\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.002462677271978464\n",
            "      model: {}\n",
            "      policy_loss: -0.0010350582500298817\n",
            "      total_loss: 4.056819438934326\n",
            "      vf_explained_var: 0.20694388449192047\n",
            "      vf_loss: 4.057362159093221\n",
            "    p_3:\n",
            "      allreduce_latency: 0.0065001646677653\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.06832937440962518\n",
            "      entropy: 7.803475928225742e-13\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 7.67865333027006e-18\n",
            "      model: {}\n",
            "      policy_loss: 6.20881716410319e-10\n",
            "      total_loss: 9.903762559096018\n",
            "      vf_explained_var: -0.5\n",
            "      vf_loss: 9.903762678305307\n",
            "    p_4:\n",
            "      allreduce_latency: 0.00659995608859592\n",
            "      cur_kl_coeff: 0.2\n",
            "      cur_lr: 0.10249406161443776\n",
            "      entropy: 0.001117203939271221\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 1.9138595044860657e-05\n",
            "      model: {}\n",
            "      policy_loss: 1.123816602759891e-05\n",
            "      total_loss: 2.852906492021349\n",
            "      vf_explained_var: 0.02000485546886921\n",
            "      vf_loss: 2.8528913855552673\n",
            "  num_steps_sampled: 2340\n",
            "  num_steps_trained: 2340\n",
            "iterations_since_restore: 13\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 83.275\n",
            "  ram_util_percent: 31.7\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_0: 0.0\n",
            "  p_1: 1.0\n",
            "  p_2: 9.0\n",
            "  p_3: 10.0\n",
            "  p_4: 8.0\n",
            "  p_5: 9.0\n",
            "policy_reward_mean:\n",
            "  p_0: -6.4411764705882355\n",
            "  p_1: 0.3870967741935484\n",
            "  p_2: 0.6111111111111112\n",
            "  p_3: -1.625\n",
            "  p_4: 7.5\n",
            "  p_5: 9.0\n",
            "policy_reward_min:\n",
            "  p_0: -10.0\n",
            "  p_1: 0.0\n",
            "  p_2: -8.0\n",
            "  p_3: -9.0\n",
            "  p_4: 4.0\n",
            "  p_5: 9.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.27407206669812\n",
            "  mean_env_wait_ms: 0.11643345791416984\n",
            "  mean_inference_ms: 5.851886257041756\n",
            "  mean_raw_obs_processing_ms: 4.398915868282871\n",
            "time_since_restore: 20.85443353652954\n",
            "time_this_iter_s: 1.6297283172607422\n",
            "time_total_s: 20.85443353652954\n",
            "timers:\n",
            "  learn_time_ms: 1569.739\n",
            "timestamp: 1598342359\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 2340\n",
            "training_iteration: 13\n",
            "\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.06118953489378039\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.06118953489378039\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.05037788116472846\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.90765878306066\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.05037788116472846\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.90765878306066\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.027195348841680175\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.027195348841680175\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.03263441861001621\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.03263441861001621\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.02175627907334414\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.02175627907334414\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.06118953489378039\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.06118953489378039\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "training loop = 14 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9725527468009749\n",
            "  p_0_gamma_mean: 0.9410313146019282\n",
            "  p_0_gamma_min: 0.9076587830606598\n",
            "  p_1_gamma_max: 0.9736806821996851\n",
            "  p_1_gamma_mean: 0.9359540588387767\n",
            "  p_1_gamma_min: 0.9076587830606598\n",
            "  p_2_gamma_max: 0.9736806821996851\n",
            "  p_2_gamma_mean: 0.9504596511090662\n",
            "  p_2_gamma_min: 0.9076587830606598\n",
            "  p_3_gamma_max: 0.9887892005419217\n",
            "  p_3_gamma_mean: 0.9461751603840166\n",
            "  p_3_gamma_min: 0.9076587830606598\n",
            "  p_4_gamma_max: 0.9887892005419217\n",
            "  p_4_gamma_mean: 0.9471883489680838\n",
            "  p_4_gamma_min: 0.9076587830606598\n",
            "  p_5_gamma_max: 0.9887892005419217\n",
            "  p_5_gamma_mean: 0.9824840175802606\n",
            "  p_5_gamma_min: 0.9549629876590181\n",
            "date: 2020-08-25_07-59-22\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 252\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_1:\n",
            "      allreduce_latency: 0.005964438120524089\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.06559619943324017\n",
            "      entropy: 1.7059727118369967e-06\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 2.6408254472087833e-09\n",
            "      model: {}\n",
            "      policy_loss: -1.857678100236626e-07\n",
            "      total_loss: 0.002395770454313606\n",
            "      vf_explained_var: -1.0\n",
            "      vf_loss: 0.0023959390042970576\n",
            "    p_2:\n",
            "      allreduce_latency: 0.007501761118570964\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.04512667338018502\n",
            "      entropy: 0.0021777141179579\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 6.16374093927637e-05\n",
            "      model: {}\n",
            "      policy_loss: -0.00018586218357086182\n",
            "      total_loss: 0.2912312236924966\n",
            "      vf_explained_var: -1.0\n",
            "      vf_loss: 0.29140473405520123\n",
            "    p_3:\n",
            "      allreduce_latency: 0.0073172251383463545\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.07871543931988821\n",
            "      entropy: 7.808430732154e-13\n",
            "      entropy_coeff: 0.0\n",
            "      kl: -3.58328741147594e-17\n",
            "      model: {}\n",
            "      policy_loss: -1.862645149230957e-08\n",
            "      total_loss: 7.536577383677165\n",
            "      vf_explained_var: -0.009354154579341412\n",
            "      vf_loss: 7.536577383677165\n",
            "    p_4:\n",
            "      allreduce_latency: 0.009132703145345053\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.08199524929155022\n",
            "      entropy: 0.0297465219274429\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.13077213506470073\n",
            "      model: {}\n",
            "      policy_loss: 0.034816011786460876\n",
            "      total_loss: 6.738120714823405\n",
            "      vf_explained_var: 0.14803065359592438\n",
            "      vf_loss: 6.6771500905354815\n",
            "  num_steps_sampled: 2520\n",
            "  num_steps_trained: 2520\n",
            "iterations_since_restore: 14\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 82.82499999999999\n",
            "  ram_util_percent: 31.7\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_0: -1.0\n",
            "  p_1: 1.0\n",
            "  p_2: 9.0\n",
            "  p_3: 10.0\n",
            "  p_4: 10.0\n",
            "policy_reward_mean:\n",
            "  p_0: -8.3125\n",
            "  p_1: 0.09090909090909091\n",
            "  p_2: 0.5\n",
            "  p_3: -3.63265306122449\n",
            "  p_4: 7.638888888888889\n",
            "policy_reward_min:\n",
            "  p_0: -10.0\n",
            "  p_1: 0.0\n",
            "  p_2: -8.0\n",
            "  p_3: -10.0\n",
            "  p_4: 4.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.27438664720710576\n",
            "  mean_env_wait_ms: 0.11278445034939676\n",
            "  mean_inference_ms: 5.862500036370968\n",
            "  mean_raw_obs_processing_ms: 4.39962602811964\n",
            "time_since_restore: 22.457784175872803\n",
            "time_this_iter_s: 1.6033506393432617\n",
            "time_total_s: 22.457784175872803\n",
            "timers:\n",
            "  learn_time_ms: 1577.296\n",
            "timestamp: 1598342362\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 2520\n",
            "training_iteration: 14\n",
            "\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.03916130233201945\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.03916130233201945\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.02175627907334414\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.02175627907334414\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.02175627907334414\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.02175627907334414\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.06049960384039709\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9760632579468023\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.06049960384039709\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9760632579468023\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.017405023258675312\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.017405023258675312\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.07342744187253646\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.07342744187253646\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "training loop = 15 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9887892005419217\n",
            "  p_0_gamma_mean: 0.9457128521894944\n",
            "  p_0_gamma_min: 0.9076587830606598\n",
            "  p_1_gamma_max: 0.9736806821996851\n",
            "  p_1_gamma_mean: 0.9259418864478188\n",
            "  p_1_gamma_min: 0.9076587830606598\n",
            "  p_2_gamma_max: 0.9887892005419217\n",
            "  p_2_gamma_mean: 0.9575065592026221\n",
            "  p_2_gamma_min: 0.9076587830606598\n",
            "  p_3_gamma_max: 0.9887892005419217\n",
            "  p_3_gamma_mean: 0.9518357285320335\n",
            "  p_3_gamma_min: 0.9076587830606598\n",
            "  p_4_gamma_max: 0.9887892005419217\n",
            "  p_4_gamma_mean: 0.9532770672870063\n",
            "  p_4_gamma_min: 0.9076587830606598\n",
            "  p_5_gamma_max: 0.9887892005419217\n",
            "  p_5_gamma_mean: 0.9871655551678267\n",
            "  p_5_gamma_min: 0.9725527468009749\n",
            "date: 2020-08-25_07-59-25\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 270\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_1:\n",
            "      allreduce_latency: 0.005656083424886067\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.06297235145591057\n",
            "      entropy: 1.6974596140547267e-06\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 2.8889893718038923e-10\n",
            "      model: {}\n",
            "      policy_loss: 0.0\n",
            "      total_loss: 0.0012275983899598941\n",
            "      vf_explained_var: -1.0\n",
            "      vf_loss: 0.0012275978224352002\n",
            "    p_2:\n",
            "      allreduce_latency: 0.0061359405517578125\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.06297235145591057\n",
            "      entropy: 0.001567797230867048\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 2.9173041108757993e-06\n",
            "      model: {}\n",
            "      policy_loss: -2.900660062247577e-05\n",
            "      total_loss: 0.13193105285366377\n",
            "      vf_explained_var: -1.0\n",
            "      vf_loss: 0.13195949792861938\n",
            "    p_4:\n",
            "      allreduce_latency: 0.0075778961181640625\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.027195348841680175\n",
            "      entropy: 0.0025189093564033332\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.0005906578282320597\n",
            "      model: {}\n",
            "      policy_loss: -0.0012065858269731204\n",
            "      total_loss: 9.8923659324646\n",
            "      vf_explained_var: -0.030639102682471275\n",
            "      vf_loss: 9.893454233805338\n",
            "    p_5:\n",
            "      allreduce_latency: 0.004467566808064778\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.05099127907815033\n",
            "      entropy: 6.960036262171343e-05\n",
            "      entropy_coeff: 0.0\n",
            "      kl: -4.4484405729387316e-07\n",
            "      model: {}\n",
            "      policy_loss: -2.483526865641276e-09\n",
            "      total_loss: 5.29136065642039\n",
            "      vf_explained_var: 0.04474937915802002\n",
            "      vf_loss: 5.2913609345753985\n",
            "  num_steps_sampled: 2700\n",
            "  num_steps_trained: 2700\n",
            "iterations_since_restore: 15\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 82.9\n",
            "  ram_util_percent: 31.7\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_0: -10.0\n",
            "  p_1: 0.0\n",
            "  p_2: 9.0\n",
            "  p_3: 10.0\n",
            "  p_4: 10.0\n",
            "  p_5: 7.0\n",
            "policy_reward_mean:\n",
            "  p_0: -10.0\n",
            "  p_1: 0.0\n",
            "  p_2: -0.1791044776119403\n",
            "  p_3: -6.852941176470588\n",
            "  p_4: 3.9791666666666665\n",
            "  p_5: 7.0\n",
            "policy_reward_min:\n",
            "  p_0: -10.0\n",
            "  p_1: 0.0\n",
            "  p_2: -8.0\n",
            "  p_3: -10.0\n",
            "  p_4: -7.0\n",
            "  p_5: 7.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.2742744785985877\n",
            "  mean_env_wait_ms: 0.10972677364210068\n",
            "  mean_inference_ms: 5.868956151314899\n",
            "  mean_raw_obs_processing_ms: 4.402722568360996\n",
            "time_since_restore: 24.03085470199585\n",
            "time_this_iter_s: 1.5730705261230469\n",
            "time_total_s: 24.03085470199585\n",
            "timers:\n",
            "  learn_time_ms: 1580.749\n",
            "timestamp: 1598342365\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 2700\n",
            "training_iteration: 15\n",
            "\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.017405023258675312\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.017405023258675312\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.06147461656391013\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.906666947409507\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.06147461656391013\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.906666947409507\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.058741953498029166\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.058741953498029166\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.070490344197635\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.070490344197635\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.084588413037162\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.084588413037162\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.058741953498029166\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.058741953498029166\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "training loop = 16 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9887892005419217\n",
            "  p_0_gamma_mean: 0.9500577759380507\n",
            "  p_0_gamma_min: 0.9076587830606598\n",
            "  p_1_gamma_max: 0.9887892005419217\n",
            "  p_1_gamma_mean: 0.9287516545813181\n",
            "  p_1_gamma_min: 0.9076587830606598\n",
            "  p_2_gamma_max: 0.9887892005419217\n",
            "  p_2_gamma_mean: 0.9603163273361216\n",
            "  p_2_gamma_min: 0.9076587830606598\n",
            "  p_3_gamma_max: 0.9887892005419217\n",
            "  p_3_gamma_mean: 0.9538899826134682\n",
            "  p_3_gamma_min: 0.9076587830606598\n",
            "  p_4_gamma_max: 0.9887892005419217\n",
            "  p_4_gamma_mean: 0.9579586048745725\n",
            "  p_4_gamma_min: 0.9076587830606598\n",
            "  p_5_gamma_max: 0.9887892005419217\n",
            "  p_5_gamma_mean: 0.9887892005419213\n",
            "  p_5_gamma_min: 0.9887892005419217\n",
            "date: 2020-08-25_07-59-28\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 288\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_4:\n",
            "      allreduce_latency: 0.009238613976372613\n",
            "      cur_kl_coeff: 0.2\n",
            "      cur_lr: 0.02175627907334414\n",
            "      entropy: 0.03163669797437631\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.048706448533422486\n",
            "      model: {}\n",
            "      policy_loss: 0.029565324179000325\n",
            "      total_loss: 5.273817141850789\n",
            "      vf_explained_var: 0.22128552198410034\n",
            "      vf_loss: 5.23451042175293\n",
            "    p_5:\n",
            "      allreduce_latency: 0.006511608759562175\n",
            "      cur_kl_coeff: 0.2\n",
            "      cur_lr: 0.061189534893780394\n",
            "      entropy: 7.65131779998127e-05\n",
            "      entropy_coeff: 0.0\n",
            "      kl: -1.0746206176766766e-07\n",
            "      model: {}\n",
            "      policy_loss: -2.4421347512139214e-08\n",
            "      total_loss: 7.203743802176581\n",
            "      vf_explained_var: -0.004328204318881035\n",
            "      vf_loss: 7.203743722703722\n",
            "  num_steps_sampled: 2880\n",
            "  num_steps_trained: 2880\n",
            "iterations_since_restore: 16\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 84.30000000000001\n",
            "  ram_util_percent: 31.72\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_1: 0.0\n",
            "  p_2: 9.0\n",
            "  p_3: -8.0\n",
            "  p_4: 10.0\n",
            "  p_5: 10.0\n",
            "policy_reward_mean:\n",
            "  p_1: 0.0\n",
            "  p_2: -1.9615384615384615\n",
            "  p_3: -8.238095238095237\n",
            "  p_4: 0.4393939393939394\n",
            "  p_5: 8.2\n",
            "policy_reward_min:\n",
            "  p_1: 0.0\n",
            "  p_2: -8.0\n",
            "  p_3: -10.0\n",
            "  p_4: -10.0\n",
            "  p_5: 7.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.27427534348925364\n",
            "  mean_env_wait_ms: 0.1071442201803243\n",
            "  mean_inference_ms: 5.874710606425114\n",
            "  mean_raw_obs_processing_ms: 4.406277331348096\n",
            "time_since_restore: 25.625654458999634\n",
            "time_this_iter_s: 1.5947997570037842\n",
            "time_total_s: 25.625654458999634\n",
            "timers:\n",
            "  learn_time_ms: 1576.552\n",
            "timestamp: 1598342368\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 2880\n",
            "training_iteration: 16\n",
            "\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.070490344197635\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.070490344197635\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.04917969325112811\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.906666947409507\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.04917969325112811\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.906666947409507\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.05639227535810801\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.05639227535810801\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.084588413037162\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.084588413037162\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.0676707304297296\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.0676707304297296\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.03685400262352748\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.03685400262352748\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "training loop = 17 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9887892005419217\n",
            "  p_0_gamma_mean: 0.9561806522805898\n",
            "  p_0_gamma_min: 0.9076587830606598\n",
            "  p_1_gamma_max: 0.9887892005419217\n",
            "  p_1_gamma_mean: 0.9220837277900791\n",
            "  p_1_gamma_min: 0.906666947409507\n",
            "  p_2_gamma_max: 0.9887892005419217\n",
            "  p_2_gamma_mean: 0.9646612510846779\n",
            "  p_2_gamma_min: 0.9076587830606598\n",
            "  p_3_gamma_max: 0.9887892005419217\n",
            "  p_3_gamma_mean: 0.9637820139800468\n",
            "  p_3_gamma_min: 0.9076587830606598\n",
            "  p_4_gamma_max: 0.9887892005419217\n",
            "  p_4_gamma_mean: 0.966072683647168\n",
            "  p_4_gamma_min: 0.9076587830606598\n",
            "  p_5_gamma_max: 0.9887892005419217\n",
            "  p_5_gamma_mean: 0.9887892005419213\n",
            "  p_5_gamma_min: 0.9887892005419217\n",
            "date: 2020-08-25_07-59-31\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 306\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_0:\n",
            "      allreduce_latency: 0.0064773956934611005\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.03916130233201945\n",
            "      entropy: 8.7948045784336e-18\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 3.765739076004498e-23\n",
            "      model: {}\n",
            "      policy_loss: 2.638747294743856e-09\n",
            "      total_loss: 7.459369262059529\n",
            "      vf_explained_var: 0.2313661128282547\n",
            "      vf_loss: 7.459369262059529\n",
            "    p_2:\n",
            "      allreduce_latency: 0.00681455930074056\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.02175627907334414\n",
            "      entropy: 0.008026762632653117\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.00016015384608181193\n",
            "      model: {}\n",
            "      policy_loss: -8.007977157831192e-05\n",
            "      total_loss: 35.23893483479818\n",
            "      vf_explained_var: 0.08024007081985474\n",
            "      vf_loss: 35.23898379007975\n",
            "    p_4:\n",
            "      allreduce_latency: 0.005069653193155925\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.017405023258675312\n",
            "      entropy: 0.024454293038588187\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.4811523829674192\n",
            "      model: {}\n",
            "      policy_loss: -0.019718363260229427\n",
            "      total_loss: 4.989381869633992\n",
            "      vf_explained_var: 0.22131818532943726\n",
            "      vf_loss: 4.912869453430176\n",
            "    p_5:\n",
            "      allreduce_latency: 0.00612791379292806\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.07342744187253646\n",
            "      entropy: 7.664147900262226e-05\n",
            "      entropy_coeff: 0.0\n",
            "      kl: -5.8346071677798745e-09\n",
            "      model: {}\n",
            "      policy_loss: 9.437402089436849e-08\n",
            "      total_loss: 10.099949677785238\n",
            "      vf_explained_var: 0.03148692846298218\n",
            "      vf_loss: 10.099949836730957\n",
            "  num_steps_sampled: 3060\n",
            "  num_steps_trained: 3060\n",
            "iterations_since_restore: 17\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 82.63333333333334\n",
            "  ram_util_percent: 31.8\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_0: -10.0\n",
            "  p_1: 0.0\n",
            "  p_2: 10.0\n",
            "  p_3: -8.0\n",
            "  p_4: 10.0\n",
            "  p_5: 10.0\n",
            "policy_reward_mean:\n",
            "  p_0: -10.0\n",
            "  p_1: 0.0\n",
            "  p_2: 0.5\n",
            "  p_3: -8.11111111111111\n",
            "  p_4: -0.8507462686567164\n",
            "  p_5: 8.333333333333334\n",
            "policy_reward_min:\n",
            "  p_0: -10.0\n",
            "  p_1: 0.0\n",
            "  p_2: -8.0\n",
            "  p_3: -10.0\n",
            "  p_4: -10.0\n",
            "  p_5: 7.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.2742976278702727\n",
            "  mean_env_wait_ms: 0.10489735762814643\n",
            "  mean_inference_ms: 5.878080729942992\n",
            "  mean_raw_obs_processing_ms: 4.408224074224851\n",
            "time_since_restore: 27.15154790878296\n",
            "time_this_iter_s: 1.5258934497833252\n",
            "time_total_s: 27.15154790878296\n",
            "timers:\n",
            "  learn_time_ms: 1570.289\n",
            "timestamp: 1598342371\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 3060\n",
            "training_iteration: 17\n",
            "\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.084588413037162\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.084588413037162\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.0676707304297296\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.0676707304297296\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.04511382028648641\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.04511382028648641\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.1015060956445944\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.1015060956445944\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.08120487651567551\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.08120487651567551\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.04422480314823298\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.04422480314823298\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "training loop = 18 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9887892005419217\n",
            "  p_0_gamma_mean: 0.966072683647168\n",
            "  p_0_gamma_min: 0.9076587830606598\n",
            "  p_1_gamma_max: 0.9887892005419217\n",
            "  p_1_gamma_mean: 0.9219051973728718\n",
            "  p_1_gamma_min: 0.906666947409507\n",
            "  p_2_gamma_max: 0.9887892005419217\n",
            "  p_2_gamma_mean: 0.970784127427217\n",
            "  p_2_gamma_min: 0.9076587830606598\n",
            "  p_3_gamma_max: 0.9887892005419217\n",
            "  p_3_gamma_mean: 0.9783854891266737\n",
            "  p_3_gamma_min: 0.9076587830606598\n",
            "  p_4_gamma_max: 0.9887892005419217\n",
            "  p_4_gamma_mean: 0.9806761587937952\n",
            "  p_4_gamma_min: 0.9076587830606598\n",
            "  p_5_gamma_max: 0.9887892005419217\n",
            "  p_5_gamma_mean: 0.9884492067142661\n",
            "  p_5_gamma_min: 0.9869003459438371\n",
            "date: 2020-08-25_07-59-33\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 324\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_0:\n",
            "      allreduce_latency: 0.008134285608927408\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.017405023258675312\n",
            "      entropy: 8.7944839080828e-18\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 3.04262704510658e-24\n",
            "      model: {}\n",
            "      policy_loss: 1.4901161193847656e-08\n",
            "      total_loss: 7.90637747446696\n",
            "      vf_explained_var: 0.2387692928314209\n",
            "      vf_loss: 7.906376997629802\n",
            "    p_2:\n",
            "      allreduce_latency: 0.006253480911254883\n",
            "      cur_kl_coeff: 0.2\n",
            "      cur_lr: 0.058741953498029166\n",
            "      entropy: 0.0021533426156060565\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 3.833155541441455e-05\n",
            "      model: {}\n",
            "      policy_loss: -3.8346482647789846e-05\n",
            "      total_loss: 6.8650791380140515\n",
            "      vf_explained_var: 0.02636312134563923\n",
            "      vf_loss: 6.865109735065037\n",
            "    p_3:\n",
            "      allreduce_latency: 0.006018241246541341\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.070490344197635\n",
            "      entropy: 7.784580452763664e-13\n",
            "      entropy_coeff: 0.0\n",
            "      kl: -4.739808671767733e-20\n",
            "      model: {}\n",
            "      policy_loss: -1.7462298274040222e-08\n",
            "      total_loss: 8.169245481491089\n",
            "      vf_explained_var: 0.0\n",
            "      vf_loss: 8.16924532254537\n",
            "  num_steps_sampled: 3240\n",
            "  num_steps_trained: 3240\n",
            "iterations_since_restore: 18\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 83.85\n",
            "  ram_util_percent: 31.8\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_0: -10.0\n",
            "  p_1: 0.0\n",
            "  p_2: 10.0\n",
            "  p_3: -8.0\n",
            "  p_4: 10.0\n",
            "  p_5: 10.0\n",
            "policy_reward_mean:\n",
            "  p_0: -10.0\n",
            "  p_1: 0.0\n",
            "  p_2: 5.176470588235294\n",
            "  p_3: -8.56\n",
            "  p_4: -3.269230769230769\n",
            "  p_5: 8.333333333333334\n",
            "policy_reward_min:\n",
            "  p_0: -10.0\n",
            "  p_1: 0.0\n",
            "  p_2: -8.0\n",
            "  p_3: -10.0\n",
            "  p_4: -10.0\n",
            "  p_5: 7.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.2747576972344787\n",
            "  mean_env_wait_ms: 0.10285049228518314\n",
            "  mean_inference_ms: 5.875950343466936\n",
            "  mean_raw_obs_processing_ms: 4.411888048512976\n",
            "time_since_restore: 28.63065266609192\n",
            "time_this_iter_s: 1.47910475730896\n",
            "time_total_s: 28.63065266609192\n",
            "timers:\n",
            "  learn_time_ms: 1558.454\n",
            "timestamp: 1598342373\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 3240\n",
            "training_iteration: 18\n",
            "\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.0676707304297296\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.0676707304297296\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.05306976377787958\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.05306976377787958\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.0649639012125404\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.0649639012125404\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.011291713024432319\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9153374406679863\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.011291713024432319\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9153374406679863\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.09744585181881062\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.09744585181881062\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.05306976377787958\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.05306976377787958\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "training loop = 19 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9887892005419217\n",
            "  p_0_gamma_mean: 0.9806761587937952\n",
            "  p_0_gamma_min: 0.9076587830606598\n",
            "  p_1_gamma_max: 0.9887892005419217\n",
            "  p_1_gamma_mean: 0.9365086725194989\n",
            "  p_1_gamma_min: 0.906666947409507\n",
            "  p_2_gamma_max: 0.9887892005419217\n",
            "  p_2_gamma_mean: 0.9806761587937952\n",
            "  p_2_gamma_min: 0.9076587830606598\n",
            "  p_3_gamma_max: 0.9887892005419217\n",
            "  p_3_gamma_mean: 0.9864985308748\n",
            "  p_3_gamma_min: 0.9760632579468022\n",
            "  p_4_gamma_max: 0.9887892005419217\n",
            "  p_4_gamma_mean: 0.9887892005419213\n",
            "  p_4_gamma_min: 0.9887892005419217\n",
            "  p_5_gamma_max: 0.9887892005419217\n",
            "  p_5_gamma_mean: 0.9881092128866112\n",
            "  p_5_gamma_min: 0.9869003459438371\n",
            "date: 2020-08-25_07-59-36\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 342\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_0:\n",
            "      allreduce_latency: 0.006170988082885742\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.070490344197635\n",
            "      entropy: 6.679702649228025e-18\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 6.759127375837923e-25\n",
            "      model: {}\n",
            "      policy_loss: 3.688037395477295e-07\n",
            "      total_loss: 77.74219640096028\n",
            "      vf_explained_var: 3.973643103449831e-08\n",
            "      vf_loss: 77.74219703674316\n",
            "    p_2:\n",
            "      allreduce_latency: 0.010690450668334961\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.05639227535810801\n",
            "      entropy: 0.0009990278631448746\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 6.375906044316555e-06\n",
            "      model: {}\n",
            "      policy_loss: -9.429175406694412e-06\n",
            "      total_loss: 5.600119272867839\n",
            "      vf_explained_var: 0.16115702688694\n",
            "      vf_loss: 5.600127696990967\n",
            "    p_3:\n",
            "      allreduce_latency: 0.00848825772603353\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.08458841303716198\n",
            "      entropy: 7.784451071304414e-13\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 5.687813660774144e-19\n",
            "      model: {}\n",
            "      policy_loss: -1.241763432820638e-08\n",
            "      total_loss: 10.36888058980306\n",
            "      vf_explained_var: 1.9868215517249155e-08\n",
            "      vf_loss: 10.36888058980306\n",
            "    p_4:\n",
            "      allreduce_latency: 0.007906516393025717\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.0676707304297296\n",
            "      entropy: 1.0979251866215483e-12\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.017262286853188532\n",
            "      model: {}\n",
            "      policy_loss: 0.00017522399624188742\n",
            "      total_loss: 6.62208890914917\n",
            "      vf_explained_var: 0.2018391340970993\n",
            "      vf_loss: 6.618461648623149\n",
            "  num_steps_sampled: 3420\n",
            "  num_steps_trained: 3420\n",
            "iterations_since_restore: 19\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 84.4\n",
            "  ram_util_percent: 31.8\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_0: 10.0\n",
            "  p_1: 0.0\n",
            "  p_2: 10.0\n",
            "  p_3: -8.0\n",
            "  p_4: 10.0\n",
            "  p_5: 10.0\n",
            "policy_reward_mean:\n",
            "  p_0: -2.0\n",
            "  p_1: 0.0\n",
            "  p_2: 6.979591836734694\n",
            "  p_3: -8.952380952380953\n",
            "  p_4: -7.7254901960784315\n",
            "  p_5: 8.333333333333334\n",
            "policy_reward_min:\n",
            "  p_0: -10.0\n",
            "  p_1: 0.0\n",
            "  p_2: 0.0\n",
            "  p_3: -10.0\n",
            "  p_4: -10.0\n",
            "  p_5: 7.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.27541941621831323\n",
            "  mean_env_wait_ms: 0.1010224059411858\n",
            "  mean_inference_ms: 5.8757299789566355\n",
            "  mean_raw_obs_processing_ms: 4.411267054943491\n",
            "time_since_restore: 30.201327085494995\n",
            "time_this_iter_s: 1.5706744194030762\n",
            "time_total_s: 30.201327085494995\n",
            "timers:\n",
            "  learn_time_ms: 1553.315\n",
            "timestamp: 1598342376\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 3420\n",
            "training_iteration: 19\n",
            "\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.08120487651567551\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.08120487651567551\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.04245581102230367\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.04245581102230367\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.07795668145504848\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.07795668145504848\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.0339307678496414\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9196966477821733\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.0339307678496414\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9196966477821733\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.0779566814550485\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.0779566814550485\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.06368371653345549\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.06368371653345549\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "training loop = 20 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9887892005419217\n",
            "  p_0_gamma_mean: 0.9887892005419213\n",
            "  p_0_gamma_min: 0.9887892005419217\n",
            "  p_1_gamma_max: 0.9887892005419217\n",
            "  p_1_gamma_mean: 0.9507721538384706\n",
            "  p_1_gamma_min: 0.906666947409507\n",
            "  p_2_gamma_max: 0.9887892005419217\n",
            "  p_2_gamma_mean: 0.9887892005419213\n",
            "  p_2_gamma_min: 0.9887892005419217\n",
            "  p_3_gamma_max: 0.9887892005419217\n",
            "  p_3_gamma_mean: 0.9732772140974915\n",
            "  p_3_gamma_min: 0.9153374406679863\n",
            "  p_4_gamma_max: 0.9887892005419217\n",
            "  p_4_gamma_mean: 0.9887892005419213\n",
            "  p_4_gamma_min: 0.9887892005419217\n",
            "  p_5_gamma_max: 0.9887892005419217\n",
            "  p_5_gamma_mean: 0.9877692190589561\n",
            "  p_5_gamma_min: 0.9869003459438371\n",
            "date: 2020-08-25_07-59-39\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 360\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_0:\n",
            "      allreduce_latency: 0.006066242853800456\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.08458841303716198\n",
            "      entropy: 6.679702649228025e-18\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.0\n",
            "      model: {}\n",
            "      policy_loss: -1.241763432820638e-08\n",
            "      total_loss: 31.41603724161784\n",
            "      vf_explained_var: 3.973643103449831e-08\n",
            "      vf_loss: 31.41603660583496\n",
            "    p_1:\n",
            "      allreduce_latency: 0.007631937662760417\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.0676707304297296\n",
            "      entropy: 1.4913956884280803e-06\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 1.1517869695036135e-08\n",
            "      model: {}\n",
            "      policy_loss: 1.1175870895385742e-07\n",
            "      total_loss: 1.2419856836398442\n",
            "      vf_explained_var: -0.31980887055397034\n",
            "      vf_loss: 1.2419856091340382\n",
            "    p_4:\n",
            "      allreduce_latency: 0.007259686787923177\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.08120487651567551\n",
            "      entropy: 6.845627944735908e-14\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 2.7715520682661973e-14\n",
            "      model: {}\n",
            "      policy_loss: 5.712111790974935e-08\n",
            "      total_loss: 5.685967604319255\n",
            "      vf_explained_var: 0.20511187613010406\n",
            "      vf_loss: 5.685967763264974\n",
            "    p_5:\n",
            "      allreduce_latency: 0.006072680155436198\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.04422480314823298\n",
            "      entropy: 6.0493097407743335e-05\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 6.470026351716266e-07\n",
            "      model: {}\n",
            "      policy_loss: -4.04814879099528e-07\n",
            "      total_loss: 1.2214669684569042\n",
            "      vf_explained_var: -0.8468313813209534\n",
            "      vf_loss: 1.2214672019084294\n",
            "  num_steps_sampled: 3600\n",
            "  num_steps_trained: 3600\n",
            "iterations_since_restore: 20\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 83.325\n",
            "  ram_util_percent: 31.8\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_0: 10.0\n",
            "  p_1: 0.0\n",
            "  p_2: 10.0\n",
            "  p_3: -9.0\n",
            "  p_4: -7.0\n",
            "  p_5: 10.0\n",
            "policy_reward_mean:\n",
            "  p_0: 0.0\n",
            "  p_1: -0.8\n",
            "  p_2: 8.76923076923077\n",
            "  p_3: -9.0\n",
            "  p_4: -9.081632653061224\n",
            "  p_5: 6.441860465116279\n",
            "policy_reward_min:\n",
            "  p_0: -10.0\n",
            "  p_1: -1.0\n",
            "  p_2: 0.0\n",
            "  p_3: -9.0\n",
            "  p_4: -10.0\n",
            "  p_5: 1.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.27602147749228234\n",
            "  mean_env_wait_ms: 0.0993557004937288\n",
            "  mean_inference_ms: 5.87138226619213\n",
            "  mean_raw_obs_processing_ms: 4.413463234250125\n",
            "time_since_restore: 31.859670877456665\n",
            "time_this_iter_s: 1.65834379196167\n",
            "time_total_s: 31.859670877456665\n",
            "timers:\n",
            "  learn_time_ms: 1559.872\n",
            "timestamp: 1598342379\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 3600\n",
            "training_iteration: 20\n",
            "\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.0623653451640388\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.0623653451640388\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.03396464881784294\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.03396464881784294\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.07642045984014659\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.07642045984014659\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.02714461427971312\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9196966477821733\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.02714461427971312\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9196966477821733\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.07483841419684656\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.07483841419684656\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.05094697322676439\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.05094697322676439\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "training loop = 21 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9887892005419217\n",
            "  p_0_gamma_mean: 0.9887892005419213\n",
            "  p_0_gamma_min: 0.9887892005419217\n",
            "  p_1_gamma_max: 0.9887892005419217\n",
            "  p_1_gamma_mean: 0.958545201758942\n",
            "  p_1_gamma_min: 0.906666947409507\n",
            "  p_2_gamma_max: 0.9887892005419217\n",
            "  p_2_gamma_mean: 0.9887892005419213\n",
            "  p_2_gamma_min: 0.9887892005419217\n",
            "  p_3_gamma_max: 0.9887892005419217\n",
            "  p_3_gamma_mean: 0.9618586300083465\n",
            "  p_3_gamma_min: 0.9153374406679863\n",
            "  p_4_gamma_max: 0.9887892005419217\n",
            "  p_4_gamma_mean: 0.9887892005419213\n",
            "  p_4_gamma_min: 0.9887892005419217\n",
            "  p_5_gamma_max: 0.9887892005419217\n",
            "  p_5_gamma_mean: 0.9874292252313009\n",
            "  p_5_gamma_min: 0.9869003459438371\n",
            "date: 2020-08-25_07-59-42\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 378\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_1:\n",
            "      allreduce_latency: 0.0063983069525824655\n",
            "      cur_kl_coeff: 0.2\n",
            "      cur_lr: 0.05306976377787958\n",
            "      entropy: 0.00013110635102850994\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 1.2645232053124375e-05\n",
            "      model: {}\n",
            "      policy_loss: 1.2346025970247058e-05\n",
            "      total_loss: 6.356007642216152\n",
            "      vf_explained_var: 0.3725256323814392\n",
            "      vf_loss: 6.355992847018772\n",
            "    p_4:\n",
            "      allreduce_latency: 0.006574551264444987\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.09744585181881062\n",
            "      entropy: 8.819423160911628e-16\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 1.8199600061769652e-14\n",
            "      model: {}\n",
            "      policy_loss: 2.0427008469899496e-07\n",
            "      total_loss: 29.381322860717773\n",
            "      vf_explained_var: 0.1319887936115265\n",
            "      vf_loss: 29.38132317860921\n",
            "    p_5:\n",
            "      allreduce_latency: 0.0078887939453125\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.05306976377787958\n",
            "      entropy: 0.00028798453179964173\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 8.24465640410684e-06\n",
            "      model: {}\n",
            "      policy_loss: -6.308158238728841e-07\n",
            "      total_loss: 0.4294625520706177\n",
            "      vf_explained_var: 0.04528520628809929\n",
            "      vf_loss: 0.4294615387916565\n",
            "  num_steps_sampled: 3780\n",
            "  num_steps_trained: 3780\n",
            "iterations_since_restore: 21\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 83.025\n",
            "  ram_util_percent: 31.8\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_0: 10.0\n",
            "  p_1: -1.0\n",
            "  p_2: 10.0\n",
            "  p_3: -9.0\n",
            "  p_4: 10.0\n",
            "  p_5: 10.0\n",
            "policy_reward_mean:\n",
            "  p_0: 0.0\n",
            "  p_1: -4.6\n",
            "  p_2: 9.5\n",
            "  p_3: -9.0\n",
            "  p_4: -4.456521739130435\n",
            "  p_5: 4.794117647058823\n",
            "policy_reward_min:\n",
            "  p_0: -10.0\n",
            "  p_1: -10.0\n",
            "  p_2: 9.0\n",
            "  p_3: -9.0\n",
            "  p_4: -10.0\n",
            "  p_5: 1.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.27592563625471234\n",
            "  mean_env_wait_ms: 0.09781135765963586\n",
            "  mean_inference_ms: 5.870458116712544\n",
            "  mean_raw_obs_processing_ms: 4.416583506021763\n",
            "time_since_restore: 33.46659302711487\n",
            "time_this_iter_s: 1.6069221496582031\n",
            "time_total_s: 33.46659302711487\n",
            "timers:\n",
            "  learn_time_ms: 1559.155\n",
            "timestamp: 1598342382\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 3780\n",
            "training_iteration: 21\n",
            "\n",
            "checkpoint saved at /content/gdrive/My Drive/Colab Notebooks/PBT_MARL_watered_down/chkpt/checkpoint_21/checkpoint-21\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.04989227613123104\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.04989227613123104\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9887892005419217\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.06113636787211727\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.06113636787211727\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.06113636787211727\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.06113636787211727\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.03257353713565574\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9196966477821733\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.03257353713565574\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9196966477821733\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.061136367872117264\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.061136367872117264\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.061136367872117264\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.061136367872117264\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "training loop = 22 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9887892005419217\n",
            "  p_0_gamma_mean: 0.9887892005419213\n",
            "  p_0_gamma_min: 0.9887892005419217\n",
            "  p_1_gamma_max: 0.9887892005419217\n",
            "  p_1_gamma_mean: 0.96477498818188\n",
            "  p_1_gamma_min: 0.906666947409507\n",
            "  p_2_gamma_max: 0.9887892005419217\n",
            "  p_2_gamma_mean: 0.9884492067142661\n",
            "  p_2_gamma_min: 0.9869003459438371\n",
            "  p_3_gamma_max: 0.9887892005419217\n",
            "  p_3_gamma_mean: 0.9506945647711038\n",
            "  p_3_gamma_min: 0.9153374406679863\n",
            "  p_4_gamma_max: 0.9887892005419217\n",
            "  p_4_gamma_mean: 0.9887892005419213\n",
            "  p_4_gamma_min: 0.9887892005419217\n",
            "  p_5_gamma_max: 0.9887892005419217\n",
            "  p_5_gamma_mean: 0.9870892314036458\n",
            "  p_5_gamma_min: 0.9869003459438371\n",
            "date: 2020-08-25_07-59-45\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 396\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_1:\n",
            "      allreduce_latency: 0.006885608037312825\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.04245581102230367\n",
            "      entropy: 5.205601690446334e-10\n",
            "      entropy_coeff: 0.0\n",
            "      kl: -8.226465750647606e-14\n",
            "      model: {}\n",
            "      policy_loss: 6.705522537231445e-08\n",
            "      total_loss: 10.579318841298422\n",
            "      vf_explained_var: 3.973643103449831e-08\n",
            "      vf_loss: 10.579318046569824\n",
            "    p_2:\n",
            "      allreduce_latency: 0.0058267513910929365\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.07795668145504848\n",
            "      entropy: 8.011526114968618e-09\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.016996583435684443\n",
            "      model: {}\n",
            "      policy_loss: 4.654284566640854e-07\n",
            "      total_loss: 21.932730038960774\n",
            "      vf_explained_var: 9.934107758624577e-09\n",
            "      vf_loss: 21.92932923634847\n",
            "    p_4:\n",
            "      allreduce_latency: 0.008209652370876737\n",
            "      cur_kl_coeff: 0.2\n",
            "      cur_lr: 0.0779566814550485\n",
            "      entropy: 7.40879676281509e-11\n",
            "      entropy_coeff: 0.0\n",
            "      kl: -7.457164768499835e-13\n",
            "      model: {}\n",
            "      policy_loss: 2.690487437778049e-09\n",
            "      total_loss: 6.390089458889431\n",
            "      vf_explained_var: 0.21100378036499023\n",
            "      vf_loss: 6.3900894323984785\n",
            "  num_steps_sampled: 3960\n",
            "  num_steps_trained: 3960\n",
            "iterations_since_restore: 22\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 84.46000000000001\n",
            "  ram_util_percent: 31.8\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_0: 10.0\n",
            "  p_1: -1.0\n",
            "  p_2: 10.0\n",
            "  p_3: -9.0\n",
            "  p_4: 10.0\n",
            "  p_5: 10.0\n",
            "policy_reward_mean:\n",
            "  p_0: 1.6129032258064515\n",
            "  p_1: -5.5\n",
            "  p_2: 4.0\n",
            "  p_3: -9.0\n",
            "  p_4: 1.8235294117647058\n",
            "  p_5: 2.142857142857143\n",
            "policy_reward_min:\n",
            "  p_0: -10.0\n",
            "  p_1: -10.0\n",
            "  p_2: -10.0\n",
            "  p_3: -9.0\n",
            "  p_4: -10.0\n",
            "  p_5: 1.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.27606643995587604\n",
            "  mean_env_wait_ms: 0.09632861036037027\n",
            "  mean_inference_ms: 5.869864109397001\n",
            "  mean_raw_obs_processing_ms: 4.421324165711012\n",
            "time_since_restore: 35.08969306945801\n",
            "time_this_iter_s: 1.6231000423431396\n",
            "time_total_s: 35.08969306945801\n",
            "timers:\n",
            "  learn_time_ms: 1566.224\n",
            "timestamp: 1598342385\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 3960\n",
            "training_iteration: 22\n",
            "\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.04890909429769382\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.04890909429769382\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.07336364144654073\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.07336364144654073\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.07336364144654073\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.07336364144654073\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.026058829708524595\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9196966477821733\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.026058829708524595\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9196966477821733\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.08803636973584887\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.08803636973584887\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.058690913157232585\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.058690913157232585\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "training loop = 23 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9887892005419217\n",
            "  p_0_gamma_mean: 0.9887892005419213\n",
            "  p_0_gamma_min: 0.9887892005419217\n",
            "  p_1_gamma_max: 0.9887892005419217\n",
            "  p_1_gamma_mean: 0.9792169999180595\n",
            "  p_1_gamma_min: 0.906666947409507\n",
            "  p_2_gamma_max: 0.9887892005419217\n",
            "  p_2_gamma_mean: 0.9881092128866112\n",
            "  p_2_gamma_min: 0.9869003459438371\n",
            "  p_3_gamma_max: 0.9887892005419217\n",
            "  p_3_gamma_mean: 0.9382579052743492\n",
            "  p_3_gamma_min: 0.9153374406679863\n",
            "  p_4_gamma_max: 0.9887892005419217\n",
            "  p_4_gamma_mean: 0.9884492067142661\n",
            "  p_4_gamma_min: 0.9869003459438371\n",
            "  p_5_gamma_max: 0.9869003459438371\n",
            "  p_5_gamma_mean: 0.9869003459438374\n",
            "  p_5_gamma_min: 0.9869003459438371\n",
            "date: 2020-08-25_07-59-48\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 414\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_2:\n",
            "      allreduce_latency: 0.006406545639038086\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.07642045984014659\n",
            "      entropy: 1.9560629880889988e-12\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 3.032626956269343e-12\n",
            "      model: {}\n",
            "      policy_loss: 3.2285849253336586e-08\n",
            "      total_loss: 7.913317521413167\n",
            "      vf_explained_var: 0.0\n",
            "      vf_loss: 7.913317680358887\n",
            "    p_3:\n",
            "      allreduce_latency: 0.00972445805867513\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.02714461427971312\n",
            "      entropy: 7.784551540705731e-13\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 2.3699224089186042e-19\n",
            "      model: {}\n",
            "      policy_loss: 0.0\n",
            "      total_loss: 9.703225493431091\n",
            "      vf_explained_var: -3.973643103449831e-08\n",
            "      vf_loss: 9.703225493431091\n",
            "    p_4:\n",
            "      allreduce_latency: 0.005561086866590712\n",
            "      cur_kl_coeff: 0.2\n",
            "      cur_lr: 0.07483841419684656\n",
            "      entropy: 7.46056773458391e-11\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 1.0763124370409183e-14\n",
            "      model: {}\n",
            "      policy_loss: -3.311369154188368e-09\n",
            "      total_loss: 5.715029742982653\n",
            "      vf_explained_var: 0.24890348315238953\n",
            "      vf_loss: 5.7150295045640735\n",
            "  num_steps_sampled: 4140\n",
            "  num_steps_trained: 4140\n",
            "iterations_since_restore: 23\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 84.05\n",
            "  ram_util_percent: 31.8\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_0: 10.0\n",
            "  p_1: -1.0\n",
            "  p_2: 10.0\n",
            "  p_3: -9.0\n",
            "  p_4: 10.0\n",
            "  p_5: 1.0\n",
            "policy_reward_mean:\n",
            "  p_0: 7.142857142857143\n",
            "  p_1: -5.5\n",
            "  p_2: -0.9705882352941176\n",
            "  p_3: -9.48\n",
            "  p_4: 4.545454545454546\n",
            "  p_5: 1.0\n",
            "policy_reward_min:\n",
            "  p_0: -10.0\n",
            "  p_1: -10.0\n",
            "  p_2: -10.0\n",
            "  p_3: -10.0\n",
            "  p_4: -10.0\n",
            "  p_5: 1.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.27592203833276807\n",
            "  mean_env_wait_ms: 0.09502051743450383\n",
            "  mean_inference_ms: 5.868533438874373\n",
            "  mean_raw_obs_processing_ms: 4.425782156328892\n",
            "time_since_restore: 36.66139268875122\n",
            "time_this_iter_s: 1.571699619293213\n",
            "time_total_s: 36.66139268875122\n",
            "timers:\n",
            "  learn_time_ms: 1560.511\n",
            "timestamp: 1598342388\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 4140\n",
            "training_iteration: 23\n",
            "\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.03912727543815506\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.03912727543815506\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.027720962696838\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9130923430584088\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.027720962696838\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9130923430584088\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.020847063766819676\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9196966477821733\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.020847063766819676\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9196966477821733\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.031270595650229514\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9196966477821733\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.031270595650229514\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9196966477821733\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.0704290957886791\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.0704290957886791\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.05634327663094329\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.05634327663094329\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "training loop = 24 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9887892005419217\n",
            "  p_0_gamma_mean: 0.9884492067142661\n",
            "  p_0_gamma_min: 0.9869003459438371\n",
            "  p_1_gamma_max: 0.9887892005419217\n",
            "  p_1_gamma_mean: 0.9870892314036458\n",
            "  p_1_gamma_min: 0.9869003459438371\n",
            "  p_2_gamma_max: 0.9887892005419217\n",
            "  p_2_gamma_mean: 0.9877692190589561\n",
            "  p_2_gamma_min: 0.9869003459438371\n",
            "  p_3_gamma_max: 0.9887892005419217\n",
            "  p_3_gamma_mean: 0.9258212457775943\n",
            "  p_3_gamma_min: 0.9153374406679863\n",
            "  p_4_gamma_max: 0.9887892005419217\n",
            "  p_4_gamma_mean: 0.9881092128866112\n",
            "  p_4_gamma_min: 0.9869003459438371\n",
            "  p_5_gamma_max: 0.9869003459438371\n",
            "  p_5_gamma_mean: 0.9869003459438374\n",
            "  p_5_gamma_min: 0.9869003459438371\n",
            "date: 2020-08-25_07-59-51\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 432\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_0:\n",
            "      allreduce_latency: 0.007336060206095378\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.04989227613123104\n",
            "      entropy: 6.679702649228025e-18\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.0\n",
            "      model: {}\n",
            "      policy_loss: 7.326404253641765e-08\n",
            "      total_loss: 8.128409544626871\n",
            "      vf_explained_var: -3.973643103449831e-08\n",
            "      vf_loss: 8.128409186999003\n",
            "    p_3:\n",
            "      allreduce_latency: 0.008790890375773111\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.03257353713565574\n",
            "      entropy: 7.784633217269391e-13\n",
            "      entropy_coeff: 0.0\n",
            "      kl: -3.1598603560730063e-19\n",
            "      model: {}\n",
            "      policy_loss: -6.953875223795573e-08\n",
            "      total_loss: 8.585676987965902\n",
            "      vf_explained_var: 1.9868215517249155e-08\n",
            "      vf_loss: 8.5856773853302\n",
            "    p_4:\n",
            "      allreduce_latency: 0.006865051057603624\n",
            "      cur_kl_coeff: 0.2\n",
            "      cur_lr: 0.061136367872117264\n",
            "      entropy: 2.522874810296906e-11\n",
            "      entropy_coeff: 0.0\n",
            "      kl: -6.171924043868806e-17\n",
            "      model: {}\n",
            "      policy_loss: -1.0234200292163425e-07\n",
            "      total_loss: 22.52471998002794\n",
            "      vf_explained_var: 0.5705533623695374\n",
            "      vf_loss: 22.524720191955566\n",
            "  num_steps_sampled: 4320\n",
            "  num_steps_trained: 4320\n",
            "iterations_since_restore: 24\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 83.45\n",
            "  ram_util_percent: 31.875\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_0: 10.0\n",
            "  p_1: -1.0\n",
            "  p_2: 9.0\n",
            "  p_3: -9.0\n",
            "  p_4: 10.0\n",
            "  p_5: 1.0\n",
            "policy_reward_mean:\n",
            "  p_0: 10.0\n",
            "  p_1: -5.5\n",
            "  p_2: -7.285714285714286\n",
            "  p_3: -9.857142857142858\n",
            "  p_4: 3.670886075949367\n",
            "  p_5: 1.0\n",
            "policy_reward_min:\n",
            "  p_0: 10.0\n",
            "  p_1: -10.0\n",
            "  p_2: -10.0\n",
            "  p_3: -10.0\n",
            "  p_4: -10.0\n",
            "  p_5: 1.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.27537268684562494\n",
            "  mean_env_wait_ms: 0.09407442445869808\n",
            "  mean_inference_ms: 5.871892139641727\n",
            "  mean_raw_obs_processing_ms: 4.4305307741454945\n",
            "time_since_restore: 38.301913022994995\n",
            "time_this_iter_s: 1.6405203342437744\n",
            "time_total_s: 38.301913022994995\n",
            "timers:\n",
            "  learn_time_ms: 1564.278\n",
            "timestamp: 1598342391\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 4320\n",
            "training_iteration: 24\n",
            "\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.04507462130475463\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.04507462130475463\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.04507462130475463\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.04507462130475463\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.01667765101345574\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9196966477821733\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.01667765101345574\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9196966477821733\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.06761193195713194\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.06761193195713194\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.08451491494641493\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.08451491494641493\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.05408954556570556\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.05408954556570556\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "training loop = 25 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9887892005419217\n",
            "  p_0_gamma_mean: 0.9881092128866112\n",
            "  p_0_gamma_min: 0.9869003459438371\n",
            "  p_1_gamma_max: 0.9869003459438371\n",
            "  p_1_gamma_mean: 0.9736149054244602\n",
            "  p_1_gamma_min: 0.9130923430584088\n",
            "  p_2_gamma_max: 0.9887892005419217\n",
            "  p_2_gamma_mean: 0.9753325595622013\n",
            "  p_2_gamma_min: 0.9196966477821732\n",
            "  p_3_gamma_max: 0.9196966477821732\n",
            "  p_3_gamma_mean: 0.9192607270707543\n",
            "  p_3_gamma_min: 0.9153374406679863\n",
            "  p_4_gamma_max: 0.9887892005419217\n",
            "  p_4_gamma_mean: 0.9877692190589561\n",
            "  p_4_gamma_min: 0.9869003459438371\n",
            "  p_5_gamma_max: 0.9869003459438371\n",
            "  p_5_gamma_mean: 0.9869003459438374\n",
            "  p_5_gamma_min: 0.9869003459438371\n",
            "date: 2020-08-25_07-59-53\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 450\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_0:\n",
            "      allreduce_latency: 0.006911940044826931\n",
            "      cur_kl_coeff: 0.2\n",
            "      cur_lr: 0.04890909429769382\n",
            "      entropy: 7.906759832591008e-18\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.0\n",
            "      model: {}\n",
            "      policy_loss: -1.334895690282186e-08\n",
            "      total_loss: 46.75536764992608\n",
            "      vf_explained_var: 0.25391271710395813\n",
            "      vf_loss: 46.75536812676324\n",
            "    p_2:\n",
            "      allreduce_latency: 0.007281382878621419\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.07336364144654073\n",
            "      entropy: 0.013343807727020854\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.8874899794658025\n",
            "      model: {}\n",
            "      policy_loss: 0.01805512731273969\n",
            "      total_loss: 3.93133274714152\n",
            "      vf_explained_var: -0.2711256742477417\n",
            "      vf_loss: 3.7357798417409263\n",
            "    p_4:\n",
            "      allreduce_latency: 0.006895383199055989\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.08803636973584887\n",
            "      entropy: 5.421296549699972e-13\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 1.1510288017039564e-19\n",
            "      model: {}\n",
            "      policy_loss: -8.319814999898274e-08\n",
            "      total_loss: 9.36352570851644\n",
            "      vf_explained_var: -0.20752370357513428\n",
            "      vf_loss: 9.363526344299316\n",
            "  num_steps_sampled: 4500\n",
            "  num_steps_trained: 4500\n",
            "iterations_since_restore: 25\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 83.25\n",
            "  ram_util_percent: 31.9\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_0: 10.0\n",
            "  p_1: -1.0\n",
            "  p_2: 10.0\n",
            "  p_3: -10.0\n",
            "  p_4: 10.0\n",
            "  p_5: 1.0\n",
            "policy_reward_mean:\n",
            "  p_0: 3.4242424242424243\n",
            "  p_1: -6.225806451612903\n",
            "  p_2: -2.7666666666666666\n",
            "  p_3: -10.0\n",
            "  p_4: 4.4\n",
            "  p_5: 1.0\n",
            "policy_reward_min:\n",
            "  p_0: -10.0\n",
            "  p_1: -10.0\n",
            "  p_2: -10.0\n",
            "  p_3: -10.0\n",
            "  p_4: -10.0\n",
            "  p_5: 1.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.2750268670836809\n",
            "  mean_env_wait_ms: 0.09344818600359059\n",
            "  mean_inference_ms: 5.878674402409579\n",
            "  mean_raw_obs_processing_ms: 4.434411037885391\n",
            "time_since_restore: 39.90886425971985\n",
            "time_this_iter_s: 1.6069512367248535\n",
            "time_total_s: 39.90886425971985\n",
            "timers:\n",
            "  learn_time_ms: 1568.053\n",
            "timestamp: 1598342393\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 4500\n",
            "training_iteration: 25\n",
            "\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.10141789793569791\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.10141789793569791\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.054089545565705555\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.054089545565705555\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.054089545565705555\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.054089545565705555\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.08113431834855833\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.08113431834855833\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.10141789793569791\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.10141789793569791\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.06490745467884666\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.06490745467884666\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "training loop = 26 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9887892005419217\n",
            "  p_0_gamma_mean: 0.9877692190589561\n",
            "  p_0_gamma_min: 0.9869003459438371\n",
            "  p_1_gamma_max: 0.9869003459438371\n",
            "  p_1_gamma_mean: 0.9736149054244602\n",
            "  p_1_gamma_min: 0.9130923430584088\n",
            "  p_2_gamma_max: 0.9887892005419217\n",
            "  p_2_gamma_mean: 0.9628959000654467\n",
            "  p_2_gamma_min: 0.9196966477821732\n",
            "  p_3_gamma_max: 0.9869003459438371\n",
            "  p_3_gamma_mean: 0.9317933134512725\n",
            "  p_3_gamma_min: 0.9196966477821732\n",
            "  p_4_gamma_max: 0.9887892005419217\n",
            "  p_4_gamma_mean: 0.9874292252313009\n",
            "  p_4_gamma_min: 0.9869003459438371\n",
            "  p_5_gamma_max: 0.9869003459438371\n",
            "  p_5_gamma_mean: 0.9869003459438374\n",
            "  p_5_gamma_min: 0.9869003459438371\n",
            "date: 2020-08-25_07-59-56\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 468\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_0:\n",
            "      allreduce_latency: 0.008021990458170572\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.03912727543815506\n",
            "      entropy: 8.324481262664525e-18\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.0\n",
            "      model: {}\n",
            "      policy_loss: -7.450580596923828e-08\n",
            "      total_loss: 25.493310292561848\n",
            "      vf_explained_var: -0.7319968342781067\n",
            "      vf_loss: 25.493311564127605\n",
            "    p_1:\n",
            "      allreduce_latency: 0.006633838017781575\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.027720962696837995\n",
            "      entropy: 5.220845885241706e-10\n",
            "      entropy_coeff: 0.0\n",
            "      kl: -4.9381076100845203e-14\n",
            "      model: {}\n",
            "      policy_loss: 3.042320410410563e-08\n",
            "      total_loss: 11.423389911651611\n",
            "      vf_explained_var: 3.973643103449831e-08\n",
            "      vf_loss: 11.42339007059733\n",
            "    p_2:\n",
            "      allreduce_latency: 0.004907528559366862\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.020847063766819676\n",
            "      entropy: 0.07043848171209295\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 1.6909002661705017\n",
            "      model: {}\n",
            "      policy_loss: 0.32290107011795044\n",
            "      total_loss: 11.000193277994791\n",
            "      vf_explained_var: 0.08588818460702896\n",
            "      vf_loss: 10.339112281799316\n",
            "    p_4:\n",
            "      allreduce_latency: 0.00667874018351237\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.0704290957886791\n",
            "      entropy: 7.460052598810367e-11\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 1.2404077348418424e-20\n",
            "      model: {}\n",
            "      policy_loss: -1.4901161193847656e-08\n",
            "      total_loss: 16.35226837793986\n",
            "      vf_explained_var: -0.35211074352264404\n",
            "      vf_loss: 16.35226885477702\n",
            "  num_steps_sampled: 4680\n",
            "  num_steps_trained: 4680\n",
            "iterations_since_restore: 26\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 83.675\n",
            "  ram_util_percent: 31.9\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_0: 10.0\n",
            "  p_1: -1.0\n",
            "  p_2: 10.0\n",
            "  p_3: -10.0\n",
            "  p_4: 10.0\n",
            "  p_5: 1.0\n",
            "policy_reward_mean:\n",
            "  p_0: 1.75\n",
            "  p_1: -9.035714285714286\n",
            "  p_2: -1.75\n",
            "  p_3: -10.0\n",
            "  p_4: 5.443037974683544\n",
            "  p_5: 1.0\n",
            "policy_reward_min:\n",
            "  p_0: -10.0\n",
            "  p_1: -10.0\n",
            "  p_2: -10.0\n",
            "  p_3: -10.0\n",
            "  p_4: -10.0\n",
            "  p_5: 1.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.2753554740914389\n",
            "  mean_env_wait_ms: 0.09289422611579255\n",
            "  mean_inference_ms: 5.886011704760412\n",
            "  mean_raw_obs_processing_ms: 4.436904704872297\n",
            "time_since_restore: 41.51412916183472\n",
            "time_this_iter_s: 1.6052649021148682\n",
            "time_total_s: 41.51412916183472\n",
            "timers:\n",
            "  learn_time_ms: 1568.622\n",
            "timestamp: 1598342396\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 4680\n",
            "training_iteration: 26\n",
            "\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.08113431834855833\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.08113431834855833\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.043271636452564446\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.043271636452564446\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.043271636452564446\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.043271636452564446\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.06490745467884666\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.06490745467884666\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.1217014775228375\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.1217014775228375\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.14604177302740498\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.14604177302740498\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "training loop = 27 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9887892005419217\n",
            "  p_0_gamma_mean: 0.9874292252313009\n",
            "  p_0_gamma_min: 0.9869003459438371\n",
            "  p_1_gamma_max: 0.9869003459438371\n",
            "  p_1_gamma_mean: 0.9736149054244602\n",
            "  p_1_gamma_min: 0.9130923430584088\n",
            "  p_2_gamma_max: 0.9869003459438371\n",
            "  p_2_gamma_mean: 0.9627070146056381\n",
            "  p_2_gamma_min: 0.9196966477821732\n",
            "  p_3_gamma_max: 0.9869003459438371\n",
            "  p_3_gamma_mean: 0.9438899791203721\n",
            "  p_3_gamma_min: 0.9196966477821732\n",
            "  p_4_gamma_max: 0.9887892005419217\n",
            "  p_4_gamma_mean: 0.9870892314036458\n",
            "  p_4_gamma_min: 0.9869003459438371\n",
            "  p_5_gamma_max: 0.9869003459438371\n",
            "  p_5_gamma_mean: 0.9869003459438374\n",
            "  p_5_gamma_min: 0.9869003459438371\n",
            "date: 2020-08-25_07-59-59\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 486\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_1:\n",
            "      allreduce_latency: 0.006793498992919922\n",
            "      cur_kl_coeff: 0.2\n",
            "      cur_lr: 0.04507462130475463\n",
            "      entropy: 0.000998872901416487\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 3.309033976014083e-05\n",
            "      model: {}\n",
            "      policy_loss: 8.48015770316124e-06\n",
            "      total_loss: 21.09298716651069\n",
            "      vf_explained_var: 0.4130457639694214\n",
            "      vf_loss: 21.09297360314263\n",
            "    p_2:\n",
            "      allreduce_latency: 0.007401903470357259\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.01667765101345574\n",
            "      entropy: 2.6656871890888095e-05\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 2.1298110435876336e-06\n",
            "      model: {}\n",
            "      policy_loss: 1.1809170246124268e-06\n",
            "      total_loss: 10.086026906967163\n",
            "      vf_explained_var: 0.16240261495113373\n",
            "      vf_loss: 10.08602507909139\n",
            "    p_4:\n",
            "      allreduce_latency: 0.005365610122680664\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.08451491494641493\n",
            "      entropy: 7.460052483162134e-11\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.0\n",
            "      model: {}\n",
            "      policy_loss: -2.2351741790771484e-08\n",
            "      total_loss: 7.4542460441589355\n",
            "      vf_explained_var: 0.2292913794517517\n",
            "      vf_loss: 7.454246362050374\n",
            "  num_steps_sampled: 4860\n",
            "  num_steps_trained: 4860\n",
            "iterations_since_restore: 27\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 83.95\n",
            "  ram_util_percent: 31.9\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_0: 10.0\n",
            "  p_1: 9.0\n",
            "  p_2: 10.0\n",
            "  p_3: -10.0\n",
            "  p_4: 10.0\n",
            "policy_reward_mean:\n",
            "  p_0: 1.75\n",
            "  p_1: -3.090909090909091\n",
            "  p_2: -2.813953488372093\n",
            "  p_3: -10.0\n",
            "  p_4: 4.857142857142857\n",
            "policy_reward_min:\n",
            "  p_0: -10.0\n",
            "  p_1: -10.0\n",
            "  p_2: -10.0\n",
            "  p_3: -10.0\n",
            "  p_4: -10.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.27572978543955246\n",
            "  mean_env_wait_ms: 0.09247316541846472\n",
            "  mean_inference_ms: 5.8928683573197\n",
            "  mean_raw_obs_processing_ms: 4.438547036319059\n",
            "time_since_restore: 43.06725859642029\n",
            "time_this_iter_s: 1.5531294345855713\n",
            "time_total_s: 43.06725859642029\n",
            "timers:\n",
            "  learn_time_ms: 1571.356\n",
            "timestamp: 1598342399\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 4860\n",
            "training_iteration: 27\n",
            "\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.051925963743077334\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.051925963743077334\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.051925963743077334\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.051925963743077334\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.11683341842192399\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.11683341842192399\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.077888945614616\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.077888945614616\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.09736118201827\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.09736118201827\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.11683341842192399\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.11683341842192399\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "training loop = 28 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9887892005419217\n",
            "  p_0_gamma_mean: 0.9870892314036458\n",
            "  p_0_gamma_min: 0.9869003459438371\n",
            "  p_1_gamma_max: 0.9869003459438371\n",
            "  p_1_gamma_mean: 0.9736149054244602\n",
            "  p_1_gamma_min: 0.9130923430584088\n",
            "  p_2_gamma_max: 0.9869003459438371\n",
            "  p_2_gamma_mean: 0.962707014605638\n",
            "  p_2_gamma_min: 0.9196966477821732\n",
            "  p_3_gamma_max: 0.9869003459438371\n",
            "  p_3_gamma_mean: 0.9559866447894717\n",
            "  p_3_gamma_min: 0.9196966477821732\n",
            "  p_4_gamma_max: 0.9869003459438371\n",
            "  p_4_gamma_mean: 0.9869003459438374\n",
            "  p_4_gamma_min: 0.9869003459438371\n",
            "  p_5_gamma_max: 0.9869003459438371\n",
            "  p_5_gamma_mean: 0.9869003459438374\n",
            "  p_5_gamma_min: 0.9869003459438371\n",
            "date: 2020-08-25_08-00-02\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 504\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_0:\n",
            "      allreduce_latency: 0.006501833597819011\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.1014178979356979\n",
            "      entropy: 8.794417595770361e-18\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.0\n",
            "      model: {}\n",
            "      policy_loss: 1.601874828338623e-07\n",
            "      total_loss: 16.490875085194904\n",
            "      vf_explained_var: -0.2809697985649109\n",
            "      vf_loss: 16.490874767303467\n",
            "    p_1:\n",
            "      allreduce_latency: 0.00476225217183431\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.054089545565705555\n",
            "      entropy: 0.003411729820072651\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 9.473929094383493e-05\n",
            "      model: {}\n",
            "      policy_loss: 7.330191632111867e-05\n",
            "      total_loss: 8.957914511362711\n",
            "      vf_explained_var: -0.22176502645015717\n",
            "      vf_loss: 8.957822640736898\n",
            "    p_2:\n",
            "      allreduce_latency: 0.0068980058034261065\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.054089545565705555\n",
            "      entropy: 0.00011673177020081009\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 1.7277162290459576e-06\n",
            "      model: {}\n",
            "      policy_loss: -4.0729840596516925e-07\n",
            "      total_loss: 6.262241840362549\n",
            "      vf_explained_var: 0.18594439327716827\n",
            "      vf_loss: 6.262242158253987\n",
            "    p_3:\n",
            "      allreduce_latency: 0.005611697832743327\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.08113431834855833\n",
            "      entropy: 9.654424431490098e-13\n",
            "      entropy_coeff: 0.0\n",
            "      kl: -1.7288343851468975e-19\n",
            "      model: {}\n",
            "      policy_loss: 3.663202126820882e-07\n",
            "      total_loss: 25.8790602684021\n",
            "      vf_explained_var: 0.08852246403694153\n",
            "      vf_loss: 25.8790598710378\n",
            "  num_steps_sampled: 5040\n",
            "  num_steps_trained: 5040\n",
            "iterations_since_restore: 28\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 84.075\n",
            "  ram_util_percent: 31.9\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_0: 10.0\n",
            "  p_1: 9.0\n",
            "  p_2: 10.0\n",
            "  p_3: 10.0\n",
            "  p_4: 10.0\n",
            "policy_reward_mean:\n",
            "  p_0: -1.1875\n",
            "  p_1: -0.5\n",
            "  p_2: -1.9230769230769231\n",
            "  p_3: -0.4\n",
            "  p_4: 3.076923076923077\n",
            "policy_reward_min:\n",
            "  p_0: -10.0\n",
            "  p_1: -10.0\n",
            "  p_2: -10.0\n",
            "  p_3: -10.0\n",
            "  p_4: -10.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.2762135627582085\n",
            "  mean_env_wait_ms: 0.09210660970759728\n",
            "  mean_inference_ms: 5.900496425622837\n",
            "  mean_raw_obs_processing_ms: 4.440294854846866\n",
            "time_since_restore: 44.6428108215332\n",
            "time_this_iter_s: 1.575552225112915\n",
            "time_total_s: 44.6428108215332\n",
            "timers:\n",
            "  learn_time_ms: 1580.994\n",
            "timestamp: 1598342402\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 5040\n",
            "training_iteration: 28\n",
            "\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.04154077099446187\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.04154077099446187\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.11683341842192399\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.11683341842192399\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.11683341842192399\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.11683341842192399\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.09346673473753919\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.09346673473753919\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.11683341842192399\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.11683341842192399\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.049848925193354245\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.049848925193354245\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "training loop = 29 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9869003459438371\n",
            "  p_0_gamma_mean: 0.9869003459438374\n",
            "  p_0_gamma_min: 0.9869003459438371\n",
            "  p_1_gamma_max: 0.9869003459438371\n",
            "  p_1_gamma_mean: 0.9736149054244602\n",
            "  p_1_gamma_min: 0.9130923430584088\n",
            "  p_2_gamma_max: 0.9869003459438371\n",
            "  p_2_gamma_mean: 0.9627070146056381\n",
            "  p_2_gamma_min: 0.9196966477821732\n",
            "  p_3_gamma_max: 0.9869003459438371\n",
            "  p_3_gamma_mean: 0.9680833104585713\n",
            "  p_3_gamma_min: 0.9196966477821732\n",
            "  p_4_gamma_max: 0.9869003459438371\n",
            "  p_4_gamma_mean: 0.9869003459438374\n",
            "  p_4_gamma_min: 0.9869003459438371\n",
            "  p_5_gamma_max: 0.9869003459438371\n",
            "  p_5_gamma_mean: 0.9869003459438374\n",
            "  p_5_gamma_min: 0.9869003459438371\n",
            "date: 2020-08-25_08-00-05\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 522\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_0:\n",
            "      allreduce_latency: 0.00603440072801378\n",
            "      cur_kl_coeff: 0.2\n",
            "      cur_lr: 0.08113431834855833\n",
            "      entropy: 8.79441764172484e-18\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.0\n",
            "      model: {}\n",
            "      policy_loss: 6.933179166581896e-09\n",
            "      total_loss: 6.334806760152181\n",
            "      vf_explained_var: 0.17022749781608582\n",
            "      vf_loss: 6.334806892606947\n",
            "    p_3:\n",
            "      allreduce_latency: 0.006044308344523112\n",
            "      cur_kl_coeff: 0.2\n",
            "      cur_lr: 0.06490745467884666\n",
            "      entropy: 9.654424250789735e-13\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.0\n",
            "      model: {}\n",
            "      policy_loss: 1.4901161193847656e-08\n",
            "      total_loss: 6.936257627275255\n",
            "      vf_explained_var: 0.1840481162071228\n",
            "      vf_loss: 6.936257680257161\n",
            "  num_steps_sampled: 5220\n",
            "  num_steps_trained: 5220\n",
            "iterations_since_restore: 29\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 83.975\n",
            "  ram_util_percent: 31.9\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_0: 10.0\n",
            "  p_1: 9.0\n",
            "  p_2: 10.0\n",
            "  p_3: 10.0\n",
            "  p_4: 10.0\n",
            "policy_reward_mean:\n",
            "  p_0: -4.704918032786885\n",
            "  p_1: -0.5\n",
            "  p_2: -1.25\n",
            "  p_3: 8.181818181818182\n",
            "  p_4: 2.3529411764705883\n",
            "policy_reward_min:\n",
            "  p_0: -10.0\n",
            "  p_1: -10.0\n",
            "  p_2: -9.0\n",
            "  p_3: -10.0\n",
            "  p_4: -10.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.2767744576322796\n",
            "  mean_env_wait_ms: 0.09172530766891741\n",
            "  mean_inference_ms: 5.910149822671271\n",
            "  mean_raw_obs_processing_ms: 4.445434788590013\n",
            "time_since_restore: 46.27090311050415\n",
            "time_this_iter_s: 1.6280922889709473\n",
            "time_total_s: 46.27090311050415\n",
            "timers:\n",
            "  learn_time_ms: 1587.04\n",
            "timestamp: 1598342405\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 5220\n",
            "training_iteration: 29\n",
            "\n",
            "trainer.train() result: <ray.rllib.agents.trainer_template.Custom_Trainer object at 0x7fda523aa2b0> -> 18 episodes\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.05981871023202509\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.05981871023202509\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.1402001021063088\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.1402001021063088\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.09346673473753919\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.09346673473753919\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.07477338779003136\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.07477338779003136\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.09346673473753919\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.09346673473753919\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_lr_schedule, lr=0.05697576659707402\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.9869003459438371\n",
            "\u001b[2m\u001b[36m(pid=2466)\u001b[0m update_gamma, gamma=0.911047690906334\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_lr_schedule, lr=0.05697576659707402\n",
            "\u001b[2m\u001b[36m(pid=2555)\u001b[0m update_gamma, gamma=0.911047690906334\n",
            "training loop = 30 of 30\n",
            "callback_ok: true\n",
            "custom_metrics:\n",
            "  p_0_gamma_max: 0.9869003459438371\n",
            "  p_0_gamma_mean: 0.9869003459438374\n",
            "  p_0_gamma_min: 0.9869003459438371\n",
            "  p_1_gamma_max: 0.9869003459438371\n",
            "  p_1_gamma_mean: 0.9795195456552945\n",
            "  p_1_gamma_min: 0.9130923430584088\n",
            "  p_2_gamma_max: 0.9869003459438371\n",
            "  p_2_gamma_mean: 0.9680833104585713\n",
            "  p_2_gamma_min: 0.9196966477821732\n",
            "  p_3_gamma_max: 0.9869003459438371\n",
            "  p_3_gamma_mean: 0.9801799761276709\n",
            "  p_3_gamma_min: 0.9196966477821732\n",
            "  p_4_gamma_max: 0.9869003459438371\n",
            "  p_4_gamma_mean: 0.9869003459438374\n",
            "  p_4_gamma_min: 0.9869003459438371\n",
            "  p_5_gamma_max: 0.9869003459438371\n",
            "  p_5_gamma_mean: 0.9869003459438374\n",
            "  p_5_gamma_min: 0.9869003459438371\n",
            "date: 2020-08-25_08-00-08\n",
            "done: false\n",
            "episode_len_mean: 10.0\n",
            "episode_reward_max: 0.0\n",
            "episode_reward_mean: 0.0\n",
            "episode_reward_min: 0.0\n",
            "episodes_this_iter: 18\n",
            "episodes_total: 540\n",
            "experiment_id: 992f5ff3caa745479dead671b835b523\n",
            "hostname: f8dde82dfc4b\n",
            "info:\n",
            "  learner:\n",
            "    p_0:\n",
            "      allreduce_latency: 0.00482940673828125\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.051925963743077334\n",
            "      entropy: 8.794418009360667e-18\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.0\n",
            "      model: {}\n",
            "      policy_loss: 4.967053731282552e-08\n",
            "      total_loss: 5.257412274678548\n",
            "      vf_explained_var: 0.2852210998535156\n",
            "      vf_loss: 5.257412115732829\n",
            "    p_1:\n",
            "      allreduce_latency: 0.008354425430297852\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.051925963743077334\n",
            "      entropy: 5.258141884863695e-10\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 1.1739323705836635e-12\n",
            "      model: {}\n",
            "      policy_loss: -1.708976924419403e-07\n",
            "      total_loss: 115.13106854756673\n",
            "      vf_explained_var: 0.0\n",
            "      vf_loss: 115.13106473286946\n",
            "    p_3:\n",
            "      allreduce_latency: 0.008147478103637695\n",
            "      cur_kl_coeff: 0.20000000000000004\n",
            "      cur_lr: 0.077888945614616\n",
            "      entropy: 9.65442461219046e-13\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.0\n",
            "      model: {}\n",
            "      policy_loss: 9.934107462565104e-09\n",
            "      total_loss: 12.384275595347086\n",
            "      vf_explained_var: 0.21704918146133423\n",
            "      vf_loss: 12.384275277455648\n",
            "    p_4:\n",
            "      allreduce_latency: 0.008878191312154135\n",
            "      cur_kl_coeff: 0.19999999999999998\n",
            "      cur_lr: 0.09736118201827\n",
            "      entropy: 7.460052483162134e-11\n",
            "      entropy_coeff: 0.0\n",
            "      kl: 0.0\n",
            "      model: {}\n",
            "      policy_loss: -4.718701044718424e-08\n",
            "      total_loss: 7.143646438916524\n",
            "      vf_explained_var: 0.2064557522535324\n",
            "      vf_loss: 7.143646478652954\n",
            "  num_steps_sampled: 5400\n",
            "  num_steps_trained: 5400\n",
            "iterations_since_restore: 30\n",
            "node_ip: 172.28.0.2\n",
            "num_healthy_workers: 2\n",
            "off_policy_estimator: {}\n",
            "perf:\n",
            "  cpu_util_percent: 83.4\n",
            "  ram_util_percent: 31.95\n",
            "pid: 2356\n",
            "policy_reward_max:\n",
            "  p_0: 10.0\n",
            "  p_1: 9.0\n",
            "  p_2: 10.0\n",
            "  p_3: 10.0\n",
            "  p_4: 10.0\n",
            "policy_reward_mean:\n",
            "  p_0: -7.769230769230769\n",
            "  p_1: -2.875\n",
            "  p_2: -2.838709677419355\n",
            "  p_3: 10.0\n",
            "  p_4: 8.181818181818182\n",
            "policy_reward_min:\n",
            "  p_0: -10.0\n",
            "  p_1: -10.0\n",
            "  p_2: -9.0\n",
            "  p_3: 10.0\n",
            "  p_4: -10.0\n",
            "sampler_perf:\n",
            "  mean_action_processing_ms: 0.27712533092476394\n",
            "  mean_env_wait_ms: 0.09119650047142602\n",
            "  mean_inference_ms: 5.913266123777719\n",
            "  mean_raw_obs_processing_ms: 4.450068135994169\n",
            "time_since_restore: 47.827887773513794\n",
            "time_this_iter_s: 1.5569846630096436\n",
            "time_total_s: 47.827887773513794\n",
            "timers:\n",
            "  learn_time_ms: 1576.913\n",
            "timestamp: 1598342408\n",
            "timesteps_since_restore: 0\n",
            "timesteps_total: 5400\n",
            "training_iteration: 30\n",
            "\n",
            "checkpoint saved at /content/gdrive/My Drive/Colab Notebooks/PBT_MARL_watered_down/chkpt/checkpoint_30/checkpoint-30\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}