RSI limited with RL.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2ebf1053",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Balance: 100000, Shares Held: 0\n",
      "Balance: 99340.91382029, Shares Held: 70\n"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "from gym import spaces\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "class TradingEnv(gym.Env):\n",
    "    def __init__(self):\n",
    "        super(TradingEnv, self).__init__()\n",
    "\n",
    "        # Load the CSV file with trading data\n",
    "        self.data = pd.read_csv(\"C:\\\\Users\\\\Yash\\\\Downloads\\\\tata.csv\")\n",
    "\n",
    "        # Define the action space\n",
    "        self.action_space = spaces.MultiDiscrete([3, 100])  # 3 actions: 0 (sell), 1 (hold), 2 (buy), with quantity 0-100\n",
    "\n",
    "        # Define the observation space\n",
    "        self.observation_space = spaces.Box(low=0, high=1, shape=(45, len(self.data.columns) - 1), dtype=np.float32)\n",
    "\n",
    "        # Initialize other parameters\n",
    "        self.balance = 100000\n",
    "        self.shares_held = 0\n",
    "        self.current_step = 45  # Start from the 45th candle\n",
    "        self.reward_range = (-np.inf, np.inf)\n",
    "        self.current_charge = 0\n",
    "        self.profit = 0\n",
    "\n",
    "    def reset(self):\n",
    "        # Reset the environment to the initial state\n",
    "        self.balance = 100000\n",
    "        self.shares_held = 0\n",
    "        self.current_step = 45  # Start from the 45th candle\n",
    "        self.current_charge = 0\n",
    "        self.profit = 0\n",
    "\n",
    "        # Get the initial observation\n",
    "        observation = self.get_observation()\n",
    "        return observation\n",
    "\n",
    "    def step(self, action):\n",
    "        # Execute the selected action\n",
    "        if action[0] == 0:  # Sell\n",
    "            if self.shares_held >= action[1]:\n",
    "                self.balance += action[1] * self.data.iloc[self.current_step]['Close']\n",
    "                self.shares_held -= action[1]\n",
    "                self.current_charge += 20\n",
    "            else:\n",
    "                return self.get_observation(), -10, True, {}\n",
    "\n",
    "        elif action[0] == 2:  # Buy\n",
    "            if self.balance >= action[1] * self.data.iloc[self.current_step]['Close']:\n",
    "                self.balance -= action[1] * self.data.iloc[self.current_step]['Close']\n",
    "                self.shares_held += action[1]\n",
    "                self.current_charge += 100\n",
    "\n",
    "        # Deduct charge every 90th candle\n",
    "        if self.current_step % 90 == 0:\n",
    "            self.balance -= 100\n",
    "\n",
    "        # Check RSI condition\n",
    "        if self.data.iloc[self.current_step]['RSI'] > 45 and action[0] == 2:  # Don't buy if RSI > 45\n",
    "            return self.get_observation(), -5, True, {}\n",
    "\n",
    "        # Update the current step\n",
    "        self.current_step += 1\n",
    "\n",
    "        # Calculate the reward\n",
    "        reward = self.calculate_reward()\n",
    "\n",
    "        # Check if the episode is done\n",
    "        done = self.current_step == len(self.data) - 1 or self.balance <= 0\n",
    "\n",
    "        return self.get_observation(), reward, done, {}\n",
    "\n",
    "    def get_observation(self):\n",
    "        # Get the current observation (past 45 candle data)\n",
    "        observation = self.data.iloc[self.current_step - 45:self.current_step, :-1].values\n",
    "        return observation\n",
    "\n",
    "    def calculate_reward(self):\n",
    "        # Calculate the reward based on profit/loss\n",
    "        current_value = self.balance + self.shares_held * self.data.iloc[self.current_step]['Close']\n",
    "        reward = current_value - 100000 - self.current_charge\n",
    "        return reward\n",
    "\n",
    "    def render(self):\n",
    "        # Display current balance and shares held\n",
    "        print(f\"Balance: {self.balance}, Shares Held: {self.shares_held}\")\n",
    "\n",
    "# Test the environment\n",
    "if __name__ == \"__main__\":\n",
    "    env = TradingEnv()\n",
    "    observation = env.reset()\n",
    "    done = False\n",
    "    while not done:\n",
    "        action = env.action_space.sample()  # Random action for testing\n",
    "        observation, reward, done, _ = env.step(action)\n",
    "        env.render()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9b97deab",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'MultiDiscrete' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[8], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[43mMultiDiscrete\u001b[49m([\u001b[38;5;241m3\u001b[39m, \u001b[38;5;241m100\u001b[39m])\n",
      "\u001b[1;31mNameError\u001b[0m: name 'MultiDiscrete' is not defined"
     ]
    }
   ],
   "source": [
    "MultiDiscrete([3, 100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5d0403b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Yash\\AppData\\Roaming\\Python\\Python310\\site-packages\\stable_baselines3\\common\\vec_env\\patch_gym.py:49: UserWarning: You provided an OpenAI Gym environment. We strongly recommend transitioning to Gymnasium environments. Stable-Baselines3 is automatically wrapping your environments in a compatibility layer, which could potentially cause issues.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cpu device\n",
      "-----------------------------\n",
      "| time/              |      |\n",
      "|    fps             | 833  |\n",
      "|    iterations      | 1    |\n",
      "|    time_elapsed    | 2    |\n",
      "|    total_timesteps | 2048 |\n",
      "-----------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 654        |\n",
      "|    iterations           | 2          |\n",
      "|    time_elapsed         | 6          |\n",
      "|    total_timesteps      | 4096       |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.19097495 |\n",
      "|    clip_fraction        | 0.69       |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -5.6       |\n",
      "|    explained_variance   | -0.0082    |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 3.05       |\n",
      "|    n_updates            | 10         |\n",
      "|    policy_gradient_loss | -0.121     |\n",
      "|    value_loss           | 30.7       |\n",
      "----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 609        |\n",
      "|    iterations           | 3          |\n",
      "|    time_elapsed         | 10         |\n",
      "|    total_timesteps      | 6144       |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.20952234 |\n",
      "|    clip_fraction        | 0.598      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -5.3       |\n",
      "|    explained_variance   | 0.00131    |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 1.61       |\n",
      "|    n_updates            | 20         |\n",
      "|    policy_gradient_loss | -0.105     |\n",
      "|    value_loss           | 4.11       |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 590         |\n",
      "|    iterations           | 4           |\n",
      "|    time_elapsed         | 13          |\n",
      "|    total_timesteps      | 8192        |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.020720633 |\n",
      "|    clip_fraction        | 0.227       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -5.19       |\n",
      "|    explained_variance   | -0.00769    |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.00837    |\n",
      "|    n_updates            | 30          |\n",
      "|    policy_gradient_loss | -0.0334     |\n",
      "|    value_loss           | 0.612       |\n",
      "-----------------------------------------\n",
      "---------------------------------------\n",
      "| time/                   |           |\n",
      "|    fps                  | 580       |\n",
      "|    iterations           | 5         |\n",
      "|    time_elapsed         | 17        |\n",
      "|    total_timesteps      | 10240     |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.0722488 |\n",
      "|    clip_fraction        | 0.692     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -5.26     |\n",
      "|    explained_variance   | -0.0224   |\n",
      "|    learning_rate        | 0.0003    |\n",
      "|    loss                 | -0.109    |\n",
      "|    n_updates            | 40        |\n",
      "|    policy_gradient_loss | -0.0716   |\n",
      "|    value_loss           | 0.142     |\n",
      "---------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 578        |\n",
      "|    iterations           | 6          |\n",
      "|    time_elapsed         | 21         |\n",
      "|    total_timesteps      | 12288      |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.04182916 |\n",
      "|    clip_fraction        | 0.828      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -5.23      |\n",
      "|    explained_variance   | -0.0114    |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.142     |\n",
      "|    n_updates            | 50         |\n",
      "|    policy_gradient_loss | -0.075     |\n",
      "|    value_loss           | 0.217      |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 576         |\n",
      "|    iterations           | 7           |\n",
      "|    time_elapsed         | 24          |\n",
      "|    total_timesteps      | 14336       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.031154942 |\n",
      "|    clip_fraction        | 0.618       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -5.18       |\n",
      "|    explained_variance   | -0.000289   |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0535     |\n",
      "|    n_updates            | 60          |\n",
      "|    policy_gradient_loss | -0.0507     |\n",
      "|    value_loss           | 0.326       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 576         |\n",
      "|    iterations           | 8           |\n",
      "|    time_elapsed         | 28          |\n",
      "|    total_timesteps      | 16384       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.030440731 |\n",
      "|    clip_fraction        | 0.502       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -5.09       |\n",
      "|    explained_variance   | -0.00282    |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0821     |\n",
      "|    n_updates            | 70          |\n",
      "|    policy_gradient_loss | -0.0524     |\n",
      "|    value_loss           | 0.22        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 576         |\n",
      "|    iterations           | 9           |\n",
      "|    time_elapsed         | 31          |\n",
      "|    total_timesteps      | 18432       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.047518045 |\n",
      "|    clip_fraction        | 0.506       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.98       |\n",
      "|    explained_variance   | 0.0136      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.097      |\n",
      "|    n_updates            | 80          |\n",
      "|    policy_gradient_loss | -0.0492     |\n",
      "|    value_loss           | 0.287       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 577         |\n",
      "|    iterations           | 10          |\n",
      "|    time_elapsed         | 35          |\n",
      "|    total_timesteps      | 20480       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.035531126 |\n",
      "|    clip_fraction        | 0.44        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.87       |\n",
      "|    explained_variance   | -0.0286     |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0501     |\n",
      "|    n_updates            | 90          |\n",
      "|    policy_gradient_loss | -0.0557     |\n",
      "|    value_loss           | 0.129       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 577         |\n",
      "|    iterations           | 11          |\n",
      "|    time_elapsed         | 39          |\n",
      "|    total_timesteps      | 22528       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.057422653 |\n",
      "|    clip_fraction        | 0.433       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.78       |\n",
      "|    explained_variance   | -0.00622    |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.0673      |\n",
      "|    n_updates            | 100         |\n",
      "|    policy_gradient_loss | -0.0598     |\n",
      "|    value_loss           | 0.151       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 578         |\n",
      "|    iterations           | 12          |\n",
      "|    time_elapsed         | 42          |\n",
      "|    total_timesteps      | 24576       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.033192158 |\n",
      "|    clip_fraction        | 0.319       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.72       |\n",
      "|    explained_variance   | 0.0135      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.0214      |\n",
      "|    n_updates            | 110         |\n",
      "|    policy_gradient_loss | -0.0401     |\n",
      "|    value_loss           | 0.366       |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 578           |\n",
      "|    iterations           | 13            |\n",
      "|    time_elapsed         | 46            |\n",
      "|    total_timesteps      | 26624         |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00014299608 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -4.77         |\n",
      "|    explained_variance   | -0.000384     |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 3.35e+04      |\n",
      "|    n_updates            | 120           |\n",
      "|    policy_gradient_loss | -0.000482     |\n",
      "|    value_loss           | 4.73e+04      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 578          |\n",
      "|    iterations           | 14           |\n",
      "|    time_elapsed         | 49           |\n",
      "|    total_timesteps      | 28672        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 9.799466e-05 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -4.77        |\n",
      "|    explained_variance   | 4.21e-05     |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 2.09e+04     |\n",
      "|    n_updates            | 130          |\n",
      "|    policy_gradient_loss | -0.000808    |\n",
      "|    value_loss           | 8.16e+04     |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 578          |\n",
      "|    iterations           | 15           |\n",
      "|    time_elapsed         | 53           |\n",
      "|    total_timesteps      | 30720        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0008974789 |\n",
      "|    clip_fraction        | 0.00151      |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -4.78        |\n",
      "|    explained_variance   | 0.000739     |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.96e+04     |\n",
      "|    n_updates            | 140          |\n",
      "|    policy_gradient_loss | -0.00313     |\n",
      "|    value_loss           | 2.54e+04     |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 578         |\n",
      "|    iterations           | 16          |\n",
      "|    time_elapsed         | 56          |\n",
      "|    total_timesteps      | 32768       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.004427271 |\n",
      "|    clip_fraction        | 0.0236      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.79       |\n",
      "|    explained_variance   | -0.00218    |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 4.58e+03    |\n",
      "|    n_updates            | 150         |\n",
      "|    policy_gradient_loss | -0.000948   |\n",
      "|    value_loss           | 7e+04       |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 578        |\n",
      "|    iterations           | 17         |\n",
      "|    time_elapsed         | 60         |\n",
      "|    total_timesteps      | 34816      |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.04065626 |\n",
      "|    clip_fraction        | 0.316      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -4.85      |\n",
      "|    explained_variance   | -0.128     |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 0.51       |\n",
      "|    n_updates            | 160        |\n",
      "|    policy_gradient_loss | -0.0593    |\n",
      "|    value_loss           | 1.6        |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 579         |\n",
      "|    iterations           | 18          |\n",
      "|    time_elapsed         | 63          |\n",
      "|    total_timesteps      | 36864       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.020850029 |\n",
      "|    clip_fraction        | 0.234       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.78       |\n",
      "|    explained_variance   | -0.0806     |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0575     |\n",
      "|    n_updates            | 170         |\n",
      "|    policy_gradient_loss | -0.0422     |\n",
      "|    value_loss           | 0.194       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 579         |\n",
      "|    iterations           | 19          |\n",
      "|    time_elapsed         | 67          |\n",
      "|    total_timesteps      | 38912       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.023633972 |\n",
      "|    clip_fraction        | 0.332       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.71       |\n",
      "|    explained_variance   | 0.0292      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.288       |\n",
      "|    n_updates            | 180         |\n",
      "|    policy_gradient_loss | -0.0401     |\n",
      "|    value_loss           | 0.268       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 579         |\n",
      "|    iterations           | 20          |\n",
      "|    time_elapsed         | 70          |\n",
      "|    total_timesteps      | 40960       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.028141074 |\n",
      "|    clip_fraction        | 0.359       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.66       |\n",
      "|    explained_variance   | 0.00648     |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.033      |\n",
      "|    n_updates            | 190         |\n",
      "|    policy_gradient_loss | -0.0526     |\n",
      "|    value_loss           | 0.0895      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 579         |\n",
      "|    iterations           | 21          |\n",
      "|    time_elapsed         | 74          |\n",
      "|    total_timesteps      | 43008       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.026792783 |\n",
      "|    clip_fraction        | 0.328       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.63       |\n",
      "|    explained_variance   | 0.0172      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.0189      |\n",
      "|    n_updates            | 200         |\n",
      "|    policy_gradient_loss | -0.0466     |\n",
      "|    value_loss           | 0.261       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 580         |\n",
      "|    iterations           | 22          |\n",
      "|    time_elapsed         | 77          |\n",
      "|    total_timesteps      | 45056       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.029661784 |\n",
      "|    clip_fraction        | 0.303       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.58       |\n",
      "|    explained_variance   | -0.0389     |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.0224      |\n",
      "|    n_updates            | 210         |\n",
      "|    policy_gradient_loss | -0.0607     |\n",
      "|    value_loss           | 0.131       |\n",
      "-----------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 580           |\n",
      "|    iterations           | 23            |\n",
      "|    time_elapsed         | 81            |\n",
      "|    total_timesteps      | 47104         |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00022344137 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -4.62         |\n",
      "|    explained_variance   | 0.000414      |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 7.32e+04      |\n",
      "|    n_updates            | 220           |\n",
      "|    policy_gradient_loss | -7.58e-05     |\n",
      "|    value_loss           | 1.14e+05      |\n",
      "-------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 580          |\n",
      "|    iterations           | 24           |\n",
      "|    time_elapsed         | 84           |\n",
      "|    total_timesteps      | 49152        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 2.876157e-06 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -4.63        |\n",
      "|    explained_variance   | 0.000672     |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.79e+05     |\n",
      "|    n_updates            | 230          |\n",
      "|    policy_gradient_loss | -0.000259    |\n",
      "|    value_loss           | 6.38e+05     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 580           |\n",
      "|    iterations           | 25            |\n",
      "|    time_elapsed         | 88            |\n",
      "|    total_timesteps      | 51200         |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 3.6230485e-06 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -4.62         |\n",
      "|    explained_variance   | 0.00236       |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 9.45e+04      |\n",
      "|    n_updates            | 240           |\n",
      "|    policy_gradient_loss | -0.000239     |\n",
      "|    value_loss           | 2.12e+05      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 580           |\n",
      "|    iterations           | 26            |\n",
      "|    time_elapsed         | 91            |\n",
      "|    total_timesteps      | 53248         |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 3.2968237e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -4.62         |\n",
      "|    explained_variance   | 0.00203       |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 5.65e+03      |\n",
      "|    n_updates            | 250           |\n",
      "|    policy_gradient_loss | -0.000681     |\n",
      "|    value_loss           | 1.01e+05      |\n",
      "-------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 579         |\n",
      "|    iterations           | 27          |\n",
      "|    time_elapsed         | 95          |\n",
      "|    total_timesteps      | 55296       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 9.28808e-06 |\n",
      "|    clip_fraction        | 0           |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.63       |\n",
      "|    explained_variance   | 0.007       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 1.35e+05    |\n",
      "|    n_updates            | 260         |\n",
      "|    policy_gradient_loss | -0.000402   |\n",
      "|    value_loss           | 1.88e+05    |\n",
      "-----------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 580           |\n",
      "|    iterations           | 28            |\n",
      "|    time_elapsed         | 98            |\n",
      "|    total_timesteps      | 57344         |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.7704093e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -4.63         |\n",
      "|    explained_variance   | 0.00631       |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 5.8e+04       |\n",
      "|    n_updates            | 270           |\n",
      "|    policy_gradient_loss | -0.000515     |\n",
      "|    value_loss           | 1.61e+05      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 580           |\n",
      "|    iterations           | 29            |\n",
      "|    time_elapsed         | 102           |\n",
      "|    total_timesteps      | 59392         |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.7109676e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -4.63         |\n",
      "|    explained_variance   | 0.00498       |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 2.15e+05      |\n",
      "|    n_updates            | 280           |\n",
      "|    policy_gradient_loss | -0.000492     |\n",
      "|    value_loss           | 3.31e+05      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 580          |\n",
      "|    iterations           | 30           |\n",
      "|    time_elapsed         | 105          |\n",
      "|    total_timesteps      | 61440        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 5.465059e-05 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -4.63        |\n",
      "|    explained_variance   | 0.00235      |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 7.69e+04     |\n",
      "|    n_updates            | 290          |\n",
      "|    policy_gradient_loss | -0.000988    |\n",
      "|    value_loss           | 2.63e+05     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 580           |\n",
      "|    iterations           | 31            |\n",
      "|    time_elapsed         | 109           |\n",
      "|    total_timesteps      | 63488         |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00025668213 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -4.64         |\n",
      "|    explained_variance   | 0.0127        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 2.44e+04      |\n",
      "|    n_updates            | 300           |\n",
      "|    policy_gradient_loss | -0.00176      |\n",
      "|    value_loss           | 8.58e+04      |\n",
      "-------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 580         |\n",
      "|    iterations           | 32          |\n",
      "|    time_elapsed         | 112         |\n",
      "|    total_timesteps      | 65536       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.016067572 |\n",
      "|    clip_fraction        | 0.0525      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.66       |\n",
      "|    explained_variance   | -0.546      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 119         |\n",
      "|    n_updates            | 310         |\n",
      "|    policy_gradient_loss | -0.0194     |\n",
      "|    value_loss           | 446         |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 580        |\n",
      "|    iterations           | 33         |\n",
      "|    time_elapsed         | 116        |\n",
      "|    total_timesteps      | 67584      |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.05230782 |\n",
      "|    clip_fraction        | 0.289      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -4.73      |\n",
      "|    explained_variance   | -0.293     |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 2.4        |\n",
      "|    n_updates            | 320        |\n",
      "|    policy_gradient_loss | -0.0571    |\n",
      "|    value_loss           | 25         |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 580         |\n",
      "|    iterations           | 34          |\n",
      "|    time_elapsed         | 119         |\n",
      "|    total_timesteps      | 69632       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.056339458 |\n",
      "|    clip_fraction        | 0.38        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.66       |\n",
      "|    explained_variance   | -7.16       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.00689    |\n",
      "|    n_updates            | 330         |\n",
      "|    policy_gradient_loss | -0.065      |\n",
      "|    value_loss           | 0.795       |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 580         |\n",
      "|    iterations           | 35          |\n",
      "|    time_elapsed         | 123         |\n",
      "|    total_timesteps      | 71680       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.020256128 |\n",
      "|    clip_fraction        | 0.242       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.56       |\n",
      "|    explained_variance   | -0.388      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0713     |\n",
      "|    n_updates            | 340         |\n",
      "|    policy_gradient_loss | -0.0433     |\n",
      "|    value_loss           | 0.31        |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 580        |\n",
      "|    iterations           | 36         |\n",
      "|    time_elapsed         | 126        |\n",
      "|    total_timesteps      | 73728      |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.02047662 |\n",
      "|    clip_fraction        | 0.231      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -4.53      |\n",
      "|    explained_variance   | 0.00434    |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 0.575      |\n",
      "|    n_updates            | 350        |\n",
      "|    policy_gradient_loss | -0.0282    |\n",
      "|    value_loss           | 5.29       |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 580         |\n",
      "|    iterations           | 37          |\n",
      "|    time_elapsed         | 130         |\n",
      "|    total_timesteps      | 75776       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.022735387 |\n",
      "|    clip_fraction        | 0.269       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.53       |\n",
      "|    explained_variance   | 0.029       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.388       |\n",
      "|    n_updates            | 360         |\n",
      "|    policy_gradient_loss | -0.0358     |\n",
      "|    value_loss           | 0.429       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 580         |\n",
      "|    iterations           | 38          |\n",
      "|    time_elapsed         | 134         |\n",
      "|    total_timesteps      | 77824       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.023126598 |\n",
      "|    clip_fraction        | 0.242       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.51       |\n",
      "|    explained_variance   | 0.0322      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.342       |\n",
      "|    n_updates            | 370         |\n",
      "|    policy_gradient_loss | -0.0361     |\n",
      "|    value_loss           | 0.5         |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 580        |\n",
      "|    iterations           | 39         |\n",
      "|    time_elapsed         | 137        |\n",
      "|    total_timesteps      | 79872      |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.02656513 |\n",
      "|    clip_fraction        | 0.305      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -4.47      |\n",
      "|    explained_variance   | 0.0202     |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0639    |\n",
      "|    n_updates            | 380        |\n",
      "|    policy_gradient_loss | -0.0484    |\n",
      "|    value_loss           | 0.0869     |\n",
      "----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 580          |\n",
      "|    iterations           | 40           |\n",
      "|    time_elapsed         | 141          |\n",
      "|    total_timesteps      | 81920        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0021127989 |\n",
      "|    clip_fraction        | 0.0213       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -4.49        |\n",
      "|    explained_variance   | 0.00789      |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 3.52e+03     |\n",
      "|    n_updates            | 390          |\n",
      "|    policy_gradient_loss | 0.00161      |\n",
      "|    value_loss           | 8.58e+03     |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 580         |\n",
      "|    iterations           | 41          |\n",
      "|    time_elapsed         | 144         |\n",
      "|    total_timesteps      | 83968       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.012108572 |\n",
      "|    clip_fraction        | 0.0818      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.51       |\n",
      "|    explained_variance   | -2.6        |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 4.23        |\n",
      "|    n_updates            | 400         |\n",
      "|    policy_gradient_loss | -0.0259     |\n",
      "|    value_loss           | 11.8        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 580         |\n",
      "|    iterations           | 42          |\n",
      "|    time_elapsed         | 148         |\n",
      "|    total_timesteps      | 86016       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.026434347 |\n",
      "|    clip_fraction        | 0.224       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.48       |\n",
      "|    explained_variance   | -2.69       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.128       |\n",
      "|    n_updates            | 410         |\n",
      "|    policy_gradient_loss | -0.0474     |\n",
      "|    value_loss           | 0.612       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 580         |\n",
      "|    iterations           | 43          |\n",
      "|    time_elapsed         | 151         |\n",
      "|    total_timesteps      | 88064       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.027543746 |\n",
      "|    clip_fraction        | 0.276       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.37       |\n",
      "|    explained_variance   | -0.343      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0943     |\n",
      "|    n_updates            | 420         |\n",
      "|    policy_gradient_loss | -0.0521     |\n",
      "|    value_loss           | 0.0849      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 581         |\n",
      "|    iterations           | 44          |\n",
      "|    time_elapsed         | 155         |\n",
      "|    total_timesteps      | 90112       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.021753075 |\n",
      "|    clip_fraction        | 0.229       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.31       |\n",
      "|    explained_variance   | 0.00777     |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.0589      |\n",
      "|    n_updates            | 430         |\n",
      "|    policy_gradient_loss | -0.0368     |\n",
      "|    value_loss           | 0.412       |\n",
      "-----------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 581           |\n",
      "|    iterations           | 45            |\n",
      "|    time_elapsed         | 158           |\n",
      "|    total_timesteps      | 92160         |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00038129362 |\n",
      "|    clip_fraction        | 0.000488      |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -4.32         |\n",
      "|    explained_variance   | -0.00327      |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 292           |\n",
      "|    n_updates            | 440           |\n",
      "|    policy_gradient_loss | 0.00019       |\n",
      "|    value_loss           | 1.11e+05      |\n",
      "-------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 581         |\n",
      "|    iterations           | 46          |\n",
      "|    time_elapsed         | 162         |\n",
      "|    total_timesteps      | 94208       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.019200068 |\n",
      "|    clip_fraction        | 0.175       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.28       |\n",
      "|    explained_variance   | -4.5        |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.129       |\n",
      "|    n_updates            | 450         |\n",
      "|    policy_gradient_loss | -0.0453     |\n",
      "|    value_loss           | 2.08        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 581         |\n",
      "|    iterations           | 47          |\n",
      "|    time_elapsed         | 165         |\n",
      "|    total_timesteps      | 96256       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.020985939 |\n",
      "|    clip_fraction        | 0.237       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.23       |\n",
      "|    explained_variance   | -0.0222     |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.0831      |\n",
      "|    n_updates            | 460         |\n",
      "|    policy_gradient_loss | -0.0389     |\n",
      "|    value_loss           | 0.448       |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 581          |\n",
      "|    iterations           | 48           |\n",
      "|    time_elapsed         | 169          |\n",
      "|    total_timesteps      | 98304        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0009397451 |\n",
      "|    clip_fraction        | 0.000439     |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -4.25        |\n",
      "|    explained_variance   | -0.00142     |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.19e+03     |\n",
      "|    n_updates            | 470          |\n",
      "|    policy_gradient_loss | -0.00274     |\n",
      "|    value_loss           | 1e+03        |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 581           |\n",
      "|    iterations           | 49            |\n",
      "|    time_elapsed         | 172           |\n",
      "|    total_timesteps      | 100352        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 9.5780415e-06 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -4.24         |\n",
      "|    explained_variance   | 0.027         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 9.1e+04       |\n",
      "|    n_updates            | 480           |\n",
      "|    policy_gradient_loss | -0.000318     |\n",
      "|    value_loss           | 9.33e+04      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 581           |\n",
      "|    iterations           | 50            |\n",
      "|    time_elapsed         | 176           |\n",
      "|    total_timesteps      | 102400        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.3126351e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -4.25         |\n",
      "|    explained_variance   | 0.0506        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 2.3e+04       |\n",
      "|    n_updates            | 490           |\n",
      "|    policy_gradient_loss | -0.00044      |\n",
      "|    value_loss           | 5.64e+04      |\n",
      "-------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 581         |\n",
      "|    iterations           | 51          |\n",
      "|    time_elapsed         | 179         |\n",
      "|    total_timesteps      | 104448      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.004796914 |\n",
      "|    clip_fraction        | 0.029       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.24       |\n",
      "|    explained_variance   | -1.41       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 24.4        |\n",
      "|    n_updates            | 500         |\n",
      "|    policy_gradient_loss | -0.0138     |\n",
      "|    value_loss           | 72.5        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 581         |\n",
      "|    iterations           | 52          |\n",
      "|    time_elapsed         | 183         |\n",
      "|    total_timesteps      | 106496      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.023958853 |\n",
      "|    clip_fraction        | 0.21        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.21       |\n",
      "|    explained_variance   | -5.03       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.74        |\n",
      "|    n_updates            | 510         |\n",
      "|    policy_gradient_loss | -0.0433     |\n",
      "|    value_loss           | 3.42        |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 581        |\n",
      "|    iterations           | 53         |\n",
      "|    time_elapsed         | 186        |\n",
      "|    total_timesteps      | 108544     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.02583535 |\n",
      "|    clip_fraction        | 0.28       |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -4.2       |\n",
      "|    explained_variance   | -0.809     |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 0.0961     |\n",
      "|    n_updates            | 520        |\n",
      "|    policy_gradient_loss | -0.0551    |\n",
      "|    value_loss           | 0.435      |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 581         |\n",
      "|    iterations           | 54          |\n",
      "|    time_elapsed         | 190         |\n",
      "|    total_timesteps      | 110592      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.026215259 |\n",
      "|    clip_fraction        | 0.337       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.16       |\n",
      "|    explained_variance   | 0.625       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.035      |\n",
      "|    n_updates            | 530         |\n",
      "|    policy_gradient_loss | -0.0609     |\n",
      "|    value_loss           | 0.0027      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 581         |\n",
      "|    iterations           | 55          |\n",
      "|    time_elapsed         | 193         |\n",
      "|    total_timesteps      | 112640      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.024653595 |\n",
      "|    clip_fraction        | 0.312       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.14       |\n",
      "|    explained_variance   | 0.218       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0864     |\n",
      "|    n_updates            | 540         |\n",
      "|    policy_gradient_loss | -0.0475     |\n",
      "|    value_loss           | 0.0704      |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 581          |\n",
      "|    iterations           | 56           |\n",
      "|    time_elapsed         | 197          |\n",
      "|    total_timesteps      | 114688       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0063305004 |\n",
      "|    clip_fraction        | 0.0611       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -4.18        |\n",
      "|    explained_variance   | -0.00133     |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 4.38e+04     |\n",
      "|    n_updates            | 550          |\n",
      "|    policy_gradient_loss | 6.62e-05     |\n",
      "|    value_loss           | 1.4e+05      |\n",
      "------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 581          |\n",
      "|    iterations           | 57           |\n",
      "|    time_elapsed         | 200          |\n",
      "|    total_timesteps      | 116736       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 2.052082e-05 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -4.2         |\n",
      "|    explained_variance   | 0.0316       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.48e+04     |\n",
      "|    n_updates            | 560          |\n",
      "|    policy_gradient_loss | -0.000372    |\n",
      "|    value_loss           | 2.68e+04     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 580           |\n",
      "|    iterations           | 58            |\n",
      "|    time_elapsed         | 204           |\n",
      "|    total_timesteps      | 118784        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 7.0673705e-06 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -4.2          |\n",
      "|    explained_variance   | 0.0829        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.36e+04      |\n",
      "|    n_updates            | 570           |\n",
      "|    policy_gradient_loss | -0.000369     |\n",
      "|    value_loss           | 5.85e+04      |\n",
      "-------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 580         |\n",
      "|    iterations           | 59          |\n",
      "|    time_elapsed         | 208         |\n",
      "|    total_timesteps      | 120832      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.008494614 |\n",
      "|    clip_fraction        | 0.0528      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.19       |\n",
      "|    explained_variance   | -2.72       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 2.82        |\n",
      "|    n_updates            | 580         |\n",
      "|    policy_gradient_loss | -0.0182     |\n",
      "|    value_loss           | 45.9        |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 580          |\n",
      "|    iterations           | 60           |\n",
      "|    time_elapsed         | 211          |\n",
      "|    total_timesteps      | 122880       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0048556924 |\n",
      "|    clip_fraction        | 0.0375       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -4.18        |\n",
      "|    explained_variance   | 0.464        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 0.472        |\n",
      "|    n_updates            | 590          |\n",
      "|    policy_gradient_loss | -0.0109      |\n",
      "|    value_loss           | 100          |\n",
      "------------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 580        |\n",
      "|    iterations           | 61         |\n",
      "|    time_elapsed         | 215        |\n",
      "|    total_timesteps      | 124928     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.03172509 |\n",
      "|    clip_fraction        | 0.344      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -4.09      |\n",
      "|    explained_variance   | -105       |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.103     |\n",
      "|    n_updates            | 600        |\n",
      "|    policy_gradient_loss | -0.0669    |\n",
      "|    value_loss           | 0.0558     |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 580         |\n",
      "|    iterations           | 62          |\n",
      "|    time_elapsed         | 218         |\n",
      "|    total_timesteps      | 126976      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.017985176 |\n",
      "|    clip_fraction        | 0.201       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.09       |\n",
      "|    explained_variance   | -0.658      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.171       |\n",
      "|    n_updates            | 610         |\n",
      "|    policy_gradient_loss | -0.0318     |\n",
      "|    value_loss           | 0.375       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 580         |\n",
      "|    iterations           | 63          |\n",
      "|    time_elapsed         | 222         |\n",
      "|    total_timesteps      | 129024      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.021864979 |\n",
      "|    clip_fraction        | 0.255       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.08       |\n",
      "|    explained_variance   | 0.133       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.0649      |\n",
      "|    n_updates            | 620         |\n",
      "|    policy_gradient_loss | -0.0324     |\n",
      "|    value_loss           | 0.175       |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 580          |\n",
      "|    iterations           | 64           |\n",
      "|    time_elapsed         | 225          |\n",
      "|    total_timesteps      | 131072       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0026166015 |\n",
      "|    clip_fraction        | 0.00786      |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -4.12        |\n",
      "|    explained_variance   | -0.00511     |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 4.17e+03     |\n",
      "|    n_updates            | 630          |\n",
      "|    policy_gradient_loss | -0.00029     |\n",
      "|    value_loss           | 2.09e+05     |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 580         |\n",
      "|    iterations           | 65          |\n",
      "|    time_elapsed         | 229         |\n",
      "|    total_timesteps      | 133120      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.012608655 |\n",
      "|    clip_fraction        | 0.12        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.1        |\n",
      "|    explained_variance   | -2.33       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.475       |\n",
      "|    n_updates            | 640         |\n",
      "|    policy_gradient_loss | -0.0272     |\n",
      "|    value_loss           | 6.86        |\n",
      "-----------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 580           |\n",
      "|    iterations           | 66            |\n",
      "|    time_elapsed         | 232           |\n",
      "|    total_timesteps      | 135168        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00053436717 |\n",
      "|    clip_fraction        | 0.00083       |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -4.13         |\n",
      "|    explained_variance   | 0.0593        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 3.94e+04      |\n",
      "|    n_updates            | 650           |\n",
      "|    policy_gradient_loss | -0.000806     |\n",
      "|    value_loss           | 5.11e+04      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 580          |\n",
      "|    iterations           | 67           |\n",
      "|    time_elapsed         | 236          |\n",
      "|    total_timesteps      | 137216       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0051496127 |\n",
      "|    clip_fraction        | 0.0148       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -4.11        |\n",
      "|    explained_variance   | 0.0718       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 0.303        |\n",
      "|    n_updates            | 660          |\n",
      "|    policy_gradient_loss | -0.0063      |\n",
      "|    value_loss           | 7.64e+03     |\n",
      "------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 580         |\n",
      "|    iterations           | 68          |\n",
      "|    time_elapsed         | 239         |\n",
      "|    total_timesteps      | 139264      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.027332913 |\n",
      "|    clip_fraction        | 0.263       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.09       |\n",
      "|    explained_variance   | -8.82       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0834     |\n",
      "|    n_updates            | 670         |\n",
      "|    policy_gradient_loss | -0.0547     |\n",
      "|    value_loss           | 0.434       |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 580          |\n",
      "|    iterations           | 69           |\n",
      "|    time_elapsed         | 243          |\n",
      "|    total_timesteps      | 141312       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0008731092 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -4.11        |\n",
      "|    explained_variance   | 0.112        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 3.01e+04     |\n",
      "|    n_updates            | 680          |\n",
      "|    policy_gradient_loss | -0.000689    |\n",
      "|    value_loss           | 5.75e+04     |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 580         |\n",
      "|    iterations           | 70          |\n",
      "|    time_elapsed         | 246         |\n",
      "|    total_timesteps      | 143360      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.017353088 |\n",
      "|    clip_fraction        | 0.114       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.1        |\n",
      "|    explained_variance   | -3.49       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.547       |\n",
      "|    n_updates            | 690         |\n",
      "|    policy_gradient_loss | -0.0302     |\n",
      "|    value_loss           | 12.8        |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 581          |\n",
      "|    iterations           | 71           |\n",
      "|    time_elapsed         | 250          |\n",
      "|    total_timesteps      | 145408       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0001406729 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -4.07        |\n",
      "|    explained_variance   | 0.129        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.06e+03     |\n",
      "|    n_updates            | 700          |\n",
      "|    policy_gradient_loss | -0.000307    |\n",
      "|    value_loss           | 2.88e+04     |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 581         |\n",
      "|    iterations           | 72          |\n",
      "|    time_elapsed         | 253         |\n",
      "|    total_timesteps      | 147456      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.019008053 |\n",
      "|    clip_fraction        | 0.136       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.1        |\n",
      "|    explained_variance   | -4.66       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.657       |\n",
      "|    n_updates            | 710         |\n",
      "|    policy_gradient_loss | -0.0319     |\n",
      "|    value_loss           | 7.52        |\n",
      "-----------------------------------------\n",
      "---------------------------------------\n",
      "| time/                   |           |\n",
      "|    fps                  | 581       |\n",
      "|    iterations           | 73        |\n",
      "|    time_elapsed         | 257       |\n",
      "|    total_timesteps      | 149504    |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.0169436 |\n",
      "|    clip_fraction        | 0.18      |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -4.07     |\n",
      "|    explained_variance   | -0.217    |\n",
      "|    learning_rate        | 0.0003    |\n",
      "|    loss                 | 0.52      |\n",
      "|    n_updates            | 720       |\n",
      "|    policy_gradient_loss | -0.0405   |\n",
      "|    value_loss           | 1.7       |\n",
      "---------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 581         |\n",
      "|    iterations           | 74          |\n",
      "|    time_elapsed         | 260         |\n",
      "|    total_timesteps      | 151552      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.019718366 |\n",
      "|    clip_fraction        | 0.247       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.05       |\n",
      "|    explained_variance   | -0.0293     |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.336       |\n",
      "|    n_updates            | 730         |\n",
      "|    policy_gradient_loss | -0.0394     |\n",
      "|    value_loss           | 0.846       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 581         |\n",
      "|    iterations           | 75          |\n",
      "|    time_elapsed         | 264         |\n",
      "|    total_timesteps      | 153600      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.028518517 |\n",
      "|    clip_fraction        | 0.286       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -4.04       |\n",
      "|    explained_variance   | -0.538      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.201       |\n",
      "|    n_updates            | 740         |\n",
      "|    policy_gradient_loss | -0.0383     |\n",
      "|    value_loss           | 0.313       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 581         |\n",
      "|    iterations           | 76          |\n",
      "|    time_elapsed         | 267         |\n",
      "|    total_timesteps      | 155648      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.029882573 |\n",
      "|    clip_fraction        | 0.34        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.99       |\n",
      "|    explained_variance   | -22         |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 2.56        |\n",
      "|    n_updates            | 750         |\n",
      "|    policy_gradient_loss | -0.0414     |\n",
      "|    value_loss           | 1.49        |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 581          |\n",
      "|    iterations           | 77           |\n",
      "|    time_elapsed         | 271          |\n",
      "|    total_timesteps      | 157696       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0044043213 |\n",
      "|    clip_fraction        | 0.0152       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -4.02        |\n",
      "|    explained_variance   | 0.738        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 85           |\n",
      "|    n_updates            | 760          |\n",
      "|    policy_gradient_loss | -0.0106      |\n",
      "|    value_loss           | 71           |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 581         |\n",
      "|    iterations           | 78          |\n",
      "|    time_elapsed         | 274         |\n",
      "|    total_timesteps      | 159744      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.024968404 |\n",
      "|    clip_fraction        | 0.257       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.99       |\n",
      "|    explained_variance   | -3.12       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0716     |\n",
      "|    n_updates            | 770         |\n",
      "|    policy_gradient_loss | -0.047      |\n",
      "|    value_loss           | 0.16        |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 581         |\n",
      "|    iterations           | 79          |\n",
      "|    time_elapsed         | 278         |\n",
      "|    total_timesteps      | 161792      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.017710153 |\n",
      "|    clip_fraction        | 0.244       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.96       |\n",
      "|    explained_variance   | -8.51       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.00637     |\n",
      "|    n_updates            | 780         |\n",
      "|    policy_gradient_loss | -0.0293     |\n",
      "|    value_loss           | 1.78        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 581         |\n",
      "|    iterations           | 80          |\n",
      "|    time_elapsed         | 281         |\n",
      "|    total_timesteps      | 163840      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.024028799 |\n",
      "|    clip_fraction        | 0.291       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.91       |\n",
      "|    explained_variance   | 0.105       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.075      |\n",
      "|    n_updates            | 790         |\n",
      "|    policy_gradient_loss | -0.0468     |\n",
      "|    value_loss           | 0.0915      |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 581          |\n",
      "|    iterations           | 81           |\n",
      "|    time_elapsed         | 285          |\n",
      "|    total_timesteps      | 165888       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0027993196 |\n",
      "|    clip_fraction        | 0.00239      |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.94        |\n",
      "|    explained_variance   | 0.668        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 0.643        |\n",
      "|    n_updates            | 800          |\n",
      "|    policy_gradient_loss | -0.00681     |\n",
      "|    value_loss           | 182          |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 581           |\n",
      "|    iterations           | 82            |\n",
      "|    time_elapsed         | 288           |\n",
      "|    total_timesteps      | 167936        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 6.6561624e-06 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.95         |\n",
      "|    explained_variance   | 0.13          |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 4.5e+04       |\n",
      "|    n_updates            | 810           |\n",
      "|    policy_gradient_loss | -0.000383     |\n",
      "|    value_loss           | 5.46e+04      |\n",
      "-------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 581         |\n",
      "|    iterations           | 83          |\n",
      "|    time_elapsed         | 292         |\n",
      "|    total_timesteps      | 169984      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.017337944 |\n",
      "|    clip_fraction        | 0.118       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.95       |\n",
      "|    explained_variance   | -4.62       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.274       |\n",
      "|    n_updates            | 820         |\n",
      "|    policy_gradient_loss | -0.0301     |\n",
      "|    value_loss           | 5.55        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 581         |\n",
      "|    iterations           | 84          |\n",
      "|    time_elapsed         | 295         |\n",
      "|    total_timesteps      | 172032      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.024408165 |\n",
      "|    clip_fraction        | 0.222       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.93       |\n",
      "|    explained_variance   | -6.76       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.125       |\n",
      "|    n_updates            | 830         |\n",
      "|    policy_gradient_loss | -0.049      |\n",
      "|    value_loss           | 0.613       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 581         |\n",
      "|    iterations           | 85          |\n",
      "|    time_elapsed         | 299         |\n",
      "|    total_timesteps      | 174080      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.016597584 |\n",
      "|    clip_fraction        | 0.198       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.94       |\n",
      "|    explained_variance   | -0.702      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.042      |\n",
      "|    n_updates            | 840         |\n",
      "|    policy_gradient_loss | -0.0373     |\n",
      "|    value_loss           | 0.391       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 581         |\n",
      "|    iterations           | 86          |\n",
      "|    time_elapsed         | 302         |\n",
      "|    total_timesteps      | 176128      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.031105444 |\n",
      "|    clip_fraction        | 0.305       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.89       |\n",
      "|    explained_variance   | 0.0996      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.0258      |\n",
      "|    n_updates            | 850         |\n",
      "|    policy_gradient_loss | -0.0387     |\n",
      "|    value_loss           | 0.171       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 581         |\n",
      "|    iterations           | 87          |\n",
      "|    time_elapsed         | 306         |\n",
      "|    total_timesteps      | 178176      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.015418236 |\n",
      "|    clip_fraction        | 0.188       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.83       |\n",
      "|    explained_variance   | -13.9       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.526       |\n",
      "|    n_updates            | 860         |\n",
      "|    policy_gradient_loss | -0.0294     |\n",
      "|    value_loss           | 4.66        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 581         |\n",
      "|    iterations           | 88          |\n",
      "|    time_elapsed         | 309         |\n",
      "|    total_timesteps      | 180224      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.025991479 |\n",
      "|    clip_fraction        | 0.311       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.81       |\n",
      "|    explained_variance   | 0.27        |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0714     |\n",
      "|    n_updates            | 870         |\n",
      "|    policy_gradient_loss | -0.057      |\n",
      "|    value_loss           | 0.00385     |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 581        |\n",
      "|    iterations           | 89         |\n",
      "|    time_elapsed         | 313        |\n",
      "|    total_timesteps      | 182272     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.01841101 |\n",
      "|    clip_fraction        | 0.228      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -3.77      |\n",
      "|    explained_variance   | 0.288      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0592    |\n",
      "|    n_updates            | 880        |\n",
      "|    policy_gradient_loss | -0.0373    |\n",
      "|    value_loss           | 0.0938     |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 581           |\n",
      "|    iterations           | 90            |\n",
      "|    time_elapsed         | 316           |\n",
      "|    total_timesteps      | 184320        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00032652495 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.79         |\n",
      "|    explained_variance   | 0.137         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 3.28e+04      |\n",
      "|    n_updates            | 890           |\n",
      "|    policy_gradient_loss | -0.00041      |\n",
      "|    value_loss           | 5.42e+04      |\n",
      "-------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 582         |\n",
      "|    iterations           | 91          |\n",
      "|    time_elapsed         | 320         |\n",
      "|    total_timesteps      | 186368      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.014449552 |\n",
      "|    clip_fraction        | 0.0973      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.78       |\n",
      "|    explained_variance   | -5.4        |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0215     |\n",
      "|    n_updates            | 900         |\n",
      "|    policy_gradient_loss | -0.0292     |\n",
      "|    value_loss           | 3.48        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 582         |\n",
      "|    iterations           | 92          |\n",
      "|    time_elapsed         | 323         |\n",
      "|    total_timesteps      | 188416      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.021561664 |\n",
      "|    clip_fraction        | 0.225       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.77       |\n",
      "|    explained_variance   | -19.1       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0998     |\n",
      "|    n_updates            | 910         |\n",
      "|    policy_gradient_loss | -0.0509     |\n",
      "|    value_loss           | 0.198       |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 582          |\n",
      "|    iterations           | 93           |\n",
      "|    time_elapsed         | 327          |\n",
      "|    total_timesteps      | 190464       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0010248437 |\n",
      "|    clip_fraction        | 0.00449      |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.79        |\n",
      "|    explained_variance   | 0.159        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 5.2e+04      |\n",
      "|    n_updates            | 920          |\n",
      "|    policy_gradient_loss | 0.000474     |\n",
      "|    value_loss           | 4.65e+04     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 582           |\n",
      "|    iterations           | 94            |\n",
      "|    time_elapsed         | 330           |\n",
      "|    total_timesteps      | 192512        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.5495752e-06 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.81         |\n",
      "|    explained_variance   | 0.153         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 8.85e+04      |\n",
      "|    n_updates            | 930           |\n",
      "|    policy_gradient_loss | -0.000197     |\n",
      "|    value_loss           | 1.25e+05      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 582           |\n",
      "|    iterations           | 95            |\n",
      "|    time_elapsed         | 334           |\n",
      "|    total_timesteps      | 194560        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.1878175e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.78         |\n",
      "|    explained_variance   | 0.189         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 172           |\n",
      "|    n_updates            | 940           |\n",
      "|    policy_gradient_loss | -0.000581     |\n",
      "|    value_loss           | 2.84e+04      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 582          |\n",
      "|    iterations           | 96           |\n",
      "|    time_elapsed         | 337          |\n",
      "|    total_timesteps      | 196608       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 7.942435e-08 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.84        |\n",
      "|    explained_variance   | 0.0204       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 3.47e+06     |\n",
      "|    n_updates            | 950          |\n",
      "|    policy_gradient_loss | -3.04e-05    |\n",
      "|    value_loss           | 1.01e+07     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 582           |\n",
      "|    iterations           | 97            |\n",
      "|    time_elapsed         | 341           |\n",
      "|    total_timesteps      | 198656        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.8859282e-08 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.86         |\n",
      "|    explained_variance   | 0.0124        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 8.58e+06      |\n",
      "|    n_updates            | 960           |\n",
      "|    policy_gradient_loss | -3.2e-05      |\n",
      "|    value_loss           | 9.75e+06      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 582          |\n",
      "|    iterations           | 98           |\n",
      "|    time_elapsed         | 344          |\n",
      "|    total_timesteps      | 200704       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0052544083 |\n",
      "|    clip_fraction        | 0.0108       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.78        |\n",
      "|    explained_variance   | -1.1         |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 8.91         |\n",
      "|    n_updates            | 970          |\n",
      "|    policy_gradient_loss | -0.00728     |\n",
      "|    value_loss           | 578          |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 582         |\n",
      "|    iterations           | 99          |\n",
      "|    time_elapsed         | 348         |\n",
      "|    total_timesteps      | 202752      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.011805605 |\n",
      "|    clip_fraction        | 0.0673      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.78       |\n",
      "|    explained_variance   | -7.28       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.492       |\n",
      "|    n_updates            | 980         |\n",
      "|    policy_gradient_loss | -0.0241     |\n",
      "|    value_loss           | 51.2        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 582         |\n",
      "|    iterations           | 100         |\n",
      "|    time_elapsed         | 351         |\n",
      "|    total_timesteps      | 204800      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.012153792 |\n",
      "|    clip_fraction        | 0.0842      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.8        |\n",
      "|    explained_variance   | 0.789       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 28.4        |\n",
      "|    n_updates            | 990         |\n",
      "|    policy_gradient_loss | -0.0199     |\n",
      "|    value_loss           | 91.2        |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 582         |\n",
      "|    iterations           | 101         |\n",
      "|    time_elapsed         | 354         |\n",
      "|    total_timesteps      | 206848      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.022907479 |\n",
      "|    clip_fraction        | 0.221       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.79       |\n",
      "|    explained_variance   | -6.07       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 1.27        |\n",
      "|    n_updates            | 1000        |\n",
      "|    policy_gradient_loss | -0.0364     |\n",
      "|    value_loss           | 15.5        |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 582        |\n",
      "|    iterations           | 102        |\n",
      "|    time_elapsed         | 358        |\n",
      "|    total_timesteps      | 208896     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.02149045 |\n",
      "|    clip_fraction        | 0.256      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -3.76      |\n",
      "|    explained_variance   | 0.0129     |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 0.231      |\n",
      "|    n_updates            | 1010       |\n",
      "|    policy_gradient_loss | -0.0381    |\n",
      "|    value_loss           | 0.6        |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 582         |\n",
      "|    iterations           | 103         |\n",
      "|    time_elapsed         | 361         |\n",
      "|    total_timesteps      | 210944      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.023630342 |\n",
      "|    clip_fraction        | 0.272       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.79       |\n",
      "|    explained_variance   | 0.0494      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.223       |\n",
      "|    n_updates            | 1020        |\n",
      "|    policy_gradient_loss | -0.0339     |\n",
      "|    value_loss           | 0.354       |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 582        |\n",
      "|    iterations           | 104        |\n",
      "|    time_elapsed         | 365        |\n",
      "|    total_timesteps      | 212992     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.05511565 |\n",
      "|    clip_fraction        | 0.448      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -3.75      |\n",
      "|    explained_variance   | 0.8        |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0979    |\n",
      "|    n_updates            | 1030       |\n",
      "|    policy_gradient_loss | -0.0746    |\n",
      "|    value_loss           | 0.00224    |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 582         |\n",
      "|    iterations           | 105         |\n",
      "|    time_elapsed         | 369         |\n",
      "|    total_timesteps      | 215040      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.027278112 |\n",
      "|    clip_fraction        | 0.326       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.7        |\n",
      "|    explained_variance   | 0.935       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0644     |\n",
      "|    n_updates            | 1040        |\n",
      "|    policy_gradient_loss | -0.0547     |\n",
      "|    value_loss           | 0.00181     |\n",
      "-----------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 582           |\n",
      "|    iterations           | 106           |\n",
      "|    time_elapsed         | 372           |\n",
      "|    total_timesteps      | 217088        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00076150417 |\n",
      "|    clip_fraction        | 0.00137       |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.77         |\n",
      "|    explained_variance   | 0.146         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 6.99e+04      |\n",
      "|    n_updates            | 1050          |\n",
      "|    policy_gradient_loss | -0.000307     |\n",
      "|    value_loss           | 2.36e+05      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 582           |\n",
      "|    iterations           | 107           |\n",
      "|    time_elapsed         | 376           |\n",
      "|    total_timesteps      | 219136        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 2.8219074e-07 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.76         |\n",
      "|    explained_variance   | 0.167         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 9.84e+04      |\n",
      "|    n_updates            | 1060          |\n",
      "|    policy_gradient_loss | -0.000122     |\n",
      "|    value_loss           | 1.66e+05      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 582           |\n",
      "|    iterations           | 108           |\n",
      "|    time_elapsed         | 379           |\n",
      "|    total_timesteps      | 221184        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.8949795e-06 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.74         |\n",
      "|    explained_variance   | 0.19          |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 7.16e+04      |\n",
      "|    n_updates            | 1070          |\n",
      "|    policy_gradient_loss | -0.000238     |\n",
      "|    value_loss           | 8.9e+04       |\n",
      "-------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 582         |\n",
      "|    iterations           | 109         |\n",
      "|    time_elapsed         | 382         |\n",
      "|    total_timesteps      | 223232      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.009652247 |\n",
      "|    clip_fraction        | 0.0667      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.71       |\n",
      "|    explained_variance   | -3.85       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.537       |\n",
      "|    n_updates            | 1080        |\n",
      "|    policy_gradient_loss | -0.0248     |\n",
      "|    value_loss           | 20.2        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 583         |\n",
      "|    iterations           | 110         |\n",
      "|    time_elapsed         | 386         |\n",
      "|    total_timesteps      | 225280      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.017975315 |\n",
      "|    clip_fraction        | 0.144       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.68       |\n",
      "|    explained_variance   | -22.6       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.0518      |\n",
      "|    n_updates            | 1090        |\n",
      "|    policy_gradient_loss | -0.0365     |\n",
      "|    value_loss           | 16.1        |\n",
      "-----------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 583           |\n",
      "|    iterations           | 111           |\n",
      "|    time_elapsed         | 389           |\n",
      "|    total_timesteps      | 227328        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00020714029 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.67         |\n",
      "|    explained_variance   | 0.261         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.52e+04      |\n",
      "|    n_updates            | 1100          |\n",
      "|    policy_gradient_loss | -0.000428     |\n",
      "|    value_loss           | 2.85e+04      |\n",
      "-------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 583         |\n",
      "|    iterations           | 112         |\n",
      "|    time_elapsed         | 393         |\n",
      "|    total_timesteps      | 229376      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.014663544 |\n",
      "|    clip_fraction        | 0.143       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.65       |\n",
      "|    explained_variance   | -20.2       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 206         |\n",
      "|    n_updates            | 1110        |\n",
      "|    policy_gradient_loss | -0.0319     |\n",
      "|    value_loss           | 28.6        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 582         |\n",
      "|    iterations           | 113         |\n",
      "|    time_elapsed         | 396         |\n",
      "|    total_timesteps      | 231424      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.031442784 |\n",
      "|    clip_fraction        | 0.343       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.64       |\n",
      "|    explained_variance   | -27.2       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0885     |\n",
      "|    n_updates            | 1120        |\n",
      "|    policy_gradient_loss | -0.0581     |\n",
      "|    value_loss           | 0.0521      |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 582          |\n",
      "|    iterations           | 114          |\n",
      "|    time_elapsed         | 400          |\n",
      "|    total_timesteps      | 233472       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0059727766 |\n",
      "|    clip_fraction        | 0.0403       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.65        |\n",
      "|    explained_variance   | -0.932       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 0.399        |\n",
      "|    n_updates            | 1130         |\n",
      "|    policy_gradient_loss | -0.00661     |\n",
      "|    value_loss           | 198          |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 582          |\n",
      "|    iterations           | 115          |\n",
      "|    time_elapsed         | 404          |\n",
      "|    total_timesteps      | 235520       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 7.986365e-05 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.66        |\n",
      "|    explained_variance   | 0.322        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 611          |\n",
      "|    n_updates            | 1140         |\n",
      "|    policy_gradient_loss | -0.00114     |\n",
      "|    value_loss           | 1.04e+04     |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 582         |\n",
      "|    iterations           | 116         |\n",
      "|    time_elapsed         | 408         |\n",
      "|    total_timesteps      | 237568      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.018569482 |\n",
      "|    clip_fraction        | 0.209       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.64       |\n",
      "|    explained_variance   | -4.76       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.137       |\n",
      "|    n_updates            | 1150        |\n",
      "|    policy_gradient_loss | -0.0395     |\n",
      "|    value_loss           | 0.65        |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 581          |\n",
      "|    iterations           | 117          |\n",
      "|    time_elapsed         | 411          |\n",
      "|    total_timesteps      | 239616       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0142132025 |\n",
      "|    clip_fraction        | 0.139        |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.65        |\n",
      "|    explained_variance   | -18.7        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | -0.00823     |\n",
      "|    n_updates            | 1160         |\n",
      "|    policy_gradient_loss | -0.0239      |\n",
      "|    value_loss           | 22.9         |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 581         |\n",
      "|    iterations           | 118         |\n",
      "|    time_elapsed         | 415         |\n",
      "|    total_timesteps      | 241664      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.020126604 |\n",
      "|    clip_fraction        | 0.251       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.65       |\n",
      "|    explained_variance   | 0.0185      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0492     |\n",
      "|    n_updates            | 1170        |\n",
      "|    policy_gradient_loss | -0.0328     |\n",
      "|    value_loss           | 0.0947      |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 580          |\n",
      "|    iterations           | 119          |\n",
      "|    time_elapsed         | 419          |\n",
      "|    total_timesteps      | 243712       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0007185258 |\n",
      "|    clip_fraction        | 0.000781     |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.68        |\n",
      "|    explained_variance   | 0.271        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 18.8         |\n",
      "|    n_updates            | 1180         |\n",
      "|    policy_gradient_loss | -0.00101     |\n",
      "|    value_loss           | 3.07e+04     |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 580         |\n",
      "|    iterations           | 120         |\n",
      "|    time_elapsed         | 423         |\n",
      "|    total_timesteps      | 245760      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.019359464 |\n",
      "|    clip_fraction        | 0.18        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.63       |\n",
      "|    explained_variance   | -6.98       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0233     |\n",
      "|    n_updates            | 1190        |\n",
      "|    policy_gradient_loss | -0.0425     |\n",
      "|    value_loss           | 1.19        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 580         |\n",
      "|    iterations           | 121         |\n",
      "|    time_elapsed         | 427         |\n",
      "|    total_timesteps      | 247808      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.030266136 |\n",
      "|    clip_fraction        | 0.269       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.58       |\n",
      "|    explained_variance   | -2.35       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0667     |\n",
      "|    n_updates            | 1200        |\n",
      "|    policy_gradient_loss | -0.0492     |\n",
      "|    value_loss           | 0.214       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 580         |\n",
      "|    iterations           | 122         |\n",
      "|    time_elapsed         | 430         |\n",
      "|    total_timesteps      | 249856      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.034888238 |\n",
      "|    clip_fraction        | 0.289       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.49       |\n",
      "|    explained_variance   | -0.179      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.122      |\n",
      "|    n_updates            | 1210        |\n",
      "|    policy_gradient_loss | -0.0502     |\n",
      "|    value_loss           | 0.0925      |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 579         |\n",
      "|    iterations           | 123         |\n",
      "|    time_elapsed         | 434         |\n",
      "|    total_timesteps      | 251904      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.016029939 |\n",
      "|    clip_fraction        | 0.201       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.4        |\n",
      "|    explained_variance   | -24.5       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 204         |\n",
      "|    n_updates            | 1220        |\n",
      "|    policy_gradient_loss | -0.0217     |\n",
      "|    value_loss           | 24          |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 579        |\n",
      "|    iterations           | 124        |\n",
      "|    time_elapsed         | 437        |\n",
      "|    total_timesteps      | 253952     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.03377691 |\n",
      "|    clip_fraction        | 0.337      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -3.4       |\n",
      "|    explained_variance   | 0.734      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0779    |\n",
      "|    n_updates            | 1230       |\n",
      "|    policy_gradient_loss | -0.059     |\n",
      "|    value_loss           | 0.00214    |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 579         |\n",
      "|    iterations           | 125         |\n",
      "|    time_elapsed         | 441         |\n",
      "|    total_timesteps      | 256000      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.031469457 |\n",
      "|    clip_fraction        | 0.33        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.42       |\n",
      "|    explained_variance   | 0.91        |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0948     |\n",
      "|    n_updates            | 1240        |\n",
      "|    policy_gradient_loss | -0.0556     |\n",
      "|    value_loss           | 0.00225     |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 579          |\n",
      "|    iterations           | 126          |\n",
      "|    time_elapsed         | 445          |\n",
      "|    total_timesteps      | 258048       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0006385157 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.51        |\n",
      "|    explained_variance   | 0.18         |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 7.47e+04     |\n",
      "|    n_updates            | 1250         |\n",
      "|    policy_gradient_loss | -0.00136     |\n",
      "|    value_loss           | 2.04e+05     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 579           |\n",
      "|    iterations           | 127           |\n",
      "|    time_elapsed         | 448           |\n",
      "|    total_timesteps      | 260096        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.1238153e-06 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.51         |\n",
      "|    explained_variance   | 0.216         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 4.34e+04      |\n",
      "|    n_updates            | 1260          |\n",
      "|    policy_gradient_loss | -0.000191     |\n",
      "|    value_loss           | 1.12e+05      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 579          |\n",
      "|    iterations           | 128          |\n",
      "|    time_elapsed         | 452          |\n",
      "|    total_timesteps      | 262144       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 5.093694e-06 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.51        |\n",
      "|    explained_variance   | 0.252        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 2.1e+03      |\n",
      "|    n_updates            | 1270         |\n",
      "|    policy_gradient_loss | -0.000348    |\n",
      "|    value_loss           | 8.5e+04      |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 579           |\n",
      "|    iterations           | 129           |\n",
      "|    time_elapsed         | 455           |\n",
      "|    total_timesteps      | 264192        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 2.2613676e-08 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.68         |\n",
      "|    explained_variance   | 0.0425        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 4.02e+06      |\n",
      "|    n_updates            | 1280          |\n",
      "|    policy_gradient_loss | -2.2e-05      |\n",
      "|    value_loss           | 9.33e+06      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 579          |\n",
      "|    iterations           | 130          |\n",
      "|    time_elapsed         | 459          |\n",
      "|    total_timesteps      | 266240       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 5.707756e-06 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.5         |\n",
      "|    explained_variance   | 0.168        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.83e+04     |\n",
      "|    n_updates            | 1290         |\n",
      "|    policy_gradient_loss | -0.000364    |\n",
      "|    value_loss           | 6.54e+04     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 579           |\n",
      "|    iterations           | 131           |\n",
      "|    time_elapsed         | 463           |\n",
      "|    total_timesteps      | 268288        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 4.1400664e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.48         |\n",
      "|    explained_variance   | 0.3           |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 838           |\n",
      "|    n_updates            | 1300          |\n",
      "|    policy_gradient_loss | -0.00092      |\n",
      "|    value_loss           | 2.58e+04      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 579          |\n",
      "|    iterations           | 132          |\n",
      "|    time_elapsed         | 466          |\n",
      "|    total_timesteps      | 270336       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 8.309784e-06 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.5         |\n",
      "|    explained_variance   | 0.271        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 6.19e+04     |\n",
      "|    n_updates            | 1310         |\n",
      "|    policy_gradient_loss | -0.000469    |\n",
      "|    value_loss           | 8.43e+04     |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 579          |\n",
      "|    iterations           | 133          |\n",
      "|    time_elapsed         | 470          |\n",
      "|    total_timesteps      | 272384       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 8.699484e-06 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.5         |\n",
      "|    explained_variance   | 0.285        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 3.41e+04     |\n",
      "|    n_updates            | 1320         |\n",
      "|    policy_gradient_loss | -0.00053     |\n",
      "|    value_loss           | 1.03e+05     |\n",
      "------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 578         |\n",
      "|    iterations           | 134         |\n",
      "|    time_elapsed         | 474         |\n",
      "|    total_timesteps      | 274432      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.014546348 |\n",
      "|    clip_fraction        | 0.0985      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.45       |\n",
      "|    explained_variance   | -8.67       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 5.01        |\n",
      "|    n_updates            | 1330        |\n",
      "|    policy_gradient_loss | -0.0246     |\n",
      "|    value_loss           | 70.9        |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 578        |\n",
      "|    iterations           | 135        |\n",
      "|    time_elapsed         | 477        |\n",
      "|    total_timesteps      | 276480     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.02737576 |\n",
      "|    clip_fraction        | 0.279      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -3.44      |\n",
      "|    explained_variance   | -8.34      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 0.0116     |\n",
      "|    n_updates            | 1340       |\n",
      "|    policy_gradient_loss | -0.0515    |\n",
      "|    value_loss           | 0.758      |\n",
      "----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 578          |\n",
      "|    iterations           | 136          |\n",
      "|    time_elapsed         | 481          |\n",
      "|    total_timesteps      | 278528       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0012957857 |\n",
      "|    clip_fraction        | 0.00161      |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.46        |\n",
      "|    explained_variance   | 0.303        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 2.16e+04     |\n",
      "|    n_updates            | 1350         |\n",
      "|    policy_gradient_loss | -0.00175     |\n",
      "|    value_loss           | 8.87e+04     |\n",
      "------------------------------------------\n",
      "---------------------------------------\n",
      "| time/                   |           |\n",
      "|    fps                  | 577       |\n",
      "|    iterations           | 137       |\n",
      "|    time_elapsed         | 485       |\n",
      "|    total_timesteps      | 280576    |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 4.734e-06 |\n",
      "|    clip_fraction        | 0         |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -3.49     |\n",
      "|    explained_variance   | 0.293     |\n",
      "|    learning_rate        | 0.0003    |\n",
      "|    loss                 | 7.26e+04  |\n",
      "|    n_updates            | 1360      |\n",
      "|    policy_gradient_loss | -0.000416 |\n",
      "|    value_loss           | 1.64e+05  |\n",
      "---------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 577           |\n",
      "|    iterations           | 138           |\n",
      "|    time_elapsed         | 489           |\n",
      "|    total_timesteps      | 282624        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 2.8737675e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.43         |\n",
      "|    explained_variance   | 0.287         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 3.96e+04      |\n",
      "|    n_updates            | 1370          |\n",
      "|    policy_gradient_loss | -0.00095      |\n",
      "|    value_loss           | 6.47e+04      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 577           |\n",
      "|    iterations           | 139           |\n",
      "|    time_elapsed         | 493           |\n",
      "|    total_timesteps      | 284672        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 4.9210794e-06 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.48         |\n",
      "|    explained_variance   | 0.196         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 8.59e+04      |\n",
      "|    n_updates            | 1380          |\n",
      "|    policy_gradient_loss | -0.000449     |\n",
      "|    value_loss           | 3.99e+05      |\n",
      "-------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 576         |\n",
      "|    iterations           | 140         |\n",
      "|    time_elapsed         | 496         |\n",
      "|    total_timesteps      | 286720      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.010895722 |\n",
      "|    clip_fraction        | 0.0725      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.44       |\n",
      "|    explained_variance   | 0.738       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.713       |\n",
      "|    n_updates            | 1390        |\n",
      "|    policy_gradient_loss | -0.017      |\n",
      "|    value_loss           | 332         |\n",
      "-----------------------------------------\n",
      "---------------------------------------\n",
      "| time/                   |           |\n",
      "|    fps                  | 576       |\n",
      "|    iterations           | 141       |\n",
      "|    time_elapsed         | 500       |\n",
      "|    total_timesteps      | 288768    |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.0653919 |\n",
      "|    clip_fraction        | 0.264     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -3.38     |\n",
      "|    explained_variance   | -21       |\n",
      "|    learning_rate        | 0.0003    |\n",
      "|    loss                 | 1.78      |\n",
      "|    n_updates            | 1400      |\n",
      "|    policy_gradient_loss | -0.0474   |\n",
      "|    value_loss           | 59.3      |\n",
      "---------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 576         |\n",
      "|    iterations           | 142         |\n",
      "|    time_elapsed         | 504         |\n",
      "|    total_timesteps      | 290816      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.016838947 |\n",
      "|    clip_fraction        | 0.128       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.23       |\n",
      "|    explained_variance   | 0.797       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.0791      |\n",
      "|    n_updates            | 1410        |\n",
      "|    policy_gradient_loss | -0.0157     |\n",
      "|    value_loss           | 268         |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 576          |\n",
      "|    iterations           | 143          |\n",
      "|    time_elapsed         | 507          |\n",
      "|    total_timesteps      | 292864       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0023294282 |\n",
      "|    clip_fraction        | 0.00313      |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.27        |\n",
      "|    explained_variance   | 0.653        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 43.9         |\n",
      "|    n_updates            | 1420         |\n",
      "|    policy_gradient_loss | -0.00562     |\n",
      "|    value_loss           | 2.03e+03     |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 576         |\n",
      "|    iterations           | 144         |\n",
      "|    time_elapsed         | 511         |\n",
      "|    total_timesteps      | 294912      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.038462274 |\n",
      "|    clip_fraction        | 0.34        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.22       |\n",
      "|    explained_variance   | -5.15       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0735     |\n",
      "|    n_updates            | 1430        |\n",
      "|    policy_gradient_loss | -0.0421     |\n",
      "|    value_loss           | 0.338       |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 576         |\n",
      "|    iterations           | 145         |\n",
      "|    time_elapsed         | 514         |\n",
      "|    total_timesteps      | 296960      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.041183855 |\n",
      "|    clip_fraction        | 0.394       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.15       |\n",
      "|    explained_variance   | 0.362       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0776     |\n",
      "|    n_updates            | 1440        |\n",
      "|    policy_gradient_loss | -0.067      |\n",
      "|    value_loss           | 0.00532     |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 576         |\n",
      "|    iterations           | 146         |\n",
      "|    time_elapsed         | 518         |\n",
      "|    total_timesteps      | 299008      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.046428908 |\n",
      "|    clip_fraction        | 0.336       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.09       |\n",
      "|    explained_variance   | 0.914       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0581     |\n",
      "|    n_updates            | 1450        |\n",
      "|    policy_gradient_loss | -0.0545     |\n",
      "|    value_loss           | 0.00325     |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 576          |\n",
      "|    iterations           | 147          |\n",
      "|    time_elapsed         | 522          |\n",
      "|    total_timesteps      | 301056       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0004701123 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.4         |\n",
      "|    explained_variance   | 0.105        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.13e+06     |\n",
      "|    n_updates            | 1460         |\n",
      "|    policy_gradient_loss | 0.000158     |\n",
      "|    value_loss           | 3.21e+06     |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 576          |\n",
      "|    iterations           | 148          |\n",
      "|    time_elapsed         | 525          |\n",
      "|    total_timesteps      | 303104       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 1.481385e-08 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.24        |\n",
      "|    explained_variance   | 0.0297       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.9e+06      |\n",
      "|    n_updates            | 1470         |\n",
      "|    policy_gradient_loss | -2.7e-05     |\n",
      "|    value_loss           | 8.32e+06     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 576           |\n",
      "|    iterations           | 149           |\n",
      "|    time_elapsed         | 529           |\n",
      "|    total_timesteps      | 305152        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.9208528e-09 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.38         |\n",
      "|    explained_variance   | 0.0422        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 7.27e+06      |\n",
      "|    n_updates            | 1480          |\n",
      "|    policy_gradient_loss | -1.7e-05      |\n",
      "|    value_loss           | 1.55e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 576           |\n",
      "|    iterations           | 150           |\n",
      "|    time_elapsed         | 532           |\n",
      "|    total_timesteps      | 307200        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 3.5593985e-08 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.34         |\n",
      "|    explained_variance   | 0.0638        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 2.03e+06      |\n",
      "|    n_updates            | 1490          |\n",
      "|    policy_gradient_loss | -4.16e-05     |\n",
      "|    value_loss           | 3.47e+06      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 576           |\n",
      "|    iterations           | 151           |\n",
      "|    time_elapsed         | 536           |\n",
      "|    total_timesteps      | 309248        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 3.2305252e-08 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.37         |\n",
      "|    explained_variance   | 0.0646        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 3.46e+06      |\n",
      "|    n_updates            | 1500          |\n",
      "|    policy_gradient_loss | -3.8e-05      |\n",
      "|    value_loss           | 5.12e+06      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 576          |\n",
      "|    iterations           | 152          |\n",
      "|    time_elapsed         | 540          |\n",
      "|    total_timesteps      | 311296       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 9.546056e-09 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.44        |\n",
      "|    explained_variance   | 0.0448       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 6.71e+06     |\n",
      "|    n_updates            | 1510         |\n",
      "|    policy_gradient_loss | -2.76e-05    |\n",
      "|    value_loss           | 1.54e+07     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 576           |\n",
      "|    iterations           | 153           |\n",
      "|    time_elapsed         | 543           |\n",
      "|    total_timesteps      | 313344        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 4.8894435e-09 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.53         |\n",
      "|    explained_variance   | 0.0254        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.65e+07      |\n",
      "|    n_updates            | 1520          |\n",
      "|    policy_gradient_loss | -1.88e-05     |\n",
      "|    value_loss           | 3.7e+07       |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 575          |\n",
      "|    iterations           | 154          |\n",
      "|    time_elapsed         | 547          |\n",
      "|    total_timesteps      | 315392       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 6.171176e-07 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.3         |\n",
      "|    explained_variance   | 0.0605       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 7.02e+05     |\n",
      "|    n_updates            | 1530         |\n",
      "|    policy_gradient_loss | -0.000128    |\n",
      "|    value_loss           | 1.58e+06     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 575           |\n",
      "|    iterations           | 155           |\n",
      "|    time_elapsed         | 551           |\n",
      "|    total_timesteps      | 317440        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 3.7267455e-07 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.33         |\n",
      "|    explained_variance   | 0.0914        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 8.7e+05       |\n",
      "|    n_updates            | 1540          |\n",
      "|    policy_gradient_loss | -0.000112     |\n",
      "|    value_loss           | 3.07e+06      |\n",
      "-------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 575          |\n",
      "|    iterations           | 156          |\n",
      "|    time_elapsed         | 554          |\n",
      "|    total_timesteps      | 319488       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 4.208414e-08 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.32        |\n",
      "|    explained_variance   | 0.0321       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 5.06e+06     |\n",
      "|    n_updates            | 1550         |\n",
      "|    policy_gradient_loss | -4.38e-05    |\n",
      "|    value_loss           | 2.76e+07     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 575           |\n",
      "|    iterations           | 157           |\n",
      "|    time_elapsed         | 558           |\n",
      "|    total_timesteps      | 321536        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.4895253e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.26         |\n",
      "|    explained_variance   | 0.263         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 8.46e+04      |\n",
      "|    n_updates            | 1560          |\n",
      "|    policy_gradient_loss | -0.000515     |\n",
      "|    value_loss           | 2.08e+05      |\n",
      "-------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 575         |\n",
      "|    iterations           | 158         |\n",
      "|    time_elapsed         | 562         |\n",
      "|    total_timesteps      | 323584      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 3.90246e-05 |\n",
      "|    clip_fraction        | 0           |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.24       |\n",
      "|    explained_variance   | 0.318       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 1.02e+05    |\n",
      "|    n_updates            | 1570        |\n",
      "|    policy_gradient_loss | -0.00087    |\n",
      "|    value_loss           | 1.55e+05    |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 575          |\n",
      "|    iterations           | 159          |\n",
      "|    time_elapsed         | 565          |\n",
      "|    total_timesteps      | 325632       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 8.134605e-05 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.23        |\n",
      "|    explained_variance   | 0.372        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 5.9e+04      |\n",
      "|    n_updates            | 1580         |\n",
      "|    policy_gradient_loss | -0.00132     |\n",
      "|    value_loss           | 1.34e+05     |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 575          |\n",
      "|    iterations           | 160          |\n",
      "|    time_elapsed         | 569          |\n",
      "|    total_timesteps      | 327680       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 2.697925e-07 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.35        |\n",
      "|    explained_variance   | 0.047        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.68e+07     |\n",
      "|    n_updates            | 1590         |\n",
      "|    policy_gradient_loss | -6.55e-05    |\n",
      "|    value_loss           | 2.11e+07     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 575           |\n",
      "|    iterations           | 161           |\n",
      "|    time_elapsed         | 573           |\n",
      "|    total_timesteps      | 329728        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00022649072 |\n",
      "|    clip_fraction        | 4.88e-05      |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.22         |\n",
      "|    explained_variance   | 0.212         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 2.12e+04      |\n",
      "|    n_updates            | 1600          |\n",
      "|    policy_gradient_loss | -0.00191      |\n",
      "|    value_loss           | 1.59e+05      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 575          |\n",
      "|    iterations           | 162          |\n",
      "|    time_elapsed         | 576          |\n",
      "|    total_timesteps      | 331776       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 4.276866e-05 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.28        |\n",
      "|    explained_variance   | 0.316        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 2.33e+05     |\n",
      "|    n_updates            | 1610         |\n",
      "|    policy_gradient_loss | -0.00102     |\n",
      "|    value_loss           | 3.44e+05     |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 575          |\n",
      "|    iterations           | 163          |\n",
      "|    time_elapsed         | 580          |\n",
      "|    total_timesteps      | 333824       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 5.582406e-07 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.24        |\n",
      "|    explained_variance   | 0.0506       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 5.98e+06     |\n",
      "|    n_updates            | 1620         |\n",
      "|    policy_gradient_loss | -0.000167    |\n",
      "|    value_loss           | 1.39e+07     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 574           |\n",
      "|    iterations           | 164           |\n",
      "|    time_elapsed         | 584           |\n",
      "|    total_timesteps      | 335872        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00047933712 |\n",
      "|    clip_fraction        | 0.000488      |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.18         |\n",
      "|    explained_variance   | 0.329         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 8.38e+03      |\n",
      "|    n_updates            | 1630          |\n",
      "|    policy_gradient_loss | -0.00298      |\n",
      "|    value_loss           | 7.51e+04      |\n",
      "-------------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 574        |\n",
      "|    iterations           | 165        |\n",
      "|    time_elapsed         | 588        |\n",
      "|    total_timesteps      | 337920     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.02767881 |\n",
      "|    clip_fraction        | 0.2        |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -3.15      |\n",
      "|    explained_variance   | -7.18      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 1.22       |\n",
      "|    n_updates            | 1640       |\n",
      "|    policy_gradient_loss | -0.0342    |\n",
      "|    value_loss           | 72         |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 574         |\n",
      "|    iterations           | 166         |\n",
      "|    time_elapsed         | 592         |\n",
      "|    total_timesteps      | 339968      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.048749253 |\n",
      "|    clip_fraction        | 0.358       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.07       |\n",
      "|    explained_variance   | -7.78       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.174       |\n",
      "|    n_updates            | 1650        |\n",
      "|    policy_gradient_loss | -0.0596     |\n",
      "|    value_loss           | 2.81        |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 573        |\n",
      "|    iterations           | 167        |\n",
      "|    time_elapsed         | 596        |\n",
      "|    total_timesteps      | 342016     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.09409399 |\n",
      "|    clip_fraction        | 0.401      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -2.99      |\n",
      "|    explained_variance   | -105       |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0794    |\n",
      "|    n_updates            | 1660       |\n",
      "|    policy_gradient_loss | -0.0628    |\n",
      "|    value_loss           | 0.115      |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 168         |\n",
      "|    time_elapsed         | 599         |\n",
      "|    total_timesteps      | 344064      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.018257704 |\n",
      "|    clip_fraction        | 0.154       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.81       |\n",
      "|    explained_variance   | 0.0028      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 1.06        |\n",
      "|    n_updates            | 1670        |\n",
      "|    policy_gradient_loss | -0.0382     |\n",
      "|    value_loss           | 2.53        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 169         |\n",
      "|    time_elapsed         | 603         |\n",
      "|    total_timesteps      | 346112      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.024003504 |\n",
      "|    clip_fraction        | 0.147       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.8        |\n",
      "|    explained_variance   | -0.0381     |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.773       |\n",
      "|    n_updates            | 1680        |\n",
      "|    policy_gradient_loss | -0.0393     |\n",
      "|    value_loss           | 1.39        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 170         |\n",
      "|    time_elapsed         | 607         |\n",
      "|    total_timesteps      | 348160      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.020688009 |\n",
      "|    clip_fraction        | 0.193       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.84       |\n",
      "|    explained_variance   | -0.00945    |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.274       |\n",
      "|    n_updates            | 1690        |\n",
      "|    policy_gradient_loss | -0.0298     |\n",
      "|    value_loss           | 0.271       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 171         |\n",
      "|    time_elapsed         | 610         |\n",
      "|    total_timesteps      | 350208      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.050972424 |\n",
      "|    clip_fraction        | 0.41        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.8        |\n",
      "|    explained_variance   | 0.67        |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.11       |\n",
      "|    n_updates            | 1700        |\n",
      "|    policy_gradient_loss | -0.0684     |\n",
      "|    value_loss           | 0.0029      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 172         |\n",
      "|    time_elapsed         | 614         |\n",
      "|    total_timesteps      | 352256      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.017947303 |\n",
      "|    clip_fraction        | 0.205       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.75       |\n",
      "|    explained_variance   | 0.173       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.028      |\n",
      "|    n_updates            | 1710        |\n",
      "|    policy_gradient_loss | -0.0255     |\n",
      "|    value_loss           | 0.186       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 173         |\n",
      "|    time_elapsed         | 618         |\n",
      "|    total_timesteps      | 354304      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.014857333 |\n",
      "|    clip_fraction        | 0.174       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.76       |\n",
      "|    explained_variance   | 0.116       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.124       |\n",
      "|    n_updates            | 1720        |\n",
      "|    policy_gradient_loss | -0.0228     |\n",
      "|    value_loss           | 0.275       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 174         |\n",
      "|    time_elapsed         | 621         |\n",
      "|    total_timesteps      | 356352      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.023007724 |\n",
      "|    clip_fraction        | 0.239       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.81       |\n",
      "|    explained_variance   | 0.21        |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0979     |\n",
      "|    n_updates            | 1730        |\n",
      "|    policy_gradient_loss | -0.0331     |\n",
      "|    value_loss           | 0.0941      |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 573        |\n",
      "|    iterations           | 175        |\n",
      "|    time_elapsed         | 625        |\n",
      "|    total_timesteps      | 358400     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.03488844 |\n",
      "|    clip_fraction        | 0.227      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -2.83      |\n",
      "|    explained_variance   | 0.283      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.036     |\n",
      "|    n_updates            | 1740       |\n",
      "|    policy_gradient_loss | -0.0279    |\n",
      "|    value_loss           | 0.095      |\n",
      "----------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 176           |\n",
      "|    time_elapsed         | 628           |\n",
      "|    total_timesteps      | 360448        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00070992374 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.91         |\n",
      "|    explained_variance   | 0.382         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.03e+04      |\n",
      "|    n_updates            | 1750          |\n",
      "|    policy_gradient_loss | 0.000407      |\n",
      "|    value_loss           | 5.55e+04      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 177          |\n",
      "|    time_elapsed         | 632          |\n",
      "|    total_timesteps      | 362496       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 1.761128e-05 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.93        |\n",
      "|    explained_variance   | 0.483        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 9.82e+03     |\n",
      "|    n_updates            | 1760         |\n",
      "|    policy_gradient_loss | -0.000644    |\n",
      "|    value_loss           | 4.7e+04      |\n",
      "------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 178         |\n",
      "|    time_elapsed         | 635         |\n",
      "|    total_timesteps      | 364544      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.014067211 |\n",
      "|    clip_fraction        | 0.0927      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.88       |\n",
      "|    explained_variance   | -4.28       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.0667      |\n",
      "|    n_updates            | 1770        |\n",
      "|    policy_gradient_loss | -0.0265     |\n",
      "|    value_loss           | 5.58        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 179         |\n",
      "|    time_elapsed         | 639         |\n",
      "|    total_timesteps      | 366592      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.027699582 |\n",
      "|    clip_fraction        | 0.246       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.89       |\n",
      "|    explained_variance   | -14.7       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.112      |\n",
      "|    n_updates            | 1780        |\n",
      "|    policy_gradient_loss | -0.0455     |\n",
      "|    value_loss           | 0.333       |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 180          |\n",
      "|    time_elapsed         | 642          |\n",
      "|    total_timesteps      | 368640       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0009563968 |\n",
      "|    clip_fraction        | 0.0022       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.94        |\n",
      "|    explained_variance   | 0.757        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 97.8         |\n",
      "|    n_updates            | 1790         |\n",
      "|    policy_gradient_loss | -0.00234     |\n",
      "|    value_loss           | 1.89e+03     |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 181         |\n",
      "|    time_elapsed         | 646         |\n",
      "|    total_timesteps      | 370688      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.017158827 |\n",
      "|    clip_fraction        | 0.16        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.94       |\n",
      "|    explained_variance   | -0.725      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.156       |\n",
      "|    n_updates            | 1800        |\n",
      "|    policy_gradient_loss | -0.0258     |\n",
      "|    value_loss           | 0.198       |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 573        |\n",
      "|    iterations           | 182        |\n",
      "|    time_elapsed         | 650        |\n",
      "|    total_timesteps      | 372736     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.03099743 |\n",
      "|    clip_fraction        | 0.285      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -2.95      |\n",
      "|    explained_variance   | 0.00125    |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0584    |\n",
      "|    n_updates            | 1810       |\n",
      "|    policy_gradient_loss | -0.0507    |\n",
      "|    value_loss           | 0.0605     |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 183         |\n",
      "|    time_elapsed         | 653         |\n",
      "|    total_timesteps      | 374784      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.043050036 |\n",
      "|    clip_fraction        | 0.282       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.98       |\n",
      "|    explained_variance   | 0.74        |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0413     |\n",
      "|    n_updates            | 1820        |\n",
      "|    policy_gradient_loss | -0.0392     |\n",
      "|    value_loss           | 0.00166     |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 184         |\n",
      "|    time_elapsed         | 657         |\n",
      "|    total_timesteps      | 376832      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.014303599 |\n",
      "|    clip_fraction        | 0.086       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.02       |\n",
      "|    explained_variance   | 0.0628      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 1.52        |\n",
      "|    n_updates            | 1830        |\n",
      "|    policy_gradient_loss | -0.0329     |\n",
      "|    value_loss           | 3.17        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 185         |\n",
      "|    time_elapsed         | 660         |\n",
      "|    total_timesteps      | 378880      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.006118875 |\n",
      "|    clip_fraction        | 0.0526      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.01       |\n",
      "|    explained_variance   | -20         |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 17.4        |\n",
      "|    n_updates            | 1840        |\n",
      "|    policy_gradient_loss | -0.0148     |\n",
      "|    value_loss           | 304         |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 186         |\n",
      "|    time_elapsed         | 664         |\n",
      "|    total_timesteps      | 380928      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.019688088 |\n",
      "|    clip_fraction        | 0.143       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.03       |\n",
      "|    explained_variance   | -0.173      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.332       |\n",
      "|    n_updates            | 1850        |\n",
      "|    policy_gradient_loss | -0.0351     |\n",
      "|    value_loss           | 1.32        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 187         |\n",
      "|    time_elapsed         | 668         |\n",
      "|    total_timesteps      | 382976      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.016425097 |\n",
      "|    clip_fraction        | 0.164       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.06       |\n",
      "|    explained_variance   | -0.0416     |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.13        |\n",
      "|    n_updates            | 1860        |\n",
      "|    policy_gradient_loss | -0.0252     |\n",
      "|    value_loss           | 0.454       |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 573        |\n",
      "|    iterations           | 188        |\n",
      "|    time_elapsed         | 671        |\n",
      "|    total_timesteps      | 385024     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.01892585 |\n",
      "|    clip_fraction        | 0.213      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -3.06      |\n",
      "|    explained_variance   | -0.00351   |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 0.172      |\n",
      "|    n_updates            | 1870       |\n",
      "|    policy_gradient_loss | -0.0305    |\n",
      "|    value_loss           | 0.273      |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 189         |\n",
      "|    time_elapsed         | 675         |\n",
      "|    total_timesteps      | 387072      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.032050036 |\n",
      "|    clip_fraction        | 0.293       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.06       |\n",
      "|    explained_variance   | 0.184       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0902     |\n",
      "|    n_updates            | 1880        |\n",
      "|    policy_gradient_loss | -0.0369     |\n",
      "|    value_loss           | 0.0936      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 190         |\n",
      "|    time_elapsed         | 678         |\n",
      "|    total_timesteps      | 389120      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.027620107 |\n",
      "|    clip_fraction        | 0.194       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.02       |\n",
      "|    explained_variance   | -26.3       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0536     |\n",
      "|    n_updates            | 1890        |\n",
      "|    policy_gradient_loss | -0.0177     |\n",
      "|    value_loss           | 120         |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 191         |\n",
      "|    time_elapsed         | 682         |\n",
      "|    total_timesteps      | 391168      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.016187862 |\n",
      "|    clip_fraction        | 0.198       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.02       |\n",
      "|    explained_variance   | 0.19        |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0653     |\n",
      "|    n_updates            | 1900        |\n",
      "|    policy_gradient_loss | -0.0248     |\n",
      "|    value_loss           | 0.187       |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 573        |\n",
      "|    iterations           | 192        |\n",
      "|    time_elapsed         | 685        |\n",
      "|    total_timesteps      | 393216     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.03461911 |\n",
      "|    clip_fraction        | 0.325      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -3.08      |\n",
      "|    explained_variance   | 0.301      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0494    |\n",
      "|    n_updates            | 1910       |\n",
      "|    policy_gradient_loss | -0.0385    |\n",
      "|    value_loss           | 0.0951     |\n",
      "----------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 193           |\n",
      "|    time_elapsed         | 689           |\n",
      "|    total_timesteps      | 395264        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00072025775 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.56         |\n",
      "|    explained_variance   | 0.0505        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 8.56e+06      |\n",
      "|    n_updates            | 1920          |\n",
      "|    policy_gradient_loss | 5.05e-05      |\n",
      "|    value_loss           | 2.44e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 194           |\n",
      "|    time_elapsed         | 693           |\n",
      "|    total_timesteps      | 397312        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.7462298e-10 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.7          |\n",
      "|    explained_variance   | 0.014         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.51e+07      |\n",
      "|    n_updates            | 1930          |\n",
      "|    policy_gradient_loss | -1.18e-05     |\n",
      "|    value_loss           | 2.73e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 195           |\n",
      "|    time_elapsed         | 696           |\n",
      "|    total_timesteps      | 399360        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 6.9849193e-10 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.73         |\n",
      "|    explained_variance   | 0.00617       |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 2.12e+07      |\n",
      "|    n_updates            | 1940          |\n",
      "|    policy_gradient_loss | -1.24e-05     |\n",
      "|    value_loss           | 4.54e+07      |\n",
      "-------------------------------------------\n",
      "---------------------------------------\n",
      "| time/                   |           |\n",
      "|    fps                  | 573       |\n",
      "|    iterations           | 196       |\n",
      "|    time_elapsed         | 700       |\n",
      "|    total_timesteps      | 401408    |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.0       |\n",
      "|    clip_fraction        | 0         |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -3.77     |\n",
      "|    explained_variance   | 0.00257   |\n",
      "|    learning_rate        | 0.0003    |\n",
      "|    loss                 | 3.78e+07  |\n",
      "|    n_updates            | 1950      |\n",
      "|    policy_gradient_loss | -6.86e-06 |\n",
      "|    value_loss           | 8.38e+07  |\n",
      "---------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 197          |\n",
      "|    time_elapsed         | 703          |\n",
      "|    total_timesteps      | 403456       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 3.783498e-09 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.68        |\n",
      "|    explained_variance   | 0.00428      |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.6e+07      |\n",
      "|    n_updates            | 1960         |\n",
      "|    policy_gradient_loss | -1.83e-05    |\n",
      "|    value_loss           | 2.99e+07     |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 198          |\n",
      "|    time_elapsed         | 707          |\n",
      "|    total_timesteps      | 405504       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 2.240995e-09 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.71        |\n",
      "|    explained_variance   | 0.00379      |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 3.07e+07     |\n",
      "|    n_updates            | 1970         |\n",
      "|    policy_gradient_loss | -1.8e-05     |\n",
      "|    value_loss           | 5.65e+07     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 199           |\n",
      "|    time_elapsed         | 710           |\n",
      "|    total_timesteps      | 407552        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 3.8126018e-08 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.59         |\n",
      "|    explained_variance   | 0.00484       |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.88e+07      |\n",
      "|    n_updates            | 1980          |\n",
      "|    policy_gradient_loss | -3.93e-05     |\n",
      "|    value_loss           | 2.61e+07      |\n",
      "-------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 200          |\n",
      "|    time_elapsed         | 714          |\n",
      "|    total_timesteps      | 409600       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 8.032657e-09 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.77        |\n",
      "|    explained_variance   | 0.00321      |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 3.12e+07     |\n",
      "|    n_updates            | 1990         |\n",
      "|    policy_gradient_loss | -2.23e-05    |\n",
      "|    value_loss           | 5.91e+07     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 201           |\n",
      "|    time_elapsed         | 717           |\n",
      "|    total_timesteps      | 411648        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 3.6117854e-08 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.63         |\n",
      "|    explained_variance   | 0.00513       |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.42e+07      |\n",
      "|    n_updates            | 2000          |\n",
      "|    policy_gradient_loss | -4.37e-05     |\n",
      "|    value_loss           | 2.91e+07      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 202          |\n",
      "|    time_elapsed         | 721          |\n",
      "|    total_timesteps      | 413696       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 8.576899e-08 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.66        |\n",
      "|    explained_variance   | 0.00744      |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.45e+07     |\n",
      "|    n_updates            | 2010         |\n",
      "|    policy_gradient_loss | -5.12e-05    |\n",
      "|    value_loss           | 2.99e+07     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 203           |\n",
      "|    time_elapsed         | 725           |\n",
      "|    total_timesteps      | 415744        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.4406396e-07 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.63         |\n",
      "|    explained_variance   | 0.00661       |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 6.78e+06      |\n",
      "|    n_updates            | 2020          |\n",
      "|    policy_gradient_loss | -6.52e-05     |\n",
      "|    value_loss           | 2.88e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 204           |\n",
      "|    time_elapsed         | 728           |\n",
      "|    total_timesteps      | 417792        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 3.8009603e-08 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.74         |\n",
      "|    explained_variance   | 0.00416       |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 3.08e+07      |\n",
      "|    n_updates            | 2030          |\n",
      "|    policy_gradient_loss | -4.02e-05     |\n",
      "|    value_loss           | 5.65e+07      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 205          |\n",
      "|    time_elapsed         | 732          |\n",
      "|    total_timesteps      | 419840       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 2.651359e-08 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.68        |\n",
      "|    explained_variance   | 0.00241      |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 5.49e+07     |\n",
      "|    n_updates            | 2040         |\n",
      "|    policy_gradient_loss | -3.65e-05    |\n",
      "|    value_loss           | 1.29e+08     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 206           |\n",
      "|    time_elapsed         | 735           |\n",
      "|    total_timesteps      | 421888        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 3.2238313e-07 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.72         |\n",
      "|    explained_variance   | 0.00447       |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.42e+07      |\n",
      "|    n_updates            | 2050          |\n",
      "|    policy_gradient_loss | -8.58e-05     |\n",
      "|    value_loss           | 3.28e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 207           |\n",
      "|    time_elapsed         | 739           |\n",
      "|    total_timesteps      | 423936        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 2.5040936e-07 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.64         |\n",
      "|    explained_variance   | 0.00416       |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 2.01e+07      |\n",
      "|    n_updates            | 2060          |\n",
      "|    policy_gradient_loss | -9.28e-05     |\n",
      "|    value_loss           | 5.67e+07      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 208          |\n",
      "|    time_elapsed         | 742          |\n",
      "|    total_timesteps      | 425984       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 5.790498e-07 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.68        |\n",
      "|    explained_variance   | 0.00804      |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.16e+07     |\n",
      "|    n_updates            | 2070         |\n",
      "|    policy_gradient_loss | -0.000147    |\n",
      "|    value_loss           | 2.08e+07     |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 209          |\n",
      "|    time_elapsed         | 746          |\n",
      "|    total_timesteps      | 428032       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 7.678638e-06 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.7         |\n",
      "|    explained_variance   | 0.000282     |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 7.03e+07     |\n",
      "|    n_updates            | 2080         |\n",
      "|    policy_gradient_loss | -0.000509    |\n",
      "|    value_loss           | 1.07e+08     |\n",
      "------------------------------------------\n",
      "--------------------------------------------\n",
      "| time/                   |                |\n",
      "|    fps                  | 573            |\n",
      "|    iterations           | 210            |\n",
      "|    time_elapsed         | 749            |\n",
      "|    total_timesteps      | 430080         |\n",
      "| train/                  |                |\n",
      "|    approx_kl            | 1.02678314e-07 |\n",
      "|    clip_fraction        | 0              |\n",
      "|    clip_range           | 0.2            |\n",
      "|    entropy_loss         | -3.72          |\n",
      "|    explained_variance   | 0.0064         |\n",
      "|    learning_rate        | 0.0003         |\n",
      "|    loss                 | 3.7e+07        |\n",
      "|    n_updates            | 2090           |\n",
      "|    policy_gradient_loss | -6.96e-05      |\n",
      "|    value_loss           | 7.28e+07       |\n",
      "--------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 211          |\n",
      "|    time_elapsed         | 753          |\n",
      "|    total_timesteps      | 432128       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 2.203451e-07 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.75        |\n",
      "|    explained_variance   | 0.00251      |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 4.39e+07     |\n",
      "|    n_updates            | 2100         |\n",
      "|    policy_gradient_loss | -0.000103    |\n",
      "|    value_loss           | 7.93e+07     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 212           |\n",
      "|    time_elapsed         | 756           |\n",
      "|    total_timesteps      | 434176        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 6.8507507e-07 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.72         |\n",
      "|    explained_variance   | 0.00721       |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.8e+07       |\n",
      "|    n_updates            | 2110          |\n",
      "|    policy_gradient_loss | -0.000176     |\n",
      "|    value_loss           | 3.03e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 213           |\n",
      "|    time_elapsed         | 760           |\n",
      "|    total_timesteps      | 436224        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.8601131e-06 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.69         |\n",
      "|    explained_variance   | 0.0093        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 8.39e+06      |\n",
      "|    n_updates            | 2120          |\n",
      "|    policy_gradient_loss | -0.000254     |\n",
      "|    value_loss           | 2.32e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 214           |\n",
      "|    time_elapsed         | 764           |\n",
      "|    total_timesteps      | 438272        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 7.4334093e-07 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.69         |\n",
      "|    explained_variance   | 0.00718       |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 3.06e+07      |\n",
      "|    n_updates            | 2130          |\n",
      "|    policy_gradient_loss | -0.000173     |\n",
      "|    value_loss           | 5.41e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 215           |\n",
      "|    time_elapsed         | 767           |\n",
      "|    total_timesteps      | 440320        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 9.1607217e-07 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.74         |\n",
      "|    explained_variance   | 0.00818       |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 2.07e+07      |\n",
      "|    n_updates            | 2140          |\n",
      "|    policy_gradient_loss | -0.000208     |\n",
      "|    value_loss           | 4.06e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 216           |\n",
      "|    time_elapsed         | 771           |\n",
      "|    total_timesteps      | 442368        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.1655851e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.69         |\n",
      "|    explained_variance   | 0.0147        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 4.27e+06      |\n",
      "|    n_updates            | 2150          |\n",
      "|    policy_gradient_loss | -0.000688     |\n",
      "|    value_loss           | 1.07e+07      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 217          |\n",
      "|    time_elapsed         | 774          |\n",
      "|    total_timesteps      | 444416       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 8.091738e-06 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.61        |\n",
      "|    explained_variance   | 0.0169       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.22e+07     |\n",
      "|    n_updates            | 2160         |\n",
      "|    policy_gradient_loss | -0.000547    |\n",
      "|    value_loss           | 2.13e+07     |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 218          |\n",
      "|    time_elapsed         | 778          |\n",
      "|    total_timesteps      | 446464       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 4.790112e-06 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.66        |\n",
      "|    explained_variance   | 0.0099       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.79e+07     |\n",
      "|    n_updates            | 2170         |\n",
      "|    policy_gradient_loss | -0.000393    |\n",
      "|    value_loss           | 4.43e+07     |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 219          |\n",
      "|    time_elapsed         | 781          |\n",
      "|    total_timesteps      | 448512       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 5.218055e-06 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.75        |\n",
      "|    explained_variance   | 0.00731      |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 2.33e+07     |\n",
      "|    n_updates            | 2180         |\n",
      "|    policy_gradient_loss | -0.000366    |\n",
      "|    value_loss           | 4.45e+07     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 220           |\n",
      "|    time_elapsed         | 785           |\n",
      "|    total_timesteps      | 450560        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 2.7465867e-06 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.71         |\n",
      "|    explained_variance   | 0.00512       |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 5.62e+07      |\n",
      "|    n_updates            | 2190          |\n",
      "|    policy_gradient_loss | -0.000322     |\n",
      "|    value_loss           | 1.02e+08      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 221          |\n",
      "|    time_elapsed         | 788          |\n",
      "|    total_timesteps      | 452608       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 8.559466e-06 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.66        |\n",
      "|    explained_variance   | 0.0141       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.02e+07     |\n",
      "|    n_updates            | 2200         |\n",
      "|    policy_gradient_loss | -0.000585    |\n",
      "|    value_loss           | 2.58e+07     |\n",
      "------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 222          |\n",
      "|    time_elapsed         | 792          |\n",
      "|    total_timesteps      | 454656       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 1.514313e-05 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.71        |\n",
      "|    explained_variance   | 0.0131       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.48e+07     |\n",
      "|    n_updates            | 2210         |\n",
      "|    policy_gradient_loss | -0.00063     |\n",
      "|    value_loss           | 2.66e+07     |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 223          |\n",
      "|    time_elapsed         | 795          |\n",
      "|    total_timesteps      | 456704       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 7.530354e-06 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.73        |\n",
      "|    explained_variance   | 0.00626      |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 3.61e+07     |\n",
      "|    n_updates            | 2220         |\n",
      "|    policy_gradient_loss | -0.000523    |\n",
      "|    value_loss           | 5.56e+07     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 224           |\n",
      "|    time_elapsed         | 799           |\n",
      "|    total_timesteps      | 458752        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 7.1949034e-06 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.67         |\n",
      "|    explained_variance   | 0.00697       |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 2.88e+07      |\n",
      "|    n_updates            | 2230          |\n",
      "|    policy_gradient_loss | -0.000547     |\n",
      "|    value_loss           | 5.91e+07      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 225          |\n",
      "|    time_elapsed         | 803          |\n",
      "|    total_timesteps      | 460800       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 4.050293e-06 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.68        |\n",
      "|    explained_variance   | 0.00837      |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 7.74e+07     |\n",
      "|    n_updates            | 2240         |\n",
      "|    policy_gradient_loss | -0.000453    |\n",
      "|    value_loss           | 1.19e+08     |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 226          |\n",
      "|    time_elapsed         | 806          |\n",
      "|    total_timesteps      | 462848       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 1.554616e-05 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.73        |\n",
      "|    explained_variance   | 0.00893      |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.64e+07     |\n",
      "|    n_updates            | 2250         |\n",
      "|    policy_gradient_loss | -0.000824    |\n",
      "|    value_loss           | 4.04e+07     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 227           |\n",
      "|    time_elapsed         | 810           |\n",
      "|    total_timesteps      | 464896        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.4051388e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.65         |\n",
      "|    explained_variance   | 0.0142        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.93e+07      |\n",
      "|    n_updates            | 2260          |\n",
      "|    policy_gradient_loss | -0.000768     |\n",
      "|    value_loss           | 4.53e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 228           |\n",
      "|    time_elapsed         | 813           |\n",
      "|    total_timesteps      | 466944        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 9.5513475e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.55         |\n",
      "|    explained_variance   | 0.0299        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 7.35e+06      |\n",
      "|    n_updates            | 2270          |\n",
      "|    policy_gradient_loss | -0.00166      |\n",
      "|    value_loss           | 1.39e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 229           |\n",
      "|    time_elapsed         | 817           |\n",
      "|    total_timesteps      | 468992        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00011542224 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.61         |\n",
      "|    explained_variance   | 0.0313        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 8.61e+06      |\n",
      "|    n_updates            | 2280          |\n",
      "|    policy_gradient_loss | -0.00196      |\n",
      "|    value_loss           | 1.6e+07       |\n",
      "-------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 230         |\n",
      "|    time_elapsed         | 820         |\n",
      "|    total_timesteps      | 471040      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 3.50317e-05 |\n",
      "|    clip_fraction        | 0           |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.64       |\n",
      "|    explained_variance   | 0.0144      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 2.36e+07    |\n",
      "|    n_updates            | 2290        |\n",
      "|    policy_gradient_loss | -0.0012     |\n",
      "|    value_loss           | 4.51e+07    |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 231          |\n",
      "|    time_elapsed         | 824          |\n",
      "|    total_timesteps      | 473088       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 2.724654e-05 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.72        |\n",
      "|    explained_variance   | 0.0135       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 2.73e+07     |\n",
      "|    n_updates            | 2300         |\n",
      "|    policy_gradient_loss | -0.000979    |\n",
      "|    value_loss           | 3.79e+07     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 232           |\n",
      "|    time_elapsed         | 827           |\n",
      "|    total_timesteps      | 475136        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 5.4952252e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.67         |\n",
      "|    explained_variance   | 0.0188        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.38e+07      |\n",
      "|    n_updates            | 2310          |\n",
      "|    policy_gradient_loss | -0.00132      |\n",
      "|    value_loss           | 2.78e+07      |\n",
      "-------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 233           |\n",
      "|    time_elapsed         | 831           |\n",
      "|    total_timesteps      | 477184        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00023333126 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.53         |\n",
      "|    explained_variance   | 0.0304        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 5.21e+06      |\n",
      "|    n_updates            | 2320          |\n",
      "|    policy_gradient_loss | -0.00297      |\n",
      "|    value_loss           | 1.46e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 234           |\n",
      "|    time_elapsed         | 835           |\n",
      "|    total_timesteps      | 479232        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.5183439e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.78         |\n",
      "|    explained_variance   | 0.00815       |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 3.74e+07      |\n",
      "|    n_updates            | 2330          |\n",
      "|    policy_gradient_loss | -0.000847     |\n",
      "|    value_loss           | 6.99e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 235           |\n",
      "|    time_elapsed         | 838           |\n",
      "|    total_timesteps      | 481280        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 4.5443157e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.72         |\n",
      "|    explained_variance   | 0.00917       |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 3.01e+07      |\n",
      "|    n_updates            | 2340          |\n",
      "|    policy_gradient_loss | -0.00112      |\n",
      "|    value_loss           | 7.88e+07      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 236          |\n",
      "|    time_elapsed         | 842          |\n",
      "|    total_timesteps      | 483328       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0003514033 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.51        |\n",
      "|    explained_variance   | 0.0393       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 2.64e+06     |\n",
      "|    n_updates            | 2350         |\n",
      "|    policy_gradient_loss | -0.00347     |\n",
      "|    value_loss           | 1.02e+07     |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 237          |\n",
      "|    time_elapsed         | 845          |\n",
      "|    total_timesteps      | 485376       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 8.760751e-05 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.66        |\n",
      "|    explained_variance   | 0.025        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.19e+07     |\n",
      "|    n_updates            | 2360         |\n",
      "|    policy_gradient_loss | -0.00166     |\n",
      "|    value_loss           | 2.57e+07     |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 238          |\n",
      "|    time_elapsed         | 849          |\n",
      "|    total_timesteps      | 487424       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 6.423064e-05 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.63        |\n",
      "|    explained_variance   | 0.0252       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.51e+07     |\n",
      "|    n_updates            | 2370         |\n",
      "|    policy_gradient_loss | -0.00162     |\n",
      "|    value_loss           | 2.88e+07     |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 239          |\n",
      "|    time_elapsed         | 852          |\n",
      "|    total_timesteps      | 489472       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0005033235 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.54        |\n",
      "|    explained_variance   | 0.0387       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 5.03e+06     |\n",
      "|    n_updates            | 2380         |\n",
      "|    policy_gradient_loss | -0.00431     |\n",
      "|    value_loss           | 9.81e+06     |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 240          |\n",
      "|    time_elapsed         | 856          |\n",
      "|    total_timesteps      | 491520       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 6.947419e-05 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.47        |\n",
      "|    explained_variance   | 0.02         |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.67e+07     |\n",
      "|    n_updates            | 2390         |\n",
      "|    policy_gradient_loss | -0.00158     |\n",
      "|    value_loss           | 4.49e+07     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 241           |\n",
      "|    time_elapsed         | 859           |\n",
      "|    total_timesteps      | 493568        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00016357555 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.47         |\n",
      "|    explained_variance   | 0.0242        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 9.56e+06      |\n",
      "|    n_updates            | 2400          |\n",
      "|    policy_gradient_loss | -0.00241      |\n",
      "|    value_loss           | 2.67e+07      |\n",
      "-------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 242         |\n",
      "|    time_elapsed         | 863         |\n",
      "|    total_timesteps      | 495616      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.001191855 |\n",
      "|    clip_fraction        | 0.00166     |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.53       |\n",
      "|    explained_variance   | 0.0807      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 1.73e+06    |\n",
      "|    n_updates            | 2410        |\n",
      "|    policy_gradient_loss | -0.00605    |\n",
      "|    value_loss           | 4.15e+06    |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 243          |\n",
      "|    time_elapsed         | 867          |\n",
      "|    total_timesteps      | 497664       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0003085515 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.36        |\n",
      "|    explained_variance   | 0.0504       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 8.14e+06     |\n",
      "|    n_updates            | 2420         |\n",
      "|    policy_gradient_loss | -0.00413     |\n",
      "|    value_loss           | 8.55e+06     |\n",
      "------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 244         |\n",
      "|    time_elapsed         | 870         |\n",
      "|    total_timesteps      | 499712      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.003928326 |\n",
      "|    clip_fraction        | 0.0225      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.29       |\n",
      "|    explained_variance   | 0.246       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 7.87e+04    |\n",
      "|    n_updates            | 2430        |\n",
      "|    policy_gradient_loss | -0.0128     |\n",
      "|    value_loss           | 3.06e+05    |\n",
      "-----------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 245           |\n",
      "|    time_elapsed         | 874           |\n",
      "|    total_timesteps      | 501760        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00016141564 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.22         |\n",
      "|    explained_variance   | 0.0719        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.9e+06       |\n",
      "|    n_updates            | 2440          |\n",
      "|    policy_gradient_loss | -0.00233      |\n",
      "|    value_loss           | 4.87e+06      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 246          |\n",
      "|    time_elapsed         | 877          |\n",
      "|    total_timesteps      | 503808       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0018957861 |\n",
      "|    clip_fraction        | 0.00205      |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.15        |\n",
      "|    explained_variance   | 0.352        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 2.72e+04     |\n",
      "|    n_updates            | 2450         |\n",
      "|    policy_gradient_loss | -0.00761     |\n",
      "|    value_loss           | 8.99e+04     |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 247          |\n",
      "|    time_elapsed         | 881          |\n",
      "|    total_timesteps      | 505856       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0021573454 |\n",
      "|    clip_fraction        | 0.00635      |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.15        |\n",
      "|    explained_variance   | 0.348        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 4.17e+04     |\n",
      "|    n_updates            | 2460         |\n",
      "|    policy_gradient_loss | -0.00932     |\n",
      "|    value_loss           | 1e+05        |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 248          |\n",
      "|    time_elapsed         | 885          |\n",
      "|    total_timesteps      | 507904       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0075550927 |\n",
      "|    clip_fraction        | 0.0385       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.11        |\n",
      "|    explained_variance   | -0.884       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 465          |\n",
      "|    n_updates            | 2470         |\n",
      "|    policy_gradient_loss | -0.0158      |\n",
      "|    value_loss           | 9.03e+03     |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 249         |\n",
      "|    time_elapsed         | 888         |\n",
      "|    total_timesteps      | 509952      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.003592139 |\n",
      "|    clip_fraction        | 0.00776     |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.18       |\n",
      "|    explained_variance   | 0.412       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 5.37e+04    |\n",
      "|    n_updates            | 2480        |\n",
      "|    policy_gradient_loss | -0.00994    |\n",
      "|    value_loss           | 7.54e+04    |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 573        |\n",
      "|    iterations           | 250        |\n",
      "|    time_elapsed         | 892        |\n",
      "|    total_timesteps      | 512000     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.03232216 |\n",
      "|    clip_fraction        | 0.233      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -3.14      |\n",
      "|    explained_variance   | -8.95      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 22.7       |\n",
      "|    n_updates            | 2490       |\n",
      "|    policy_gradient_loss | -0.0418    |\n",
      "|    value_loss           | 485        |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 251         |\n",
      "|    time_elapsed         | 895         |\n",
      "|    total_timesteps      | 514048      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.016483132 |\n",
      "|    clip_fraction        | 0.16        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.15       |\n",
      "|    explained_variance   | 0.0362      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 1.61e+03    |\n",
      "|    n_updates            | 2500        |\n",
      "|    policy_gradient_loss | -0.0163     |\n",
      "|    value_loss           | 2.44e+03    |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 252         |\n",
      "|    time_elapsed         | 899         |\n",
      "|    total_timesteps      | 516096      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.044554986 |\n",
      "|    clip_fraction        | 0.351       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -3.12       |\n",
      "|    explained_variance   | -55.2       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 14.3        |\n",
      "|    n_updates            | 2510        |\n",
      "|    policy_gradient_loss | -0.0322     |\n",
      "|    value_loss           | 221         |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 573        |\n",
      "|    iterations           | 253        |\n",
      "|    time_elapsed         | 902        |\n",
      "|    total_timesteps      | 518144     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.10882644 |\n",
      "|    clip_fraction        | 0.507      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -2.87      |\n",
      "|    explained_variance   | -3.04      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 0.0971     |\n",
      "|    n_updates            | 2520       |\n",
      "|    policy_gradient_loss | -0.0574    |\n",
      "|    value_loss           | 0.23       |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 254         |\n",
      "|    time_elapsed         | 906         |\n",
      "|    total_timesteps      | 520192      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.029117929 |\n",
      "|    clip_fraction        | 0.317       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.73       |\n",
      "|    explained_variance   | -27.7       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 16.9        |\n",
      "|    n_updates            | 2530        |\n",
      "|    policy_gradient_loss | -0.0253     |\n",
      "|    value_loss           | 281         |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 255         |\n",
      "|    time_elapsed         | 910         |\n",
      "|    total_timesteps      | 522240      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.033526774 |\n",
      "|    clip_fraction        | 0.279       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.81       |\n",
      "|    explained_variance   | -0.0881     |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0758     |\n",
      "|    n_updates            | 2540        |\n",
      "|    policy_gradient_loss | -0.033      |\n",
      "|    value_loss           | 0.19        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 256         |\n",
      "|    time_elapsed         | 913         |\n",
      "|    total_timesteps      | 524288      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.054111004 |\n",
      "|    clip_fraction        | 0.395       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.81       |\n",
      "|    explained_variance   | 0.669       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0588     |\n",
      "|    n_updates            | 2550        |\n",
      "|    policy_gradient_loss | -0.0598     |\n",
      "|    value_loss           | 0.00382     |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 257         |\n",
      "|    time_elapsed         | 917         |\n",
      "|    total_timesteps      | 526336      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.040494956 |\n",
      "|    clip_fraction        | 0.363       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.7        |\n",
      "|    explained_variance   | 0.928       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0411     |\n",
      "|    n_updates            | 2560        |\n",
      "|    policy_gradient_loss | -0.0516     |\n",
      "|    value_loss           | 0.00187     |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 258         |\n",
      "|    time_elapsed         | 920         |\n",
      "|    total_timesteps      | 528384      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.011976365 |\n",
      "|    clip_fraction        | 0.0932      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.69       |\n",
      "|    explained_variance   | -22.9       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 1.04e+03    |\n",
      "|    n_updates            | 2570        |\n",
      "|    policy_gradient_loss | -0.00698    |\n",
      "|    value_loss           | 244         |\n",
      "-----------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 259           |\n",
      "|    time_elapsed         | 924           |\n",
      "|    total_timesteps      | 530432        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 2.4146953e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.32         |\n",
      "|    explained_variance   | 0.0402        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 3.1e+07       |\n",
      "|    n_updates            | 2580          |\n",
      "|    policy_gradient_loss | -9.16e-06     |\n",
      "|    value_loss           | 4.38e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 260           |\n",
      "|    time_elapsed         | 927           |\n",
      "|    total_timesteps      | 532480        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.0223011e-06 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.71         |\n",
      "|    explained_variance   | 0.313         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 122           |\n",
      "|    n_updates            | 2590          |\n",
      "|    policy_gradient_loss | -0.000127     |\n",
      "|    value_loss           | 3.69e+04      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 261          |\n",
      "|    time_elapsed         | 931          |\n",
      "|    total_timesteps      | 534528       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0052180495 |\n",
      "|    clip_fraction        | 0.0175       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.69        |\n",
      "|    explained_variance   | -6.26        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 0.225        |\n",
      "|    n_updates            | 2600         |\n",
      "|    policy_gradient_loss | -0.00691     |\n",
      "|    value_loss           | 899          |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 262           |\n",
      "|    time_elapsed         | 935           |\n",
      "|    total_timesteps      | 536576        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00017126891 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.9          |\n",
      "|    explained_variance   | 0.411         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 3.33e+05      |\n",
      "|    n_updates            | 2610          |\n",
      "|    policy_gradient_loss | -7.27e-05     |\n",
      "|    value_loss           | 4.47e+05      |\n",
      "-------------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 573        |\n",
      "|    iterations           | 263        |\n",
      "|    time_elapsed         | 938        |\n",
      "|    total_timesteps      | 538624     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 6.8394e-09 |\n",
      "|    clip_fraction        | 0          |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -2.96      |\n",
      "|    explained_variance   | 0.102      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 5.81e+06   |\n",
      "|    n_updates            | 2620       |\n",
      "|    policy_gradient_loss | -2.15e-05  |\n",
      "|    value_loss           | 1.38e+07   |\n",
      "----------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 264           |\n",
      "|    time_elapsed         | 942           |\n",
      "|    total_timesteps      | 540672        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.1233205e-06 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.86         |\n",
      "|    explained_variance   | 0.498         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 8.39e+04      |\n",
      "|    n_updates            | 2630          |\n",
      "|    policy_gradient_loss | -0.000164     |\n",
      "|    value_loss           | 2e+05         |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 265           |\n",
      "|    time_elapsed         | 945           |\n",
      "|    total_timesteps      | 542720        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 3.1735108e-06 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.86         |\n",
      "|    explained_variance   | 0.546         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 7.97e+04      |\n",
      "|    n_updates            | 2640          |\n",
      "|    policy_gradient_loss | -0.000323     |\n",
      "|    value_loss           | 1.83e+05      |\n",
      "-------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 266          |\n",
      "|    time_elapsed         | 949          |\n",
      "|    total_timesteps      | 544768       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0002551125 |\n",
      "|    clip_fraction        | 0.000146     |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.73        |\n",
      "|    explained_variance   | 0.622        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 401          |\n",
      "|    n_updates            | 2650         |\n",
      "|    policy_gradient_loss | -0.00195     |\n",
      "|    value_loss           | 1.97e+04     |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 267          |\n",
      "|    time_elapsed         | 953          |\n",
      "|    total_timesteps      | 546816       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 2.601181e-05 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.77        |\n",
      "|    explained_variance   | 0.58         |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 2.35e+04     |\n",
      "|    n_updates            | 2660         |\n",
      "|    policy_gradient_loss | -0.000751    |\n",
      "|    value_loss           | 6.7e+04      |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 268           |\n",
      "|    time_elapsed         | 956           |\n",
      "|    total_timesteps      | 548864        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.4374382e-07 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.95         |\n",
      "|    explained_variance   | 0.15          |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 5.01e+06      |\n",
      "|    n_updates            | 2670          |\n",
      "|    policy_gradient_loss | -7.59e-05     |\n",
      "|    value_loss           | 8.57e+06      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 269          |\n",
      "|    time_elapsed         | 960          |\n",
      "|    total_timesteps      | 550912       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0044912756 |\n",
      "|    clip_fraction        | 0.0159       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.7         |\n",
      "|    explained_variance   | -3.86        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 6.49         |\n",
      "|    n_updates            | 2680         |\n",
      "|    policy_gradient_loss | -0.0121      |\n",
      "|    value_loss           | 696          |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 270         |\n",
      "|    time_elapsed         | 963         |\n",
      "|    total_timesteps      | 552960      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.025108691 |\n",
      "|    clip_fraction        | 0.184       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.67       |\n",
      "|    explained_variance   | -5          |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.0882      |\n",
      "|    n_updates            | 2690        |\n",
      "|    policy_gradient_loss | -0.0361     |\n",
      "|    value_loss           | 37.5        |\n",
      "-----------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 271           |\n",
      "|    time_elapsed         | 967           |\n",
      "|    total_timesteps      | 555008        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00039408522 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.95         |\n",
      "|    explained_variance   | 0.561         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 2.6e+05       |\n",
      "|    n_updates            | 2700          |\n",
      "|    policy_gradient_loss | -0.000711     |\n",
      "|    value_loss           | 3.34e+05      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 272          |\n",
      "|    time_elapsed         | 970          |\n",
      "|    total_timesteps      | 557056       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 5.257025e-07 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.98        |\n",
      "|    explained_variance   | 0.206        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 6.24e+05     |\n",
      "|    n_updates            | 2710         |\n",
      "|    policy_gradient_loss | -0.000134    |\n",
      "|    value_loss           | 4.16e+06     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 273           |\n",
      "|    time_elapsed         | 974           |\n",
      "|    total_timesteps      | 559104        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 2.4451321e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.86         |\n",
      "|    explained_variance   | 0.563         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.33e+05      |\n",
      "|    n_updates            | 2720          |\n",
      "|    policy_gradient_loss | -0.000943     |\n",
      "|    value_loss           | 2.22e+05      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 274          |\n",
      "|    time_elapsed         | 978          |\n",
      "|    total_timesteps      | 561152       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 1.193257e-07 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -3.19        |\n",
      "|    explained_variance   | 0.0886       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.4e+07      |\n",
      "|    n_updates            | 2730         |\n",
      "|    policy_gradient_loss | -6.08e-05    |\n",
      "|    value_loss           | 2.84e+07     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 275           |\n",
      "|    time_elapsed         | 981           |\n",
      "|    total_timesteps      | 563200        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.3623503e-07 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.36         |\n",
      "|    explained_variance   | 0.0723        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.78e+07      |\n",
      "|    n_updates            | 2740          |\n",
      "|    policy_gradient_loss | -7.89e-05     |\n",
      "|    value_loss           | 3.64e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 276           |\n",
      "|    time_elapsed         | 985           |\n",
      "|    total_timesteps      | 565248        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 2.5029294e-07 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -3.31         |\n",
      "|    explained_variance   | 0.0433        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.81e+07      |\n",
      "|    n_updates            | 2750          |\n",
      "|    policy_gradient_loss | -9.64e-05     |\n",
      "|    value_loss           | 4.54e+07      |\n",
      "-------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 277           |\n",
      "|    time_elapsed         | 988           |\n",
      "|    total_timesteps      | 567296        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 5.5209966e-07 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.94         |\n",
      "|    explained_variance   | 0.0598        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.29e+07      |\n",
      "|    n_updates            | 2760          |\n",
      "|    policy_gradient_loss | -0.000146     |\n",
      "|    value_loss           | 2.33e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 278           |\n",
      "|    time_elapsed         | 992           |\n",
      "|    total_timesteps      | 569344        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00026268268 |\n",
      "|    clip_fraction        | 0.000244      |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.72         |\n",
      "|    explained_variance   | 0.561         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 5.44e+04      |\n",
      "|    n_updates            | 2770          |\n",
      "|    policy_gradient_loss | -0.00251      |\n",
      "|    value_loss           | 6.18e+04      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 279           |\n",
      "|    time_elapsed         | 995           |\n",
      "|    total_timesteps      | 571392        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 2.0942243e-06 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.97         |\n",
      "|    explained_variance   | 0.182         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 7.3e+06       |\n",
      "|    n_updates            | 2780          |\n",
      "|    policy_gradient_loss | -0.000259     |\n",
      "|    value_loss           | 7.92e+06      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 280          |\n",
      "|    time_elapsed         | 999          |\n",
      "|    total_timesteps      | 573440       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0002867647 |\n",
      "|    clip_fraction        | 0.000244     |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.84        |\n",
      "|    explained_variance   | 0.531        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.5e+05      |\n",
      "|    n_updates            | 2790         |\n",
      "|    policy_gradient_loss | -0.00266     |\n",
      "|    value_loss           | 1.73e+05     |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 281         |\n",
      "|    time_elapsed         | 1003        |\n",
      "|    total_timesteps      | 575488      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.000192331 |\n",
      "|    clip_fraction        | 0           |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.86       |\n",
      "|    explained_variance   | 0.594       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 1.64e+05    |\n",
      "|    n_updates            | 2800        |\n",
      "|    policy_gradient_loss | -0.00284    |\n",
      "|    value_loss           | 2.46e+05    |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 282          |\n",
      "|    time_elapsed         | 1006         |\n",
      "|    total_timesteps      | 577536       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0021810522 |\n",
      "|    clip_fraction        | 0.00532      |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.67        |\n",
      "|    explained_variance   | 0.346        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 8.96e+03     |\n",
      "|    n_updates            | 2810         |\n",
      "|    policy_gradient_loss | -0.0103      |\n",
      "|    value_loss           | 1.18e+04     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 283           |\n",
      "|    time_elapsed         | 1010          |\n",
      "|    total_timesteps      | 579584        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 4.3358887e-06 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.82         |\n",
      "|    explained_variance   | 0.13          |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 2.46e+06      |\n",
      "|    n_updates            | 2820          |\n",
      "|    policy_gradient_loss | -0.000311     |\n",
      "|    value_loss           | 9.55e+06      |\n",
      "-------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 284         |\n",
      "|    time_elapsed         | 1013        |\n",
      "|    total_timesteps      | 581632      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.004324777 |\n",
      "|    clip_fraction        | 0.0156      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.64       |\n",
      "|    explained_variance   | 0.679       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 8.71e+03    |\n",
      "|    n_updates            | 2830        |\n",
      "|    policy_gradient_loss | -0.0112     |\n",
      "|    value_loss           | 2e+04       |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 573        |\n",
      "|    iterations           | 285        |\n",
      "|    time_elapsed         | 1017       |\n",
      "|    total_timesteps      | 583680     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.06134928 |\n",
      "|    clip_fraction        | 0.33       |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -2.58      |\n",
      "|    explained_variance   | -4.9       |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.044     |\n",
      "|    n_updates            | 2840       |\n",
      "|    policy_gradient_loss | -0.0586    |\n",
      "|    value_loss           | 22.5       |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 286         |\n",
      "|    time_elapsed         | 1021        |\n",
      "|    total_timesteps      | 585728      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.066107415 |\n",
      "|    clip_fraction        | 0.388       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.55       |\n",
      "|    explained_variance   | -21.1       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0898     |\n",
      "|    n_updates            | 2850        |\n",
      "|    policy_gradient_loss | -0.0616     |\n",
      "|    value_loss           | 1.05        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 287         |\n",
      "|    time_elapsed         | 1024        |\n",
      "|    total_timesteps      | 587776      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.030291237 |\n",
      "|    clip_fraction        | 0.263       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.51       |\n",
      "|    explained_variance   | -26.9       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 25.7        |\n",
      "|    n_updates            | 2860        |\n",
      "|    policy_gradient_loss | -0.0229     |\n",
      "|    value_loss           | 704         |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 288          |\n",
      "|    time_elapsed         | 1028         |\n",
      "|    total_timesteps      | 589824       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0007562538 |\n",
      "|    clip_fraction        | 0.00298      |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.62        |\n",
      "|    explained_variance   | 0.595        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 4.92e+04     |\n",
      "|    n_updates            | 2870         |\n",
      "|    policy_gradient_loss | -0.000395    |\n",
      "|    value_loss           | 6.75e+04     |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 289         |\n",
      "|    time_elapsed         | 1032        |\n",
      "|    total_timesteps      | 591872      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.022762787 |\n",
      "|    clip_fraction        | 0.204       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.53       |\n",
      "|    explained_variance   | -8.47       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.0267      |\n",
      "|    n_updates            | 2880        |\n",
      "|    policy_gradient_loss | -0.0393     |\n",
      "|    value_loss           | 7.61        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 290         |\n",
      "|    time_elapsed         | 1035        |\n",
      "|    total_timesteps      | 593920      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.022285096 |\n",
      "|    clip_fraction        | 0.201       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.5        |\n",
      "|    explained_variance   | -26.9       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 9.2         |\n",
      "|    n_updates            | 2890        |\n",
      "|    policy_gradient_loss | -0.0162     |\n",
      "|    value_loss           | 684         |\n",
      "-----------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 291           |\n",
      "|    time_elapsed         | 1039          |\n",
      "|    total_timesteps      | 595968        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00024218534 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.86         |\n",
      "|    explained_variance   | 0.603         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.56e+05      |\n",
      "|    n_updates            | 2900          |\n",
      "|    policy_gradient_loss | -0.000344     |\n",
      "|    value_loss           | 3.33e+05      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 292           |\n",
      "|    time_elapsed         | 1042          |\n",
      "|    total_timesteps      | 598016        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00024825896 |\n",
      "|    clip_fraction        | 0.000342      |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.6          |\n",
      "|    explained_variance   | 0.622         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 2.33e+04      |\n",
      "|    n_updates            | 2910          |\n",
      "|    policy_gradient_loss | -0.00284      |\n",
      "|    value_loss           | 4.53e+04      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 293           |\n",
      "|    time_elapsed         | 1046          |\n",
      "|    total_timesteps      | 600064        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00057305023 |\n",
      "|    clip_fraction        | 0.000586      |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.58         |\n",
      "|    explained_variance   | 0.701         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.48e+04      |\n",
      "|    n_updates            | 2920          |\n",
      "|    policy_gradient_loss | -0.0043       |\n",
      "|    value_loss           | 2.3e+04       |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 294           |\n",
      "|    time_elapsed         | 1049          |\n",
      "|    total_timesteps      | 602112        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00022583807 |\n",
      "|    clip_fraction        | 9.77e-05      |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.61         |\n",
      "|    explained_variance   | 0.639         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.96e+04      |\n",
      "|    n_updates            | 2930          |\n",
      "|    policy_gradient_loss | -0.00302      |\n",
      "|    value_loss           | 4.31e+04      |\n",
      "-------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 295         |\n",
      "|    time_elapsed         | 1053        |\n",
      "|    total_timesteps      | 604160      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.026302667 |\n",
      "|    clip_fraction        | 0.158       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.52       |\n",
      "|    explained_variance   | -22.6       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 42.4        |\n",
      "|    n_updates            | 2940        |\n",
      "|    policy_gradient_loss | -0.026      |\n",
      "|    value_loss           | 698         |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 296         |\n",
      "|    time_elapsed         | 1057        |\n",
      "|    total_timesteps      | 606208      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.116955556 |\n",
      "|    clip_fraction        | 0.36        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.34       |\n",
      "|    explained_variance   | -14.7       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0799     |\n",
      "|    n_updates            | 2950        |\n",
      "|    policy_gradient_loss | -0.0585     |\n",
      "|    value_loss           | 0.481       |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 573        |\n",
      "|    iterations           | 297        |\n",
      "|    time_elapsed         | 1060       |\n",
      "|    total_timesteps      | 608256     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.12735848 |\n",
      "|    clip_fraction        | 0.369      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -2.11      |\n",
      "|    explained_variance   | -18.6      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.105     |\n",
      "|    n_updates            | 2960       |\n",
      "|    policy_gradient_loss | -0.0545    |\n",
      "|    value_loss           | 0.0192     |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 298         |\n",
      "|    time_elapsed         | 1064        |\n",
      "|    total_timesteps      | 610304      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.018798139 |\n",
      "|    clip_fraction        | 0.101       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.01       |\n",
      "|    explained_variance   | 0.0117      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 8.6         |\n",
      "|    n_updates            | 2970        |\n",
      "|    policy_gradient_loss | -0.0252     |\n",
      "|    value_loss           | 9.18        |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 299         |\n",
      "|    time_elapsed         | 1067        |\n",
      "|    total_timesteps      | 612352      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.013130031 |\n",
      "|    clip_fraction        | 0.0746      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.99       |\n",
      "|    explained_variance   | -0.0697     |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 1.15        |\n",
      "|    n_updates            | 2980        |\n",
      "|    policy_gradient_loss | -0.0234     |\n",
      "|    value_loss           | 7.56        |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 573        |\n",
      "|    iterations           | 300        |\n",
      "|    time_elapsed         | 1071       |\n",
      "|    total_timesteps      | 614400     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.01885212 |\n",
      "|    clip_fraction        | 0.164      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -2.02      |\n",
      "|    explained_variance   | -0.103     |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 0.776      |\n",
      "|    n_updates            | 2990       |\n",
      "|    policy_gradient_loss | -0.0325    |\n",
      "|    value_loss           | 1.41       |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 301         |\n",
      "|    time_elapsed         | 1075        |\n",
      "|    total_timesteps      | 616448      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.017492969 |\n",
      "|    clip_fraction        | 0.178       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.03       |\n",
      "|    explained_variance   | -0.202      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.288       |\n",
      "|    n_updates            | 3000        |\n",
      "|    policy_gradient_loss | -0.0251     |\n",
      "|    value_loss           | 0.453       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 302         |\n",
      "|    time_elapsed         | 1078        |\n",
      "|    total_timesteps      | 618496      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.019525982 |\n",
      "|    clip_fraction        | 0.235       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.06       |\n",
      "|    explained_variance   | -0.0405     |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.048       |\n",
      "|    n_updates            | 3010        |\n",
      "|    policy_gradient_loss | -0.0307     |\n",
      "|    value_loss           | 0.197       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 303         |\n",
      "|    time_elapsed         | 1082        |\n",
      "|    total_timesteps      | 620544      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.023373783 |\n",
      "|    clip_fraction        | 0.249       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.06       |\n",
      "|    explained_variance   | 0.0405      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.0627      |\n",
      "|    n_updates            | 3020        |\n",
      "|    policy_gradient_loss | -0.0278     |\n",
      "|    value_loss           | 0.174       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 304         |\n",
      "|    time_elapsed         | 1085        |\n",
      "|    total_timesteps      | 622592      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.021477917 |\n",
      "|    clip_fraction        | 0.227       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.08       |\n",
      "|    explained_variance   | 0.133       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.0019      |\n",
      "|    n_updates            | 3030        |\n",
      "|    policy_gradient_loss | -0.0308     |\n",
      "|    value_loss           | 0.102       |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 573        |\n",
      "|    iterations           | 305        |\n",
      "|    time_elapsed         | 1089       |\n",
      "|    total_timesteps      | 624640     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.02041861 |\n",
      "|    clip_fraction        | 0.24       |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -2.12      |\n",
      "|    explained_variance   | 0.232      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 0.0081     |\n",
      "|    n_updates            | 3040       |\n",
      "|    policy_gradient_loss | -0.0267    |\n",
      "|    value_loss           | 0.102      |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 306         |\n",
      "|    time_elapsed         | 1093        |\n",
      "|    total_timesteps      | 626688      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.023536248 |\n",
      "|    clip_fraction        | 0.224       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.14       |\n",
      "|    explained_variance   | 0.232       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0394     |\n",
      "|    n_updates            | 3050        |\n",
      "|    policy_gradient_loss | -0.032      |\n",
      "|    value_loss           | 0.0981      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 307         |\n",
      "|    time_elapsed         | 1096        |\n",
      "|    total_timesteps      | 628736      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.021433517 |\n",
      "|    clip_fraction        | 0.185       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.16       |\n",
      "|    explained_variance   | 0.107       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0423     |\n",
      "|    n_updates            | 3060        |\n",
      "|    policy_gradient_loss | -0.0188     |\n",
      "|    value_loss           | 0.199       |\n",
      "-----------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 308           |\n",
      "|    time_elapsed         | 1100          |\n",
      "|    total_timesteps      | 630784        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00023156151 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.26         |\n",
      "|    explained_variance   | 0.634         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 8.2e+04       |\n",
      "|    n_updates            | 3070          |\n",
      "|    policy_gradient_loss | -0.0008       |\n",
      "|    value_loss           | 6.86e+04      |\n",
      "-------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 309         |\n",
      "|    time_elapsed         | 1103        |\n",
      "|    total_timesteps      | 632832      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.011959091 |\n",
      "|    clip_fraction        | 0.0952      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.16       |\n",
      "|    explained_variance   | -7.87       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.186       |\n",
      "|    n_updates            | 3080        |\n",
      "|    policy_gradient_loss | -0.021      |\n",
      "|    value_loss           | 2.46        |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 310         |\n",
      "|    time_elapsed         | 1107        |\n",
      "|    total_timesteps      | 634880      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.035061732 |\n",
      "|    clip_fraction        | 0.261       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.21       |\n",
      "|    explained_variance   | -22.8       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0741     |\n",
      "|    n_updates            | 3090        |\n",
      "|    policy_gradient_loss | -0.0433     |\n",
      "|    value_loss           | 0.129       |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 311          |\n",
      "|    time_elapsed         | 1110         |\n",
      "|    total_timesteps      | 636928       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0036894039 |\n",
      "|    clip_fraction        | 0.0162       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.24        |\n",
      "|    explained_variance   | -0.00786     |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 4.46e+03     |\n",
      "|    n_updates            | 3100         |\n",
      "|    policy_gradient_loss | -0.00551     |\n",
      "|    value_loss           | 4.86e+03     |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 312         |\n",
      "|    time_elapsed         | 1114        |\n",
      "|    total_timesteps      | 638976      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.023473099 |\n",
      "|    clip_fraction        | 0.232       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.23       |\n",
      "|    explained_variance   | -1.37       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.0611      |\n",
      "|    n_updates            | 3110        |\n",
      "|    policy_gradient_loss | -0.0372     |\n",
      "|    value_loss           | 0.109       |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 573        |\n",
      "|    iterations           | 313        |\n",
      "|    time_elapsed         | 1118       |\n",
      "|    total_timesteps      | 641024     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.08078439 |\n",
      "|    clip_fraction        | 0.375      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -2.21      |\n",
      "|    explained_variance   | 0.855      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0614    |\n",
      "|    n_updates            | 3120       |\n",
      "|    policy_gradient_loss | -0.0596    |\n",
      "|    value_loss           | 0.00569    |\n",
      "----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 573        |\n",
      "|    iterations           | 314        |\n",
      "|    time_elapsed         | 1121       |\n",
      "|    total_timesteps      | 643072     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.00253323 |\n",
      "|    clip_fraction        | 0.00864    |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -2.27      |\n",
      "|    explained_variance   | 0.806      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 1.28e+03   |\n",
      "|    n_updates            | 3130       |\n",
      "|    policy_gradient_loss | -0.00393   |\n",
      "|    value_loss           | 4.41e+03   |\n",
      "----------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 315           |\n",
      "|    time_elapsed         | 1125          |\n",
      "|    total_timesteps      | 645120        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.7255341e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.31         |\n",
      "|    explained_variance   | 0.615         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 3.87e+04      |\n",
      "|    n_updates            | 3140          |\n",
      "|    policy_gradient_loss | -0.000578     |\n",
      "|    value_loss           | 1.01e+05      |\n",
      "-------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 316         |\n",
      "|    time_elapsed         | 1128        |\n",
      "|    total_timesteps      | 647168      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.016987044 |\n",
      "|    clip_fraction        | 0.13        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.24       |\n",
      "|    explained_variance   | -5.24       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.000825    |\n",
      "|    n_updates            | 3150        |\n",
      "|    policy_gradient_loss | -0.031      |\n",
      "|    value_loss           | 5.88        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 317         |\n",
      "|    time_elapsed         | 1132        |\n",
      "|    total_timesteps      | 649216      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.033731543 |\n",
      "|    clip_fraction        | 0.217       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.27       |\n",
      "|    explained_variance   | -8.6        |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0594     |\n",
      "|    n_updates            | 3160        |\n",
      "|    policy_gradient_loss | -0.0393     |\n",
      "|    value_loss           | 0.486       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 318         |\n",
      "|    time_elapsed         | 1136        |\n",
      "|    total_timesteps      | 651264      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.018678367 |\n",
      "|    clip_fraction        | 0.121       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.31       |\n",
      "|    explained_variance   | -0.255      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 1.52        |\n",
      "|    n_updates            | 3170        |\n",
      "|    policy_gradient_loss | -0.0301     |\n",
      "|    value_loss           | 2.78        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 319         |\n",
      "|    time_elapsed         | 1139        |\n",
      "|    total_timesteps      | 653312      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.012058687 |\n",
      "|    clip_fraction        | 0.111       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.29       |\n",
      "|    explained_variance   | -0.0374     |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 1.09        |\n",
      "|    n_updates            | 3180        |\n",
      "|    policy_gradient_loss | -0.0279     |\n",
      "|    value_loss           | 1.49        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 320         |\n",
      "|    time_elapsed         | 1143        |\n",
      "|    total_timesteps      | 655360      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.012069024 |\n",
      "|    clip_fraction        | 0.123       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.33       |\n",
      "|    explained_variance   | -49         |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 2.79        |\n",
      "|    n_updates            | 3190        |\n",
      "|    policy_gradient_loss | -0.0224     |\n",
      "|    value_loss           | 392         |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------\n",
      "| time/                   |                |\n",
      "|    fps                  | 573            |\n",
      "|    iterations           | 321            |\n",
      "|    time_elapsed         | 1146           |\n",
      "|    total_timesteps      | 657408         |\n",
      "| train/                  |                |\n",
      "|    approx_kl            | 0.000119640696 |\n",
      "|    clip_fraction        | 0              |\n",
      "|    clip_range           | 0.2            |\n",
      "|    entropy_loss         | -2.4           |\n",
      "|    explained_variance   | 0.646          |\n",
      "|    learning_rate        | 0.0003         |\n",
      "|    loss                 | 3.5e+04        |\n",
      "|    n_updates            | 3200           |\n",
      "|    policy_gradient_loss | -0.000964      |\n",
      "|    value_loss           | 6.6e+04        |\n",
      "--------------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 573        |\n",
      "|    iterations           | 322        |\n",
      "|    time_elapsed         | 1150       |\n",
      "|    total_timesteps      | 659456     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.00889275 |\n",
      "|    clip_fraction        | 0.0506     |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -2.34      |\n",
      "|    explained_variance   | -1.68      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 0.083      |\n",
      "|    n_updates            | 3210       |\n",
      "|    policy_gradient_loss | -0.00733   |\n",
      "|    value_loss           | 4.29e+03   |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 323         |\n",
      "|    time_elapsed         | 1154        |\n",
      "|    total_timesteps      | 661504      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.030179117 |\n",
      "|    clip_fraction        | 0.211       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.3        |\n",
      "|    explained_variance   | -4.74       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.00188     |\n",
      "|    n_updates            | 3220        |\n",
      "|    policy_gradient_loss | -0.0336     |\n",
      "|    value_loss           | 0.217       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 324         |\n",
      "|    time_elapsed         | 1157        |\n",
      "|    total_timesteps      | 663552      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.031074233 |\n",
      "|    clip_fraction        | 0.266       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.29       |\n",
      "|    explained_variance   | -0.718      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.0777      |\n",
      "|    n_updates            | 3230        |\n",
      "|    policy_gradient_loss | -0.0384     |\n",
      "|    value_loss           | 0.117       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 325         |\n",
      "|    time_elapsed         | 1161        |\n",
      "|    total_timesteps      | 665600      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.032711443 |\n",
      "|    clip_fraction        | 0.235       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.25       |\n",
      "|    explained_variance   | 0.0945      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.14        |\n",
      "|    n_updates            | 3240        |\n",
      "|    policy_gradient_loss | -0.0295     |\n",
      "|    value_loss           | 0.202       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 326         |\n",
      "|    time_elapsed         | 1164        |\n",
      "|    total_timesteps      | 667648      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.024136916 |\n",
      "|    clip_fraction        | 0.274       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.2        |\n",
      "|    explained_variance   | 0.383       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0497     |\n",
      "|    n_updates            | 3250        |\n",
      "|    policy_gradient_loss | -0.0504     |\n",
      "|    value_loss           | 0.003       |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 327          |\n",
      "|    time_elapsed         | 1168         |\n",
      "|    total_timesteps      | 669696       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0014248865 |\n",
      "|    clip_fraction        | 0.00273      |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.2         |\n",
      "|    explained_variance   | 0.719        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.37e+04     |\n",
      "|    n_updates            | 3260         |\n",
      "|    policy_gradient_loss | -0.000175    |\n",
      "|    value_loss           | 2.38e+04     |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 328         |\n",
      "|    time_elapsed         | 1172        |\n",
      "|    total_timesteps      | 671744      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.029736973 |\n",
      "|    clip_fraction        | 0.192       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.21       |\n",
      "|    explained_variance   | -186        |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0822     |\n",
      "|    n_updates            | 3270        |\n",
      "|    policy_gradient_loss | -0.0385     |\n",
      "|    value_loss           | 0.157       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 329         |\n",
      "|    time_elapsed         | 1175        |\n",
      "|    total_timesteps      | 673792      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.012211361 |\n",
      "|    clip_fraction        | 0.0959      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.29       |\n",
      "|    explained_variance   | -0.154      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.151       |\n",
      "|    n_updates            | 3280        |\n",
      "|    policy_gradient_loss | -0.0175     |\n",
      "|    value_loss           | 0.207       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 330         |\n",
      "|    time_elapsed         | 1179        |\n",
      "|    total_timesteps      | 675840      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.027226778 |\n",
      "|    clip_fraction        | 0.254       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.29       |\n",
      "|    explained_variance   | 0.606       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0523     |\n",
      "|    n_updates            | 3290        |\n",
      "|    policy_gradient_loss | -0.0384     |\n",
      "|    value_loss           | 0.00423     |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 573        |\n",
      "|    iterations           | 331        |\n",
      "|    time_elapsed         | 1182       |\n",
      "|    total_timesteps      | 677888     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.05296006 |\n",
      "|    clip_fraction        | 0.316      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -2.27      |\n",
      "|    explained_variance   | 0.928      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0577    |\n",
      "|    n_updates            | 3300       |\n",
      "|    policy_gradient_loss | -0.052     |\n",
      "|    value_loss           | 0.0049     |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 332           |\n",
      "|    time_elapsed         | 1186          |\n",
      "|    total_timesteps      | 679936        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00024042689 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.5          |\n",
      "|    explained_variance   | 0.0717        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 2.58e+07      |\n",
      "|    n_updates            | 3310          |\n",
      "|    policy_gradient_loss | -1.42e-05     |\n",
      "|    value_loss           | 4.71e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 333           |\n",
      "|    time_elapsed         | 1190          |\n",
      "|    total_timesteps      | 681984        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00043901813 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.31         |\n",
      "|    explained_variance   | -6.6          |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 3.62          |\n",
      "|    n_updates            | 3320          |\n",
      "|    policy_gradient_loss | -0.00234      |\n",
      "|    value_loss           | 1.4e+03       |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 334          |\n",
      "|    time_elapsed         | 1193         |\n",
      "|    total_timesteps      | 684032       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 7.984287e-06 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.35        |\n",
      "|    explained_variance   | 0.64         |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 9.74e+04     |\n",
      "|    n_updates            | 3330         |\n",
      "|    policy_gradient_loss | -0.000496    |\n",
      "|    value_loss           | 7e+04        |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 335          |\n",
      "|    time_elapsed         | 1197         |\n",
      "|    total_timesteps      | 686080       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0011382727 |\n",
      "|    clip_fraction        | 0.000635     |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.32        |\n",
      "|    explained_variance   | 0.39         |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.09e+03     |\n",
      "|    n_updates            | 3340         |\n",
      "|    policy_gradient_loss | -0.0042      |\n",
      "|    value_loss           | 4.47e+03     |\n",
      "------------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 573        |\n",
      "|    iterations           | 336        |\n",
      "|    time_elapsed         | 1200       |\n",
      "|    total_timesteps      | 688128     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.01308238 |\n",
      "|    clip_fraction        | 0.0834     |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -2.3       |\n",
      "|    explained_variance   | -27        |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 5.63e+03   |\n",
      "|    n_updates            | 3350       |\n",
      "|    policy_gradient_loss | -0.0126    |\n",
      "|    value_loss           | 746        |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 337         |\n",
      "|    time_elapsed         | 1204        |\n",
      "|    total_timesteps      | 690176      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.045323003 |\n",
      "|    clip_fraction        | 0.29        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.26       |\n",
      "|    explained_variance   | -14.8       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.081      |\n",
      "|    n_updates            | 3360        |\n",
      "|    policy_gradient_loss | -0.0442     |\n",
      "|    value_loss           | 0.0182      |\n",
      "-----------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 338           |\n",
      "|    time_elapsed         | 1207          |\n",
      "|    total_timesteps      | 692224        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00027644206 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.64         |\n",
      "|    explained_variance   | 0.12          |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 7.56e+06      |\n",
      "|    n_updates            | 3370          |\n",
      "|    policy_gradient_loss | 0.000937      |\n",
      "|    value_loss           | 2.37e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 339           |\n",
      "|    time_elapsed         | 1211          |\n",
      "|    total_timesteps      | 694272        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 4.5110937e-08 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.74         |\n",
      "|    explained_variance   | 0.0409        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.21e+07      |\n",
      "|    n_updates            | 3380          |\n",
      "|    policy_gradient_loss | -2.82e-05     |\n",
      "|    value_loss           | 2.37e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 340           |\n",
      "|    time_elapsed         | 1215          |\n",
      "|    total_timesteps      | 696320        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 5.1833922e-08 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.62         |\n",
      "|    explained_variance   | 0.0465        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 2.14e+07      |\n",
      "|    n_updates            | 3390          |\n",
      "|    policy_gradient_loss | -3.13e-05     |\n",
      "|    value_loss           | 3.95e+07      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 341          |\n",
      "|    time_elapsed         | 1218         |\n",
      "|    total_timesteps      | 698368       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 1.276785e-07 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.73        |\n",
      "|    explained_variance   | 0.0588       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.16e+07     |\n",
      "|    n_updates            | 3400         |\n",
      "|    policy_gradient_loss | -4.46e-05    |\n",
      "|    value_loss           | 2.27e+07     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 342           |\n",
      "|    time_elapsed         | 1222          |\n",
      "|    total_timesteps      | 700416        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 2.2040331e-07 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.64         |\n",
      "|    explained_variance   | 0.0635        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 8.42e+06      |\n",
      "|    n_updates            | 3410          |\n",
      "|    policy_gradient_loss | -5.84e-05     |\n",
      "|    value_loss           | 1.79e+07      |\n",
      "-------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 343          |\n",
      "|    time_elapsed         | 1225         |\n",
      "|    total_timesteps      | 702464       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 6.322807e-06 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.51        |\n",
      "|    explained_variance   | 0.414        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 2.3e+05      |\n",
      "|    n_updates            | 3420         |\n",
      "|    policy_gradient_loss | -0.000344    |\n",
      "|    value_loss           | 4.85e+05     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 344           |\n",
      "|    time_elapsed         | 1229          |\n",
      "|    total_timesteps      | 704512        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.2945384e-07 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.7          |\n",
      "|    explained_variance   | 0.0457        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 2.8e+07       |\n",
      "|    n_updates            | 3430          |\n",
      "|    policy_gradient_loss | -4.39e-05     |\n",
      "|    value_loss           | 5.27e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 345           |\n",
      "|    time_elapsed         | 1232          |\n",
      "|    total_timesteps      | 706560        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 7.1071554e-08 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.63         |\n",
      "|    explained_variance   | 0.0367        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 2.2e+07       |\n",
      "|    n_updates            | 3440          |\n",
      "|    policy_gradient_loss | -4.77e-05     |\n",
      "|    value_loss           | 5.63e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 346           |\n",
      "|    time_elapsed         | 1236          |\n",
      "|    total_timesteps      | 708608        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 6.1490573e-07 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.64         |\n",
      "|    explained_variance   | 0.0695        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 9.25e+06      |\n",
      "|    n_updates            | 3450          |\n",
      "|    policy_gradient_loss | -9.67e-05     |\n",
      "|    value_loss           | 2.01e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 347           |\n",
      "|    time_elapsed         | 1240          |\n",
      "|    total_timesteps      | 710656        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 3.8492726e-07 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.69         |\n",
      "|    explained_variance   | 0.0683        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.27e+07      |\n",
      "|    n_updates            | 3460          |\n",
      "|    policy_gradient_loss | -9.29e-05     |\n",
      "|    value_loss           | 2.75e+07      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 348          |\n",
      "|    time_elapsed         | 1243         |\n",
      "|    total_timesteps      | 712704       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 5.278853e-07 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.62        |\n",
      "|    explained_variance   | 0.0794       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 8.17e+06     |\n",
      "|    n_updates            | 3470         |\n",
      "|    policy_gradient_loss | -0.000123    |\n",
      "|    value_loss           | 1.6e+07      |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 349          |\n",
      "|    time_elapsed         | 1247         |\n",
      "|    total_timesteps      | 714752       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 4.841713e-07 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.64        |\n",
      "|    explained_variance   | 0.0956       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 8.07e+06     |\n",
      "|    n_updates            | 3480         |\n",
      "|    policy_gradient_loss | -0.000115    |\n",
      "|    value_loss           | 1.63e+07     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 350           |\n",
      "|    time_elapsed         | 1250          |\n",
      "|    total_timesteps      | 716800        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 5.1668903e-06 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.58         |\n",
      "|    explained_variance   | 0.179         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.63e+06      |\n",
      "|    n_updates            | 3490          |\n",
      "|    policy_gradient_loss | -0.000342     |\n",
      "|    value_loss           | 3.9e+06       |\n",
      "-------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 351         |\n",
      "|    time_elapsed         | 1254        |\n",
      "|    total_timesteps      | 718848      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 1.36171e-06 |\n",
      "|    clip_fraction        | 0           |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.64       |\n",
      "|    explained_variance   | 0.122       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 6.35e+06    |\n",
      "|    n_updates            | 3500        |\n",
      "|    policy_gradient_loss | -0.000184   |\n",
      "|    value_loss           | 1.32e+07    |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 352         |\n",
      "|    time_elapsed         | 1257        |\n",
      "|    total_timesteps      | 720896      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 3.61586e-07 |\n",
      "|    clip_fraction        | 0           |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.67       |\n",
      "|    explained_variance   | 0.0673      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 2.3e+07     |\n",
      "|    n_updates            | 3510        |\n",
      "|    policy_gradient_loss | -9.86e-05   |\n",
      "|    value_loss           | 4.91e+07    |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 573          |\n",
      "|    iterations           | 353          |\n",
      "|    time_elapsed         | 1261         |\n",
      "|    total_timesteps      | 722944       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 6.485003e-06 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.63        |\n",
      "|    explained_variance   | 0.119        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 8.82e+06     |\n",
      "|    n_updates            | 3520         |\n",
      "|    policy_gradient_loss | -0.00033     |\n",
      "|    value_loss           | 1.25e+07     |\n",
      "------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 354           |\n",
      "|    time_elapsed         | 1265          |\n",
      "|    total_timesteps      | 724992        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 2.5503105e-06 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.69         |\n",
      "|    explained_variance   | 0.111         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 7.25e+06      |\n",
      "|    n_updates            | 3530          |\n",
      "|    policy_gradient_loss | -0.000289     |\n",
      "|    value_loss           | 1.81e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 355           |\n",
      "|    time_elapsed         | 1268          |\n",
      "|    total_timesteps      | 727040        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 5.3113035e-06 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.68         |\n",
      "|    explained_variance   | 0.0648        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 2.63e+07      |\n",
      "|    n_updates            | 3540          |\n",
      "|    policy_gradient_loss | -0.000296     |\n",
      "|    value_loss           | 4.07e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 356           |\n",
      "|    time_elapsed         | 1272          |\n",
      "|    total_timesteps      | 729088        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00010453607 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.56         |\n",
      "|    explained_variance   | 0.351         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 6.7e+05       |\n",
      "|    n_updates            | 3550          |\n",
      "|    policy_gradient_loss | -0.0014       |\n",
      "|    value_loss           | 1.2e+06       |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 357           |\n",
      "|    time_elapsed         | 1275          |\n",
      "|    total_timesteps      | 731136        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.1633703e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.73         |\n",
      "|    explained_variance   | 0.0819        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.46e+07      |\n",
      "|    n_updates            | 3560          |\n",
      "|    policy_gradient_loss | -0.000427     |\n",
      "|    value_loss           | 2.1e+07       |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 358           |\n",
      "|    time_elapsed         | 1279          |\n",
      "|    total_timesteps      | 733184        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.3848534e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.6          |\n",
      "|    explained_variance   | 0.09          |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 9.99e+06      |\n",
      "|    n_updates            | 3570          |\n",
      "|    policy_gradient_loss | -0.00047      |\n",
      "|    value_loss           | 2.18e+07      |\n",
      "-------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 573         |\n",
      "|    iterations           | 359         |\n",
      "|    time_elapsed         | 1283        |\n",
      "|    total_timesteps      | 735232      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 8.12806e-06 |\n",
      "|    clip_fraction        | 0           |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.74       |\n",
      "|    explained_variance   | 0.0575      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 2.67e+07    |\n",
      "|    n_updates            | 3580        |\n",
      "|    policy_gradient_loss | -0.000421   |\n",
      "|    value_loss           | 3.18e+07    |\n",
      "-----------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 360           |\n",
      "|    time_elapsed         | 1286          |\n",
      "|    total_timesteps      | 737280        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 5.3748954e-06 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.61         |\n",
      "|    explained_variance   | 0.0909        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.44e+07      |\n",
      "|    n_updates            | 3590          |\n",
      "|    policy_gradient_loss | -0.000407     |\n",
      "|    value_loss           | 2.85e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 361           |\n",
      "|    time_elapsed         | 1290          |\n",
      "|    total_timesteps      | 739328        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 4.7274807e-06 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.69         |\n",
      "|    explained_variance   | 0.0522        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 2.98e+07      |\n",
      "|    n_updates            | 3600          |\n",
      "|    policy_gradient_loss | -0.000434     |\n",
      "|    value_loss           | 3.62e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 362           |\n",
      "|    time_elapsed         | 1293          |\n",
      "|    total_timesteps      | 741376        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 4.0388928e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.59         |\n",
      "|    explained_variance   | 0.0967        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.34e+07      |\n",
      "|    n_updates            | 3610          |\n",
      "|    policy_gradient_loss | -0.000729     |\n",
      "|    value_loss           | 2.32e+07      |\n",
      "-------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 363         |\n",
      "|    time_elapsed         | 1297        |\n",
      "|    total_timesteps      | 743424      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 6.61012e-06 |\n",
      "|    clip_fraction        | 0           |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.65       |\n",
      "|    explained_variance   | 0.0699      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 2.34e+07    |\n",
      "|    n_updates            | 3620        |\n",
      "|    policy_gradient_loss | -0.000393   |\n",
      "|    value_loss           | 4.79e+07    |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 364          |\n",
      "|    time_elapsed         | 1301         |\n",
      "|    total_timesteps      | 745472       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 7.931929e-06 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.69        |\n",
      "|    explained_variance   | 0.0479       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 2.62e+07     |\n",
      "|    n_updates            | 3630         |\n",
      "|    policy_gradient_loss | -0.00042     |\n",
      "|    value_loss           | 7.03e+07     |\n",
      "------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 365          |\n",
      "|    time_elapsed         | 1304         |\n",
      "|    total_timesteps      | 747520       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 3.412724e-05 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.63        |\n",
      "|    explained_variance   | 0.134        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.01e+07     |\n",
      "|    n_updates            | 3640         |\n",
      "|    policy_gradient_loss | -0.000844    |\n",
      "|    value_loss           | 1.78e+07     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 366           |\n",
      "|    time_elapsed         | 1308          |\n",
      "|    total_timesteps      | 749568        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 2.7114118e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.67         |\n",
      "|    explained_variance   | 0.0912        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.17e+07      |\n",
      "|    n_updates            | 3650          |\n",
      "|    policy_gradient_loss | -0.000825     |\n",
      "|    value_loss           | 2.26e+07      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 367          |\n",
      "|    time_elapsed         | 1311         |\n",
      "|    total_timesteps      | 751616       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 8.176733e-05 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.52        |\n",
      "|    explained_variance   | 0.12         |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 6.58e+06     |\n",
      "|    n_updates            | 3660         |\n",
      "|    policy_gradient_loss | -0.00116     |\n",
      "|    value_loss           | 1.69e+07     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 368           |\n",
      "|    time_elapsed         | 1315          |\n",
      "|    total_timesteps      | 753664        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 5.9293234e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.56         |\n",
      "|    explained_variance   | 0.0732        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 3.07e+07      |\n",
      "|    n_updates            | 3670          |\n",
      "|    policy_gradient_loss | -0.000988     |\n",
      "|    value_loss           | 4.11e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 369           |\n",
      "|    time_elapsed         | 1318          |\n",
      "|    total_timesteps      | 755712        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00015389733 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.6          |\n",
      "|    explained_variance   | 0.104         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 3.09e+06      |\n",
      "|    n_updates            | 3680          |\n",
      "|    policy_gradient_loss | -0.00215      |\n",
      "|    value_loss           | 1.02e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 573           |\n",
      "|    iterations           | 370           |\n",
      "|    time_elapsed         | 1322          |\n",
      "|    total_timesteps      | 757760        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 2.5067682e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.63         |\n",
      "|    explained_variance   | 0.0856        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.74e+07      |\n",
      "|    n_updates            | 3690          |\n",
      "|    policy_gradient_loss | -0.000743     |\n",
      "|    value_loss           | 4.21e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 371           |\n",
      "|    time_elapsed         | 1326          |\n",
      "|    total_timesteps      | 759808        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00063119456 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.57         |\n",
      "|    explained_variance   | 0.232         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 3.01e+06      |\n",
      "|    n_updates            | 3700          |\n",
      "|    policy_gradient_loss | -0.0033       |\n",
      "|    value_loss           | 3.91e+06      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 372          |\n",
      "|    time_elapsed         | 1329         |\n",
      "|    total_timesteps      | 761856       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0018591448 |\n",
      "|    clip_fraction        | 0.011        |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.45        |\n",
      "|    explained_variance   | 0.55         |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.48e+05     |\n",
      "|    n_updates            | 3710         |\n",
      "|    policy_gradient_loss | -0.00784     |\n",
      "|    value_loss           | 3.86e+05     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 373           |\n",
      "|    time_elapsed         | 1333          |\n",
      "|    total_timesteps      | 763904        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00034216838 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.39         |\n",
      "|    explained_variance   | 0.225         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.36e+06      |\n",
      "|    n_updates            | 3720          |\n",
      "|    policy_gradient_loss | -0.00311      |\n",
      "|    value_loss           | 4.49e+06      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 374          |\n",
      "|    time_elapsed         | 1336         |\n",
      "|    total_timesteps      | 765952       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0031432693 |\n",
      "|    clip_fraction        | 0.0137       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.31        |\n",
      "|    explained_variance   | 0.543        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 2.94e+04     |\n",
      "|    n_updates            | 3730         |\n",
      "|    policy_gradient_loss | -0.0108      |\n",
      "|    value_loss           | 8.81e+04     |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 375          |\n",
      "|    time_elapsed         | 1340         |\n",
      "|    total_timesteps      | 768000       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0022191897 |\n",
      "|    clip_fraction        | 0.00977      |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.37        |\n",
      "|    explained_variance   | 0.612        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 6.76e+04     |\n",
      "|    n_updates            | 3740         |\n",
      "|    policy_gradient_loss | -0.00818     |\n",
      "|    value_loss           | 1.48e+05     |\n",
      "------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 376         |\n",
      "|    time_elapsed         | 1344        |\n",
      "|    total_timesteps      | 770048      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.014165103 |\n",
      "|    clip_fraction        | 0.0878      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.27       |\n",
      "|    explained_variance   | -17.9       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 894         |\n",
      "|    n_updates            | 3750        |\n",
      "|    policy_gradient_loss | -0.0191     |\n",
      "|    value_loss           | 5.33e+03    |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 572        |\n",
      "|    iterations           | 377        |\n",
      "|    time_elapsed         | 1347       |\n",
      "|    total_timesteps      | 772096     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.02271467 |\n",
      "|    clip_fraction        | 0.158      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -2.32      |\n",
      "|    explained_variance   | -3.95      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 213        |\n",
      "|    n_updates            | 3760       |\n",
      "|    policy_gradient_loss | -0.0197    |\n",
      "|    value_loss           | 5.34e+03   |\n",
      "----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 572        |\n",
      "|    iterations           | 378        |\n",
      "|    time_elapsed         | 1351       |\n",
      "|    total_timesteps      | 774144     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.02924414 |\n",
      "|    clip_fraction        | 0.263      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -2.34      |\n",
      "|    explained_variance   | -29.2      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 15         |\n",
      "|    n_updates            | 3770       |\n",
      "|    policy_gradient_loss | -0.0338    |\n",
      "|    value_loss           | 91.9       |\n",
      "----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 572        |\n",
      "|    iterations           | 379        |\n",
      "|    time_elapsed         | 1354       |\n",
      "|    total_timesteps      | 776192     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.04028117 |\n",
      "|    clip_fraction        | 0.292      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -2.37      |\n",
      "|    explained_variance   | -64.5      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 0.466      |\n",
      "|    n_updates            | 3780       |\n",
      "|    policy_gradient_loss | -0.0329    |\n",
      "|    value_loss           | 5.14       |\n",
      "----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 572        |\n",
      "|    iterations           | 380        |\n",
      "|    time_elapsed         | 1358       |\n",
      "|    total_timesteps      | 778240     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.04052216 |\n",
      "|    clip_fraction        | 0.336      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -2.36      |\n",
      "|    explained_variance   | -2.35      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 0.033      |\n",
      "|    n_updates            | 3790       |\n",
      "|    policy_gradient_loss | -0.0347    |\n",
      "|    value_loss           | 0.742      |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 381         |\n",
      "|    time_elapsed         | 1362        |\n",
      "|    total_timesteps      | 780288      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.022119153 |\n",
      "|    clip_fraction        | 0.257       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.33       |\n",
      "|    explained_variance   | -0.0894     |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.284       |\n",
      "|    n_updates            | 3800        |\n",
      "|    policy_gradient_loss | -0.0265     |\n",
      "|    value_loss           | 0.948       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 382         |\n",
      "|    time_elapsed         | 1365        |\n",
      "|    total_timesteps      | 782336      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.036897045 |\n",
      "|    clip_fraction        | 0.276       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.34       |\n",
      "|    explained_variance   | -0.11       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.209       |\n",
      "|    n_updates            | 3810        |\n",
      "|    policy_gradient_loss | -0.0253     |\n",
      "|    value_loss           | 0.764       |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 572        |\n",
      "|    iterations           | 383        |\n",
      "|    time_elapsed         | 1369       |\n",
      "|    total_timesteps      | 784384     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.06729814 |\n",
      "|    clip_fraction        | 0.363      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -2.37      |\n",
      "|    explained_variance   | 0.0452     |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0659    |\n",
      "|    n_updates            | 3820       |\n",
      "|    policy_gradient_loss | -0.0442    |\n",
      "|    value_loss           | 0.0988     |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 384         |\n",
      "|    time_elapsed         | 1372        |\n",
      "|    total_timesteps      | 786432      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.016455889 |\n",
      "|    clip_fraction        | 0.149       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.24       |\n",
      "|    explained_variance   | -0.0174     |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 1.83        |\n",
      "|    n_updates            | 3830        |\n",
      "|    policy_gradient_loss | -0.0318     |\n",
      "|    value_loss           | 2.78        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 385         |\n",
      "|    time_elapsed         | 1376        |\n",
      "|    total_timesteps      | 788480      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.011211475 |\n",
      "|    clip_fraction        | 0.0429      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.27       |\n",
      "|    explained_variance   | -1.44       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 4.63e+03    |\n",
      "|    n_updates            | 3840        |\n",
      "|    policy_gradient_loss | -0.00746    |\n",
      "|    value_loss           | 4.87e+03    |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 386         |\n",
      "|    time_elapsed         | 1379        |\n",
      "|    total_timesteps      | 790528      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.005571103 |\n",
      "|    clip_fraction        | 0.0364      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.26       |\n",
      "|    explained_variance   | -27.7       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 50.9        |\n",
      "|    n_updates            | 3850        |\n",
      "|    policy_gradient_loss | -0.00816    |\n",
      "|    value_loss           | 2.22e+03    |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 572        |\n",
      "|    iterations           | 387        |\n",
      "|    time_elapsed         | 1383       |\n",
      "|    total_timesteps      | 792576     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.03882616 |\n",
      "|    clip_fraction        | 0.246      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -2.31      |\n",
      "|    explained_variance   | -1.66      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0624    |\n",
      "|    n_updates            | 3860       |\n",
      "|    policy_gradient_loss | -0.0378    |\n",
      "|    value_loss           | 0.305      |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 388         |\n",
      "|    time_elapsed         | 1387        |\n",
      "|    total_timesteps      | 794624      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.019979086 |\n",
      "|    clip_fraction        | 0.255       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.3        |\n",
      "|    explained_variance   | -0.00387    |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.18        |\n",
      "|    n_updates            | 3870        |\n",
      "|    policy_gradient_loss | -0.0184     |\n",
      "|    value_loss           | 0.431       |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 572        |\n",
      "|    iterations           | 389        |\n",
      "|    time_elapsed         | 1390       |\n",
      "|    total_timesteps      | 796672     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.02509415 |\n",
      "|    clip_fraction        | 0.256      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -2.28      |\n",
      "|    explained_variance   | 0.0328     |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 0.0731     |\n",
      "|    n_updates            | 3880       |\n",
      "|    policy_gradient_loss | -0.0229    |\n",
      "|    value_loss           | 0.262      |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 390         |\n",
      "|    time_elapsed         | 1394        |\n",
      "|    total_timesteps      | 798720      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.016956983 |\n",
      "|    clip_fraction        | 0.193       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.28       |\n",
      "|    explained_variance   | -27.9       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.00365    |\n",
      "|    n_updates            | 3890        |\n",
      "|    policy_gradient_loss | -0.0183     |\n",
      "|    value_loss           | 639         |\n",
      "-----------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 391           |\n",
      "|    time_elapsed         | 1397          |\n",
      "|    total_timesteps      | 800768        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00013912487 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.35         |\n",
      "|    explained_variance   | 0.673         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.68e+04      |\n",
      "|    n_updates            | 3900          |\n",
      "|    policy_gradient_loss | -0.00013      |\n",
      "|    value_loss           | 3.99e+04      |\n",
      "-------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 392         |\n",
      "|    time_elapsed         | 1401        |\n",
      "|    total_timesteps      | 802816      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.012711657 |\n",
      "|    clip_fraction        | 0.0842      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.3        |\n",
      "|    explained_variance   | -22         |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.314       |\n",
      "|    n_updates            | 3910        |\n",
      "|    policy_gradient_loss | -0.0118     |\n",
      "|    value_loss           | 889         |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 393         |\n",
      "|    time_elapsed         | 1405        |\n",
      "|    total_timesteps      | 804864      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.050913163 |\n",
      "|    clip_fraction        | 0.298       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.25       |\n",
      "|    explained_variance   | -7.66       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0844     |\n",
      "|    n_updates            | 3920        |\n",
      "|    policy_gradient_loss | -0.0486     |\n",
      "|    value_loss           | 0.625       |\n",
      "-----------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 394           |\n",
      "|    time_elapsed         | 1408          |\n",
      "|    total_timesteps      | 806912        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00049430435 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.19         |\n",
      "|    explained_variance   | 0.739         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.09e+04      |\n",
      "|    n_updates            | 3930          |\n",
      "|    policy_gradient_loss | -0.000394     |\n",
      "|    value_loss           | 2.05e+04      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 395          |\n",
      "|    time_elapsed         | 1412         |\n",
      "|    total_timesteps      | 808960       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0134503385 |\n",
      "|    clip_fraction        | 0.111        |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.17        |\n",
      "|    explained_variance   | -26.7        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 0.346        |\n",
      "|    n_updates            | 3940         |\n",
      "|    policy_gradient_loss | -0.0197      |\n",
      "|    value_loss           | 932          |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 396         |\n",
      "|    time_elapsed         | 1416        |\n",
      "|    total_timesteps      | 811008      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.038359314 |\n",
      "|    clip_fraction        | 0.293       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.14       |\n",
      "|    explained_variance   | -3.7        |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0225     |\n",
      "|    n_updates            | 3950        |\n",
      "|    policy_gradient_loss | -0.0445     |\n",
      "|    value_loss           | 0.289       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 397         |\n",
      "|    time_elapsed         | 1419        |\n",
      "|    total_timesteps      | 813056      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.018478548 |\n",
      "|    clip_fraction        | 0.16        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.13       |\n",
      "|    explained_variance   | -27.4       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 24.7        |\n",
      "|    n_updates            | 3960        |\n",
      "|    policy_gradient_loss | -0.0124     |\n",
      "|    value_loss           | 869         |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 572        |\n",
      "|    iterations           | 398        |\n",
      "|    time_elapsed         | 1423       |\n",
      "|    total_timesteps      | 815104     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.05813299 |\n",
      "|    clip_fraction        | 0.356      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -2.07      |\n",
      "|    explained_variance   | -0.387     |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.102     |\n",
      "|    n_updates            | 3970       |\n",
      "|    policy_gradient_loss | -0.0633    |\n",
      "|    value_loss           | 0.00593    |\n",
      "----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 399          |\n",
      "|    time_elapsed         | 1426         |\n",
      "|    total_timesteps      | 817152       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0004899625 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.73        |\n",
      "|    explained_variance   | 0.0767       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 2.2e+07      |\n",
      "|    n_updates            | 3980         |\n",
      "|    policy_gradient_loss | -0.000521    |\n",
      "|    value_loss           | 3.41e+07     |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 400          |\n",
      "|    time_elapsed         | 1430         |\n",
      "|    total_timesteps      | 819200       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 8.061761e-09 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.71        |\n",
      "|    explained_variance   | 0.0593       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 2.33e+07     |\n",
      "|    n_updates            | 3990         |\n",
      "|    policy_gradient_loss | -1.69e-05    |\n",
      "|    value_loss           | 3.38e+07     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 401           |\n",
      "|    time_elapsed         | 1433          |\n",
      "|    total_timesteps      | 821248        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.5978003e-08 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.8          |\n",
      "|    explained_variance   | 0.063         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.61e+07      |\n",
      "|    n_updates            | 4000          |\n",
      "|    policy_gradient_loss | -2.31e-05     |\n",
      "|    value_loss           | 3.23e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 402           |\n",
      "|    time_elapsed         | 1437          |\n",
      "|    total_timesteps      | 823296        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.5244586e-07 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.58         |\n",
      "|    explained_variance   | 0.0795        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 7.67e+06      |\n",
      "|    n_updates            | 4010          |\n",
      "|    policy_gradient_loss | -4.78e-05     |\n",
      "|    value_loss           | 1.43e+07      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 403          |\n",
      "|    time_elapsed         | 1441         |\n",
      "|    total_timesteps      | 825344       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 3.853347e-08 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.67        |\n",
      "|    explained_variance   | 0.0883       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 7.61e+06     |\n",
      "|    n_updates            | 4020         |\n",
      "|    policy_gradient_loss | -3.6e-05     |\n",
      "|    value_loss           | 2.69e+07     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 404           |\n",
      "|    time_elapsed         | 1444          |\n",
      "|    total_timesteps      | 827392        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 2.7139322e-07 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.48         |\n",
      "|    explained_variance   | 0.116         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 5.06e+06      |\n",
      "|    n_updates            | 4030          |\n",
      "|    policy_gradient_loss | -7.71e-05     |\n",
      "|    value_loss           | 1.17e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 405           |\n",
      "|    time_elapsed         | 1448          |\n",
      "|    total_timesteps      | 829440        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.5681144e-07 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.67         |\n",
      "|    explained_variance   | 0.0836        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 9.94e+06      |\n",
      "|    n_updates            | 4040          |\n",
      "|    policy_gradient_loss | -6.05e-05     |\n",
      "|    value_loss           | 2.53e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 406           |\n",
      "|    time_elapsed         | 1451          |\n",
      "|    total_timesteps      | 831488        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.4156103e-07 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.78         |\n",
      "|    explained_variance   | 0.0735        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 2.39e+07      |\n",
      "|    n_updates            | 4050          |\n",
      "|    policy_gradient_loss | -5.09e-05     |\n",
      "|    value_loss           | 3.99e+07      |\n",
      "-------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 407         |\n",
      "|    time_elapsed         | 1455        |\n",
      "|    total_timesteps      | 833536      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 1.42731e-06 |\n",
      "|    clip_fraction        | 0           |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.53       |\n",
      "|    explained_variance   | 0.208       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 2.2e+06     |\n",
      "|    n_updates            | 4060        |\n",
      "|    policy_gradient_loss | -0.000188   |\n",
      "|    value_loss           | 5.22e+06    |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 408          |\n",
      "|    time_elapsed         | 1458         |\n",
      "|    total_timesteps      | 835584       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 8.349598e-07 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.47        |\n",
      "|    explained_variance   | 0.0977       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 5.84e+06     |\n",
      "|    n_updates            | 4070         |\n",
      "|    policy_gradient_loss | -0.000141    |\n",
      "|    value_loss           | 1.92e+07     |\n",
      "------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 409         |\n",
      "|    time_elapsed         | 1462        |\n",
      "|    total_timesteps      | 837632      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 9.04256e-08 |\n",
      "|    clip_fraction        | 0           |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.78       |\n",
      "|    explained_variance   | 0.0507      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 4.23e+07    |\n",
      "|    n_updates            | 4080        |\n",
      "|    policy_gradient_loss | -5.12e-05   |\n",
      "|    value_loss           | 7.45e+07    |\n",
      "-----------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 410           |\n",
      "|    time_elapsed         | 1466          |\n",
      "|    total_timesteps      | 839680        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 3.1217933e-06 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.67         |\n",
      "|    explained_variance   | 0.141         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 3.32e+06      |\n",
      "|    n_updates            | 4090          |\n",
      "|    policy_gradient_loss | -0.000239     |\n",
      "|    value_loss           | 1.07e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 411           |\n",
      "|    time_elapsed         | 1469          |\n",
      "|    total_timesteps      | 841728        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.7391867e-06 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.6          |\n",
      "|    explained_variance   | 0.132         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.23e+07      |\n",
      "|    n_updates            | 4100          |\n",
      "|    policy_gradient_loss | -0.000192     |\n",
      "|    value_loss           | 1.69e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 412           |\n",
      "|    time_elapsed         | 1473          |\n",
      "|    total_timesteps      | 843776        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.6268692e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.54         |\n",
      "|    explained_variance   | 0.327         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.35e+06      |\n",
      "|    n_updates            | 4110          |\n",
      "|    policy_gradient_loss | -0.000598     |\n",
      "|    value_loss           | 2.27e+06      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 413           |\n",
      "|    time_elapsed         | 1476          |\n",
      "|    total_timesteps      | 845824        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 2.8961222e-06 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.6          |\n",
      "|    explained_variance   | 0.161         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 5.12e+06      |\n",
      "|    n_updates            | 4120          |\n",
      "|    policy_gradient_loss | -0.000268     |\n",
      "|    value_loss           | 1.24e+07      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 414          |\n",
      "|    time_elapsed         | 1480         |\n",
      "|    total_timesteps      | 847872       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 4.680216e-06 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.51        |\n",
      "|    explained_variance   | 0.131        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1e+07        |\n",
      "|    n_updates            | 4130         |\n",
      "|    policy_gradient_loss | -0.00033     |\n",
      "|    value_loss           | 1.76e+07     |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 415          |\n",
      "|    time_elapsed         | 1484         |\n",
      "|    total_timesteps      | 849920       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 7.924362e-06 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.55        |\n",
      "|    explained_variance   | 0.141        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 7.53e+06     |\n",
      "|    n_updates            | 4140         |\n",
      "|    policy_gradient_loss | -0.000418    |\n",
      "|    value_loss           | 1.52e+07     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 416           |\n",
      "|    time_elapsed         | 1487          |\n",
      "|    total_timesteps      | 851968        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 4.9725582e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.64         |\n",
      "|    explained_variance   | 0.221         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 2.25e+06      |\n",
      "|    n_updates            | 4150          |\n",
      "|    policy_gradient_loss | -0.00105      |\n",
      "|    value_loss           | 5.3e+06       |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 417           |\n",
      "|    time_elapsed         | 1491          |\n",
      "|    total_timesteps      | 854016        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.4701218e-06 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.53         |\n",
      "|    explained_variance   | 0.063         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 4.4e+07       |\n",
      "|    n_updates            | 4160          |\n",
      "|    policy_gradient_loss | -0.000185     |\n",
      "|    value_loss           | 9.64e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 418           |\n",
      "|    time_elapsed         | 1494          |\n",
      "|    total_timesteps      | 856064        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 5.3074036e-06 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.6          |\n",
      "|    explained_variance   | 0.0905        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.19e+07      |\n",
      "|    n_updates            | 4170          |\n",
      "|    policy_gradient_loss | -0.000354     |\n",
      "|    value_loss           | 3.1e+07       |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 419          |\n",
      "|    time_elapsed         | 1498         |\n",
      "|    total_timesteps      | 858112       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 1.979174e-05 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.45        |\n",
      "|    explained_variance   | 0.191        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 5.13e+06     |\n",
      "|    n_updates            | 4180         |\n",
      "|    policy_gradient_loss | -0.000709    |\n",
      "|    value_loss           | 1.01e+07     |\n",
      "------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 420           |\n",
      "|    time_elapsed         | 1501          |\n",
      "|    total_timesteps      | 860160        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 7.0127426e-06 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.56         |\n",
      "|    explained_variance   | 0.0915        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 9.94e+06      |\n",
      "|    n_updates            | 4190          |\n",
      "|    policy_gradient_loss | -0.000408     |\n",
      "|    value_loss           | 5.4e+07       |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 421           |\n",
      "|    time_elapsed         | 1505          |\n",
      "|    total_timesteps      | 862208        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 5.4416887e-06 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.58         |\n",
      "|    explained_variance   | 0.0785        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 2.01e+07      |\n",
      "|    n_updates            | 4200          |\n",
      "|    policy_gradient_loss | -0.000423     |\n",
      "|    value_loss           | 3.69e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 422           |\n",
      "|    time_elapsed         | 1509          |\n",
      "|    total_timesteps      | 864256        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 2.1246873e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.45         |\n",
      "|    explained_variance   | 0.134         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 3.94e+06      |\n",
      "|    n_updates            | 4210          |\n",
      "|    policy_gradient_loss | -0.000908     |\n",
      "|    value_loss           | 1.6e+07       |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 423           |\n",
      "|    time_elapsed         | 1512          |\n",
      "|    total_timesteps      | 866304        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.2664095e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.39         |\n",
      "|    explained_variance   | 0.13          |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.75e+06      |\n",
      "|    n_updates            | 4220          |\n",
      "|    policy_gradient_loss | -0.000662     |\n",
      "|    value_loss           | 2.06e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 424           |\n",
      "|    time_elapsed         | 1516          |\n",
      "|    total_timesteps      | 868352        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 3.2139476e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.5          |\n",
      "|    explained_variance   | 0.147         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 9.88e+06      |\n",
      "|    n_updates            | 4230          |\n",
      "|    policy_gradient_loss | -0.000876     |\n",
      "|    value_loss           | 2.13e+07      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 425          |\n",
      "|    time_elapsed         | 1519         |\n",
      "|    total_timesteps      | 870400       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 1.111679e-05 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.57        |\n",
      "|    explained_variance   | 0.112        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.42e+07     |\n",
      "|    n_updates            | 4240         |\n",
      "|    policy_gradient_loss | -0.000531    |\n",
      "|    value_loss           | 3.57e+07     |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 426          |\n",
      "|    time_elapsed         | 1523         |\n",
      "|    total_timesteps      | 872448       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 2.674904e-05 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.68        |\n",
      "|    explained_variance   | 0.0607       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 2.66e+07     |\n",
      "|    n_updates            | 4250         |\n",
      "|    policy_gradient_loss | -0.000655    |\n",
      "|    value_loss           | 5.05e+07     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 427           |\n",
      "|    time_elapsed         | 1526          |\n",
      "|    total_timesteps      | 874496        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.8834835e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.44         |\n",
      "|    explained_variance   | 0.101         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 2.22e+07      |\n",
      "|    n_updates            | 4260          |\n",
      "|    policy_gradient_loss | -0.000749     |\n",
      "|    value_loss           | 3.45e+07      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 428           |\n",
      "|    time_elapsed         | 1530          |\n",
      "|    total_timesteps      | 876544        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00040206054 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.4          |\n",
      "|    explained_variance   | 0.257         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.88e+06      |\n",
      "|    n_updates            | 4270          |\n",
      "|    policy_gradient_loss | -0.00293      |\n",
      "|    value_loss           | 3.49e+06      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 429           |\n",
      "|    time_elapsed         | 1534          |\n",
      "|    total_timesteps      | 878592        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00016490408 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.21         |\n",
      "|    explained_variance   | 0.119         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 3.45e+06      |\n",
      "|    n_updates            | 4280          |\n",
      "|    policy_gradient_loss | -0.00195      |\n",
      "|    value_loss           | 1.7e+07       |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 430          |\n",
      "|    time_elapsed         | 1537         |\n",
      "|    total_timesteps      | 880640       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0022963295 |\n",
      "|    clip_fraction        | 0.0123       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.28        |\n",
      "|    explained_variance   | 0.657        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.17e+05     |\n",
      "|    n_updates            | 4290         |\n",
      "|    policy_gradient_loss | -0.00983     |\n",
      "|    value_loss           | 2.98e+05     |\n",
      "------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 431          |\n",
      "|    time_elapsed         | 1541         |\n",
      "|    total_timesteps      | 882688       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0020473138 |\n",
      "|    clip_fraction        | 0.0148       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.11        |\n",
      "|    explained_variance   | 0.64         |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 3.18e+04     |\n",
      "|    n_updates            | 4300         |\n",
      "|    policy_gradient_loss | -0.00944     |\n",
      "|    value_loss           | 1.26e+05     |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 432          |\n",
      "|    time_elapsed         | 1544         |\n",
      "|    total_timesteps      | 884736       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0016235145 |\n",
      "|    clip_fraction        | 0.00728      |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.06        |\n",
      "|    explained_variance   | 0.68         |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 2.22e+04     |\n",
      "|    n_updates            | 4310         |\n",
      "|    policy_gradient_loss | -0.00758     |\n",
      "|    value_loss           | 5.64e+04     |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 433          |\n",
      "|    time_elapsed         | 1548         |\n",
      "|    total_timesteps      | 886784       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0075331577 |\n",
      "|    clip_fraction        | 0.024        |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.04        |\n",
      "|    explained_variance   | -18.1        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.25e+03     |\n",
      "|    n_updates            | 4320         |\n",
      "|    policy_gradient_loss | -0.0104      |\n",
      "|    value_loss           | 1.4e+04      |\n",
      "------------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 572        |\n",
      "|    iterations           | 434        |\n",
      "|    time_elapsed         | 1552       |\n",
      "|    total_timesteps      | 888832     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.02044918 |\n",
      "|    clip_fraction        | 0.147      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -2.06      |\n",
      "|    explained_variance   | -28.9      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 113        |\n",
      "|    n_updates            | 4330       |\n",
      "|    policy_gradient_loss | -0.0215    |\n",
      "|    value_loss           | 3.2e+03    |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 435         |\n",
      "|    time_elapsed         | 1555        |\n",
      "|    total_timesteps      | 890880      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.042412814 |\n",
      "|    clip_fraction        | 0.274       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.09       |\n",
      "|    explained_variance   | -25.5       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.447       |\n",
      "|    n_updates            | 4340        |\n",
      "|    policy_gradient_loss | -0.0316     |\n",
      "|    value_loss           | 73.6        |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 572        |\n",
      "|    iterations           | 436        |\n",
      "|    time_elapsed         | 1559       |\n",
      "|    total_timesteps      | 892928     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.04691811 |\n",
      "|    clip_fraction        | 0.341      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -2.06      |\n",
      "|    explained_variance   | -2.48      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 0.473      |\n",
      "|    n_updates            | 4350       |\n",
      "|    policy_gradient_loss | -0.0348    |\n",
      "|    value_loss           | 1.87       |\n",
      "----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 572        |\n",
      "|    iterations           | 437        |\n",
      "|    time_elapsed         | 1562       |\n",
      "|    total_timesteps      | 894976     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.04352416 |\n",
      "|    clip_fraction        | 0.357      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -2.07      |\n",
      "|    explained_variance   | -1.54      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0945    |\n",
      "|    n_updates            | 4360       |\n",
      "|    policy_gradient_loss | -0.0437    |\n",
      "|    value_loss           | 0.4        |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 438         |\n",
      "|    time_elapsed         | 1566        |\n",
      "|    total_timesteps      | 897024      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.020057617 |\n",
      "|    clip_fraction        | 0.225       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.05       |\n",
      "|    explained_variance   | -0.0138     |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.248       |\n",
      "|    n_updates            | 4370        |\n",
      "|    policy_gradient_loss | -0.0171     |\n",
      "|    value_loss           | 0.595       |\n",
      "-----------------------------------------\n",
      "---------------------------------------\n",
      "| time/                   |           |\n",
      "|    fps                  | 572       |\n",
      "|    iterations           | 439       |\n",
      "|    time_elapsed         | 1570      |\n",
      "|    total_timesteps      | 899072    |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.0158707 |\n",
      "|    clip_fraction        | 0.14      |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -2.1      |\n",
      "|    explained_variance   | 0.182     |\n",
      "|    learning_rate        | 0.0003    |\n",
      "|    loss                 | 1.5e+03   |\n",
      "|    n_updates            | 4380      |\n",
      "|    policy_gradient_loss | 0.00054   |\n",
      "|    value_loss           | 8.26e+03  |\n",
      "---------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 440          |\n",
      "|    time_elapsed         | 1573         |\n",
      "|    total_timesteps      | 901120       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0026078501 |\n",
      "|    clip_fraction        | 0.0231       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.11        |\n",
      "|    explained_variance   | -0.145       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 835          |\n",
      "|    n_updates            | 4390         |\n",
      "|    policy_gradient_loss | -0.00257     |\n",
      "|    value_loss           | 9.9e+03      |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 441         |\n",
      "|    time_elapsed         | 1577        |\n",
      "|    total_timesteps      | 903168      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.022917602 |\n",
      "|    clip_fraction        | 0.183       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.08       |\n",
      "|    explained_variance   | -27.7       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 86.9        |\n",
      "|    n_updates            | 4400        |\n",
      "|    policy_gradient_loss | -0.00431    |\n",
      "|    value_loss           | 821         |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 442          |\n",
      "|    time_elapsed         | 1580         |\n",
      "|    total_timesteps      | 905216       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0006188791 |\n",
      "|    clip_fraction        | 0.00342      |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.32        |\n",
      "|    explained_variance   | 0.577        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.44e+05     |\n",
      "|    n_updates            | 4410         |\n",
      "|    policy_gradient_loss | -0.00377     |\n",
      "|    value_loss           | 2.01e+05     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 443           |\n",
      "|    time_elapsed         | 1584          |\n",
      "|    total_timesteps      | 907264        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.7936138e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.28         |\n",
      "|    explained_variance   | 0.671         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 9.05e+04      |\n",
      "|    n_updates            | 4420          |\n",
      "|    policy_gradient_loss | -0.000754     |\n",
      "|    value_loss           | 2.07e+05      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 444           |\n",
      "|    time_elapsed         | 1587          |\n",
      "|    total_timesteps      | 909312        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 3.0300085e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.25         |\n",
      "|    explained_variance   | 0.225         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 3.33e+05      |\n",
      "|    n_updates            | 4430          |\n",
      "|    policy_gradient_loss | -0.000985     |\n",
      "|    value_loss           | 3.35e+05      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 445          |\n",
      "|    time_elapsed         | 1591         |\n",
      "|    total_timesteps      | 911360       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 3.098545e-05 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.17        |\n",
      "|    explained_variance   | 0.637        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.13e+05     |\n",
      "|    n_updates            | 4440         |\n",
      "|    policy_gradient_loss | -0.00103     |\n",
      "|    value_loss           | 1.37e+05     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 446           |\n",
      "|    time_elapsed         | 1595          |\n",
      "|    total_timesteps      | 913408        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00082335755 |\n",
      "|    clip_fraction        | 0.00322       |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.14         |\n",
      "|    explained_variance   | -0.151        |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 6.52e+03      |\n",
      "|    n_updates            | 4450          |\n",
      "|    policy_gradient_loss | -0.00627      |\n",
      "|    value_loss           | 2.43e+04      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 447          |\n",
      "|    time_elapsed         | 1598         |\n",
      "|    total_timesteps      | 915456       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 8.385547e-05 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.16        |\n",
      "|    explained_variance   | 0.703        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 9.25e+04     |\n",
      "|    n_updates            | 4460         |\n",
      "|    policy_gradient_loss | -0.00174     |\n",
      "|    value_loss           | 8.97e+04     |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 448          |\n",
      "|    time_elapsed         | 1602         |\n",
      "|    total_timesteps      | 917504       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0042394884 |\n",
      "|    clip_fraction        | 0.0277       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.09        |\n",
      "|    explained_variance   | -6.07        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 67.2         |\n",
      "|    n_updates            | 4470         |\n",
      "|    policy_gradient_loss | -0.00967     |\n",
      "|    value_loss           | 1.04e+04     |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 449         |\n",
      "|    time_elapsed         | 1605        |\n",
      "|    total_timesteps      | 919552      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.023607936 |\n",
      "|    clip_fraction        | 0.194       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.05       |\n",
      "|    explained_variance   | -24.5       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 1.38        |\n",
      "|    n_updates            | 4480        |\n",
      "|    policy_gradient_loss | -0.0258     |\n",
      "|    value_loss           | 936         |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 450         |\n",
      "|    time_elapsed         | 1609        |\n",
      "|    total_timesteps      | 921600      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.003342925 |\n",
      "|    clip_fraction        | 0.00913     |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.07       |\n",
      "|    explained_variance   | 0.368       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 4.98e+03    |\n",
      "|    n_updates            | 4490        |\n",
      "|    policy_gradient_loss | -0.00201    |\n",
      "|    value_loss           | 3.84e+04    |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 572        |\n",
      "|    iterations           | 451        |\n",
      "|    time_elapsed         | 1613       |\n",
      "|    total_timesteps      | 923648     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.04270257 |\n",
      "|    clip_fraction        | 0.296      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.99      |\n",
      "|    explained_variance   | -12        |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0436    |\n",
      "|    n_updates            | 4500       |\n",
      "|    policy_gradient_loss | -0.0525    |\n",
      "|    value_loss           | 0.474      |\n",
      "----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 572        |\n",
      "|    iterations           | 452        |\n",
      "|    time_elapsed         | 1616       |\n",
      "|    total_timesteps      | 925696     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.03763808 |\n",
      "|    clip_fraction        | 0.291      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.98      |\n",
      "|    explained_variance   | -0.105     |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 0.269      |\n",
      "|    n_updates            | 4510       |\n",
      "|    policy_gradient_loss | -0.0164    |\n",
      "|    value_loss           | 1.62       |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 572        |\n",
      "|    iterations           | 453        |\n",
      "|    time_elapsed         | 1620       |\n",
      "|    total_timesteps      | 927744     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.05896184 |\n",
      "|    clip_fraction        | 0.345      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.92      |\n",
      "|    explained_variance   | -0.684     |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0826    |\n",
      "|    n_updates            | 4520       |\n",
      "|    policy_gradient_loss | -0.0536    |\n",
      "|    value_loss           | 0.0306     |\n",
      "----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 572        |\n",
      "|    iterations           | 454        |\n",
      "|    time_elapsed         | 1623       |\n",
      "|    total_timesteps      | 929792     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.04031845 |\n",
      "|    clip_fraction        | 0.298      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.77      |\n",
      "|    explained_variance   | 0.506      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0815    |\n",
      "|    n_updates            | 4530       |\n",
      "|    policy_gradient_loss | -0.0484    |\n",
      "|    value_loss           | 0.0116     |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 455         |\n",
      "|    time_elapsed         | 1627        |\n",
      "|    total_timesteps      | 931840      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.011496555 |\n",
      "|    clip_fraction        | 0.131       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.62       |\n",
      "|    explained_variance   | -0.0463     |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 1.57        |\n",
      "|    n_updates            | 4540        |\n",
      "|    policy_gradient_loss | -0.0299     |\n",
      "|    value_loss           | 3.51        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 456         |\n",
      "|    time_elapsed         | 1631        |\n",
      "|    total_timesteps      | 933888      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.017840765 |\n",
      "|    clip_fraction        | 0.125       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.65       |\n",
      "|    explained_variance   | -0.0481     |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 1.24        |\n",
      "|    n_updates            | 4550        |\n",
      "|    policy_gradient_loss | -0.036      |\n",
      "|    value_loss           | 2.45        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 457         |\n",
      "|    time_elapsed         | 1634        |\n",
      "|    total_timesteps      | 935936      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.015604507 |\n",
      "|    clip_fraction        | 0.103       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.72       |\n",
      "|    explained_variance   | -0.223      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.737       |\n",
      "|    n_updates            | 4560        |\n",
      "|    policy_gradient_loss | -0.0191     |\n",
      "|    value_loss           | 0.942       |\n",
      "-----------------------------------------\n",
      "---------------------------------------\n",
      "| time/                   |           |\n",
      "|    fps                  | 572       |\n",
      "|    iterations           | 458       |\n",
      "|    time_elapsed         | 1638      |\n",
      "|    total_timesteps      | 937984    |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.0175471 |\n",
      "|    clip_fraction        | 0.184     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -1.77     |\n",
      "|    explained_variance   | -0.205    |\n",
      "|    learning_rate        | 0.0003    |\n",
      "|    loss                 | -0.043    |\n",
      "|    n_updates            | 4570      |\n",
      "|    policy_gradient_loss | -0.0236   |\n",
      "|    value_loss           | 0.184     |\n",
      "---------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 572        |\n",
      "|    iterations           | 459        |\n",
      "|    time_elapsed         | 1641       |\n",
      "|    total_timesteps      | 940032     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.06398996 |\n",
      "|    clip_fraction        | 0.358      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.78      |\n",
      "|    explained_variance   | 0.161      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.107     |\n",
      "|    n_updates            | 4580       |\n",
      "|    policy_gradient_loss | -0.0626    |\n",
      "|    value_loss           | 0.00514    |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 460         |\n",
      "|    time_elapsed         | 1645        |\n",
      "|    total_timesteps      | 942080      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.009416783 |\n",
      "|    clip_fraction        | 0.094       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.6        |\n",
      "|    explained_variance   | -0.00453    |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 2.03        |\n",
      "|    n_updates            | 4590        |\n",
      "|    policy_gradient_loss | -0.02       |\n",
      "|    value_loss           | 31.1        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 461         |\n",
      "|    time_elapsed         | 1649        |\n",
      "|    total_timesteps      | 944128      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.028811254 |\n",
      "|    clip_fraction        | 0.108       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.62       |\n",
      "|    explained_variance   | 0.0486      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 1.56        |\n",
      "|    n_updates            | 4600        |\n",
      "|    policy_gradient_loss | -0.0315     |\n",
      "|    value_loss           | 2.91        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 462         |\n",
      "|    time_elapsed         | 1652        |\n",
      "|    total_timesteps      | 946176      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.012182104 |\n",
      "|    clip_fraction        | 0.0997      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.68       |\n",
      "|    explained_variance   | -0.317      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.839       |\n",
      "|    n_updates            | 4610        |\n",
      "|    policy_gradient_loss | -0.0187     |\n",
      "|    value_loss           | 0.978       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 463         |\n",
      "|    time_elapsed         | 1656        |\n",
      "|    total_timesteps      | 948224      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.015194731 |\n",
      "|    clip_fraction        | 0.117       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.72       |\n",
      "|    explained_variance   | -0.00195    |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.284       |\n",
      "|    n_updates            | 4620        |\n",
      "|    policy_gradient_loss | -0.0109     |\n",
      "|    value_loss           | 53.3        |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 464         |\n",
      "|    time_elapsed         | 1659        |\n",
      "|    total_timesteps      | 950272      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.025692526 |\n",
      "|    clip_fraction        | 0.251       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.73       |\n",
      "|    explained_variance   | -0.11       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.0804      |\n",
      "|    n_updates            | 4630        |\n",
      "|    policy_gradient_loss | -0.0213     |\n",
      "|    value_loss           | 0.263       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 465         |\n",
      "|    time_elapsed         | 1663        |\n",
      "|    total_timesteps      | 952320      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.019363549 |\n",
      "|    clip_fraction        | 0.217       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.75       |\n",
      "|    explained_variance   | 0.0686      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.223       |\n",
      "|    n_updates            | 4640        |\n",
      "|    policy_gradient_loss | -0.0239     |\n",
      "|    value_loss           | 0.261       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 466         |\n",
      "|    time_elapsed         | 1667        |\n",
      "|    total_timesteps      | 954368      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.035770997 |\n",
      "|    clip_fraction        | 0.318       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.75       |\n",
      "|    explained_variance   | 0.645       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.088      |\n",
      "|    n_updates            | 4650        |\n",
      "|    policy_gradient_loss | -0.0506     |\n",
      "|    value_loss           | 0.00252     |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 467         |\n",
      "|    time_elapsed         | 1670        |\n",
      "|    total_timesteps      | 956416      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.012344031 |\n",
      "|    clip_fraction        | 0.106       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.76       |\n",
      "|    explained_variance   | -27.3       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 13.2        |\n",
      "|    n_updates            | 4660        |\n",
      "|    policy_gradient_loss | -0.00875    |\n",
      "|    value_loss           | 831         |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 572        |\n",
      "|    iterations           | 468        |\n",
      "|    time_elapsed         | 1674       |\n",
      "|    total_timesteps      | 958464     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.02725949 |\n",
      "|    clip_fraction        | 0.229      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.81      |\n",
      "|    explained_variance   | 0.14       |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0747    |\n",
      "|    n_updates            | 4670       |\n",
      "|    policy_gradient_loss | -0.0258    |\n",
      "|    value_loss           | 0.0891     |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 469         |\n",
      "|    time_elapsed         | 1677        |\n",
      "|    total_timesteps      | 960512      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.041476954 |\n",
      "|    clip_fraction        | 0.308       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.82       |\n",
      "|    explained_variance   | 0.943       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.037      |\n",
      "|    n_updates            | 4680        |\n",
      "|    policy_gradient_loss | -0.0439     |\n",
      "|    value_loss           | 0.0018      |\n",
      "-----------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 470           |\n",
      "|    time_elapsed         | 1681          |\n",
      "|    total_timesteps      | 962560        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00042590807 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -1.86         |\n",
      "|    explained_variance   | 0.696         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 9.88e+03      |\n",
      "|    n_updates            | 4690          |\n",
      "|    policy_gradient_loss | -0.000226     |\n",
      "|    value_loss           | 4.62e+04      |\n",
      "-------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 471         |\n",
      "|    time_elapsed         | 1685        |\n",
      "|    total_timesteps      | 964608      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.022834547 |\n",
      "|    clip_fraction        | 0.132       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.81       |\n",
      "|    explained_variance   | -3.61       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.108       |\n",
      "|    n_updates            | 4700        |\n",
      "|    policy_gradient_loss | -0.0261     |\n",
      "|    value_loss           | 10          |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 472          |\n",
      "|    time_elapsed         | 1688         |\n",
      "|    total_timesteps      | 966656       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0028446545 |\n",
      "|    clip_fraction        | 0.0128       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -1.78        |\n",
      "|    explained_variance   | -27.9        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 39.5         |\n",
      "|    n_updates            | 4710         |\n",
      "|    policy_gradient_loss | -0.00484     |\n",
      "|    value_loss           | 2.58e+03     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 473           |\n",
      "|    time_elapsed         | 1692          |\n",
      "|    total_timesteps      | 968704        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 2.5184156e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -1.92         |\n",
      "|    explained_variance   | 0.599         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 1.64e+05      |\n",
      "|    n_updates            | 4720          |\n",
      "|    policy_gradient_loss | 0.00014       |\n",
      "|    value_loss           | 4.49e+05      |\n",
      "-------------------------------------------\n",
      "--------------------------------------------\n",
      "| time/                   |                |\n",
      "|    fps                  | 572            |\n",
      "|    iterations           | 474            |\n",
      "|    time_elapsed         | 1695           |\n",
      "|    total_timesteps      | 970752         |\n",
      "| train/                  |                |\n",
      "|    approx_kl            | 1.02154445e-07 |\n",
      "|    clip_fraction        | 0              |\n",
      "|    clip_range           | 0.2            |\n",
      "|    entropy_loss         | -1.94          |\n",
      "|    explained_variance   | 0.382          |\n",
      "|    learning_rate        | 0.0003         |\n",
      "|    loss                 | 6.84e+05       |\n",
      "|    n_updates            | 4730           |\n",
      "|    policy_gradient_loss | -5.74e-05      |\n",
      "|    value_loss           | 1.63e+06       |\n",
      "--------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 475           |\n",
      "|    time_elapsed         | 1699          |\n",
      "|    total_timesteps      | 972800        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.4260877e-09 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -2.04         |\n",
      "|    explained_variance   | 0.13          |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 2.17e+07      |\n",
      "|    n_updates            | 4740          |\n",
      "|    policy_gradient_loss | -1.1e-05      |\n",
      "|    value_loss           | 4.51e+07      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 476          |\n",
      "|    time_elapsed         | 1703         |\n",
      "|    total_timesteps      | 974848       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0007686764 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -1.76        |\n",
      "|    explained_variance   | -3.45        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 21.7         |\n",
      "|    n_updates            | 4750         |\n",
      "|    policy_gradient_loss | -0.00151     |\n",
      "|    value_loss           | 4.07e+03     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 477           |\n",
      "|    time_elapsed         | 1706          |\n",
      "|    total_timesteps      | 976896        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 1.7306214e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -1.82         |\n",
      "|    explained_variance   | 0.721         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 2e+04         |\n",
      "|    n_updates            | 4760          |\n",
      "|    policy_gradient_loss | -0.000438     |\n",
      "|    value_loss           | 9.57e+04      |\n",
      "-------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 478          |\n",
      "|    time_elapsed         | 1710         |\n",
      "|    total_timesteps      | 978944       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0046845926 |\n",
      "|    clip_fraction        | 0.00967      |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -1.77        |\n",
      "|    explained_variance   | -20.4        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 26.7         |\n",
      "|    n_updates            | 4770         |\n",
      "|    policy_gradient_loss | -0.00614     |\n",
      "|    value_loss           | 1.67e+03     |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 479          |\n",
      "|    time_elapsed         | 1713         |\n",
      "|    total_timesteps      | 980992       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 8.868636e-06 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -2.02        |\n",
      "|    explained_variance   | 0.172        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 8.13e+06     |\n",
      "|    n_updates            | 4780         |\n",
      "|    policy_gradient_loss | -0.000131    |\n",
      "|    value_loss           | 2e+07        |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 480          |\n",
      "|    time_elapsed         | 1717         |\n",
      "|    total_timesteps      | 983040       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0015822408 |\n",
      "|    clip_fraction        | 0.00366      |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -1.79        |\n",
      "|    explained_variance   | -4.83        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 42.5         |\n",
      "|    n_updates            | 4790         |\n",
      "|    policy_gradient_loss | -0.00325     |\n",
      "|    value_loss           | 2.6e+03      |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 572          |\n",
      "|    iterations           | 481          |\n",
      "|    time_elapsed         | 1721         |\n",
      "|    total_timesteps      | 985088       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 3.945132e-05 |\n",
      "|    clip_fraction        | 0            |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -1.91        |\n",
      "|    explained_variance   | 0.3          |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.62e+06     |\n",
      "|    n_updates            | 4800         |\n",
      "|    policy_gradient_loss | 2.55e-05     |\n",
      "|    value_loss           | 3.79e+06     |\n",
      "------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 482           |\n",
      "|    time_elapsed         | 1724          |\n",
      "|    total_timesteps      | 987136        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00039509445 |\n",
      "|    clip_fraction        | 9.77e-05      |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -1.82         |\n",
      "|    explained_variance   | 0.803         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 5.26e+03      |\n",
      "|    n_updates            | 4810          |\n",
      "|    policy_gradient_loss | -0.00292      |\n",
      "|    value_loss           | 1.69e+04      |\n",
      "-------------------------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 572           |\n",
      "|    iterations           | 483           |\n",
      "|    time_elapsed         | 1728          |\n",
      "|    total_timesteps      | 989184        |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 0.00072489673 |\n",
      "|    clip_fraction        | 0.000146      |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -1.82         |\n",
      "|    explained_variance   | 0.745         |\n",
      "|    learning_rate        | 0.0003        |\n",
      "|    loss                 | 3.78e+04      |\n",
      "|    n_updates            | 4820          |\n",
      "|    policy_gradient_loss | -0.00318      |\n",
      "|    value_loss           | 3.56e+04      |\n",
      "-------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 484         |\n",
      "|    time_elapsed         | 1731        |\n",
      "|    total_timesteps      | 991232      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.036790654 |\n",
      "|    clip_fraction        | 0.162       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.81       |\n",
      "|    explained_variance   | -39.7       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 10.8        |\n",
      "|    n_updates            | 4830        |\n",
      "|    policy_gradient_loss | -0.022      |\n",
      "|    value_loss           | 48.6        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 485         |\n",
      "|    time_elapsed         | 1735        |\n",
      "|    total_timesteps      | 993280      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.014150651 |\n",
      "|    clip_fraction        | 0.107       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.82       |\n",
      "|    explained_variance   | -27.2       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 109         |\n",
      "|    n_updates            | 4840        |\n",
      "|    policy_gradient_loss | -0.00607    |\n",
      "|    value_loss           | 1.43e+03    |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 572        |\n",
      "|    iterations           | 486        |\n",
      "|    time_elapsed         | 1739       |\n",
      "|    total_timesteps      | 995328     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.07381343 |\n",
      "|    clip_fraction        | 0.298      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.84      |\n",
      "|    explained_variance   | -162       |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 0.0362     |\n",
      "|    n_updates            | 4850       |\n",
      "|    policy_gradient_loss | -0.0284    |\n",
      "|    value_loss           | 2.98       |\n",
      "----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 572        |\n",
      "|    iterations           | 487        |\n",
      "|    time_elapsed         | 1742       |\n",
      "|    total_timesteps      | 997376     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.04195126 |\n",
      "|    clip_fraction        | 0.256      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.87      |\n",
      "|    explained_variance   | -10.5      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 0.066      |\n",
      "|    n_updates            | 4860       |\n",
      "|    policy_gradient_loss | -0.0182    |\n",
      "|    value_loss           | 0.244      |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 488         |\n",
      "|    time_elapsed         | 1746        |\n",
      "|    total_timesteps      | 999424      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.028304616 |\n",
      "|    clip_fraction        | 0.237       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2          |\n",
      "|    explained_variance   | -2.84       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.0151      |\n",
      "|    n_updates            | 4870        |\n",
      "|    policy_gradient_loss | -0.0371     |\n",
      "|    value_loss           | 0.125       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 572         |\n",
      "|    iterations           | 489         |\n",
      "|    time_elapsed         | 1749        |\n",
      "|    total_timesteps      | 1001472     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.018976595 |\n",
      "|    clip_fraction        | 0.192       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -2.06       |\n",
      "|    explained_variance   | -0.372      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.0929      |\n",
      "|    n_updates            | 4880        |\n",
      "|    policy_gradient_loss | -0.027      |\n",
      "|    value_loss           | 0.185       |\n",
      "-----------------------------------------\n",
      "Mean Reward: -5.00, Std Reward: 0.00\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Yash\\AppData\\Roaming\\Python\\Python310\\site-packages\\stable_baselines3\\common\\evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n",
      "C:\\Users\\Yash\\AppData\\Roaming\\Python\\Python310\\site-packages\\stable_baselines3\\common\\vec_env\\base_vec_env.py:234: UserWarning: You tried to call render() but no `render_mode` was passed to the env constructor.\n",
      "  warnings.warn(\"You tried to call render() but no `render_mode` was passed to the env constructor.\")\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "unsupported format string passed to numpy.ndarray.__format__",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[11], line 41\u001b[0m\n\u001b[0;32m     39\u001b[0m     total_reward \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m reward\n\u001b[0;32m     40\u001b[0m     env\u001b[38;5;241m.\u001b[39mrender()\n\u001b[1;32m---> 41\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mEpisode Reward: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtotal_reward\u001b[38;5;132;01m:\u001b[39;00m\u001b[38;5;124m.2f\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",
      "\u001b[1;31mTypeError\u001b[0m: unsupported format string passed to numpy.ndarray.__format__"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "import numpy as np\n",
    "from stable_baselines3 import PPO\n",
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "from stable_baselines3.common.vec_env import DummyVecEnv\n",
    "\n",
    "\n",
    "# Import your custom TradingEnv (update the import statement if needed)\n",
    "# from trading_env import TradingEnv  # Make sure to replace 'trading_env' with your file name\n",
    "\n",
    "# Create a function to evaluate the trained model\n",
    "def evaluate_trained_model(model, env, num_episodes=10):\n",
    "    mean_rewards, std_rewards = evaluate_policy(model, env, n_eval_episodes=num_episodes)\n",
    "    return mean_rewards, std_rewards\n",
    "\n",
    "# Create and wrap the custom environment\n",
    "env = TradingEnv()\n",
    "env = DummyVecEnv([lambda: env])\n",
    "\n",
    "# Create the PPO agent\n",
    "model = PPO(\"MlpPolicy\", env, verbose=1)\n",
    "\n",
    "# Train the model (you can adjust the number of training steps)\n",
    "model.learn(total_timesteps=1000000)\n",
    "\n",
    "# Evaluate the trained model\n",
    "mean_rewards, std_rewards = evaluate_trained_model(model, env)\n",
    "\n",
    "print(f\"Mean Reward: {mean_rewards:.2f}, Std Reward: {std_rewards:.2f}\")\n",
    "\n",
    "# Test the model for a few episodes\n",
    "for _ in range(5):\n",
    "    obs = env.reset()\n",
    "    done = False\n",
    "    total_reward = 0\n",
    "    while not done:\n",
    "        action, _ = model.predict(obs)\n",
    "        obs, reward, done, _ = env.step(action)\n",
    "        total_reward += reward\n",
    "        env.render()\n",
    "    print(f\"Episode Reward: {total_reward:.2f}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e26ddad3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: stable-baselines3[extra] in c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages (2.1.0)\n",
      "Requirement already satisfied: numpy>=1.20 in c:\\programdata\\anaconda3\\lib\\site-packages (from stable-baselines3[extra]) (1.23.5)\n",
      "Requirement already satisfied: gymnasium<0.30,>=0.28.1 in c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages (from stable-baselines3[extra]) (0.29.1)\n",
      "Requirement already satisfied: torch>=1.13 in c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages (from stable-baselines3[extra]) (2.0.1)\n",
      "Requirement already satisfied: pandas in c:\\programdata\\anaconda3\\lib\\site-packages (from stable-baselines3[extra]) (1.5.3)\n",
      "Requirement already satisfied: matplotlib in c:\\programdata\\anaconda3\\lib\\site-packages (from stable-baselines3[extra]) (3.7.0)\n",
      "Requirement already satisfied: cloudpickle in c:\\programdata\\anaconda3\\lib\\site-packages (from stable-baselines3[extra]) (2.0.0)\n",
      "Requirement already satisfied: pillow in c:\\programdata\\anaconda3\\lib\\site-packages (from stable-baselines3[extra]) (9.4.0)\n",
      "Requirement already satisfied: pygame in c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages (from stable-baselines3[extra]) (2.5.1)\n",
      "Requirement already satisfied: tqdm in c:\\programdata\\anaconda3\\lib\\site-packages (from stable-baselines3[extra]) (4.64.1)\n",
      "Requirement already satisfied: opencv-python in c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages (from stable-baselines3[extra]) (4.8.0.76)\n",
      "Requirement already satisfied: tensorboard>=2.9.1 in c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages (from stable-baselines3[extra]) (2.13.0)\n",
      "Requirement already satisfied: shimmy[atari]~=1.1.0 in c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages (from stable-baselines3[extra]) (1.1.0)\n",
      "Requirement already satisfied: rich in c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages (from stable-baselines3[extra]) (13.5.2)\n",
      "Requirement already satisfied: psutil in c:\\programdata\\anaconda3\\lib\\site-packages (from stable-baselines3[extra]) (5.9.0)\n",
      "Requirement already satisfied: autorom[accept-rom-license]~=0.6.1 in c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages (from stable-baselines3[extra]) (0.6.1)\n",
      "Requirement already satisfied: click in c:\\programdata\\anaconda3\\lib\\site-packages (from autorom[accept-rom-license]~=0.6.1->stable-baselines3[extra]) (8.0.4)\n",
      "Requirement already satisfied: requests in c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages (from autorom[accept-rom-license]~=0.6.1->stable-baselines3[extra]) (2.28.1)\n",
      "Requirement already satisfied: AutoROM.accept-rom-license in c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages (from autorom[accept-rom-license]~=0.6.1->stable-baselines3[extra]) (0.6.1)\n",
      "Requirement already satisfied: typing-extensions>=4.3.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from gymnasium<0.30,>=0.28.1->stable-baselines3[extra]) (4.4.0)\n",
      "Requirement already satisfied: farama-notifications>=0.0.1 in c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages (from gymnasium<0.30,>=0.28.1->stable-baselines3[extra]) (0.0.4)\n",
      "Requirement already satisfied: ale-py~=0.8.1 in c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages (from shimmy[atari]~=1.1.0->stable-baselines3[extra]) (0.8.1)\n",
      "Requirement already satisfied: werkzeug>=1.0.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (2.2.2)\n",
      "Requirement already satisfied: absl-py>=0.4 in c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (1.4.0)\n",
      "Requirement already satisfied: protobuf>=3.19.6 in c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (4.24.0)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (65.6.3)\n",
      "Requirement already satisfied: wheel>=0.26 in c:\\programdata\\anaconda3\\lib\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (0.38.4)\n",
      "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (0.7.1)\n",
      "Requirement already satisfied: google-auth<3,>=1.6.3 in c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (2.22.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in c:\\programdata\\anaconda3\\lib\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (3.4.1)\n",
      "Requirement already satisfied: grpcio>=1.48.2 in c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (1.57.0)\n",
      "Requirement already satisfied: google-auth-oauthlib<1.1,>=0.5 in c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (1.0.0)\n",
      "Requirement already satisfied: sympy in c:\\programdata\\anaconda3\\lib\\site-packages (from torch>=1.13->stable-baselines3[extra]) (1.11.1)\n",
      "Requirement already satisfied: networkx in c:\\programdata\\anaconda3\\lib\\site-packages (from torch>=1.13->stable-baselines3[extra]) (2.8.4)\n",
      "Requirement already satisfied: jinja2 in c:\\programdata\\anaconda3\\lib\\site-packages (from torch>=1.13->stable-baselines3[extra]) (3.1.2)\n",
      "Requirement already satisfied: filelock in c:\\programdata\\anaconda3\\lib\\site-packages (from torch>=1.13->stable-baselines3[extra]) (3.9.0)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (1.0.5)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in c:\\programdata\\anaconda3\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (2.8.2)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (4.25.0)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (1.4.4)\n",
      "Requirement already satisfied: packaging>=20.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (22.0)\n",
      "Requirement already satisfied: cycler>=0.10 in c:\\programdata\\anaconda3\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (0.11.0)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (3.0.9)\n",
      "Requirement already satisfied: pytz>=2020.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from pandas->stable-baselines3[extra]) (2022.7)\n",
      "Requirement already satisfied: markdown-it-py>=2.2.0 in c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages (from rich->stable-baselines3[extra]) (3.0.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages (from rich->stable-baselines3[extra]) (2.16.1)\n",
      "Requirement already satisfied: colorama in c:\\programdata\\anaconda3\\lib\\site-packages (from tqdm->stable-baselines3[extra]) (0.4.6)\n",
      "Requirement already satisfied: importlib-resources in c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages (from ale-py~=0.8.1->shimmy[atari]~=1.1.0->stable-baselines3[extra]) (6.0.1)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.9.1->stable-baselines3[extra]) (0.2.8)\n",
      "Requirement already satisfied: urllib3<2.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.9.1->stable-baselines3[extra]) (1.26.14)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.9.1->stable-baselines3[extra]) (4.9)\n",
      "Requirement already satisfied: six>=1.9.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.9.1->stable-baselines3[extra]) (1.16.0)\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.9.1->stable-baselines3[extra]) (5.3.1)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages (from google-auth-oauthlib<1.1,>=0.5->tensorboard>=2.9.1->stable-baselines3[extra]) (1.3.1)\n",
      "Requirement already satisfied: mdurl~=0.1 in c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages (from markdown-it-py>=2.2.0->rich->stable-baselines3[extra]) (0.1.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests->autorom[accept-rom-license]~=0.6.1->stable-baselines3[extra]) (3.4)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests->autorom[accept-rom-license]~=0.6.1->stable-baselines3[extra]) (2.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests->autorom[accept-rom-license]~=0.6.1->stable-baselines3[extra]) (2023.7.22)\n",
      "Requirement already satisfied: MarkupSafe>=2.1.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from werkzeug>=1.0.1->tensorboard>=2.9.1->stable-baselines3[extra]) (2.1.1)\n",
      "Requirement already satisfied: mpmath>=0.19 in c:\\programdata\\anaconda3\\lib\\site-packages (from sympy->torch>=1.13->stable-baselines3[extra]) (1.2.1)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in c:\\programdata\\anaconda3\\lib\\site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard>=2.9.1->stable-baselines3[extra]) (0.4.8)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tensorboard>=2.9.1->stable-baselines3[extra]) (3.2.2)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Ignoring invalid distribution -ensorflow-intel (c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -ensorflow-intel (c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -ensorflow-intel (c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -ensorflow-intel (c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -ensorflow-intel (c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -ensorflow-intel (c:\\users\\yash\\appdata\\roaming\\python\\python310\\site-packages)\n"
     ]
    }
   ],
   "source": [
    "pip install stable-baselines3[extra]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af7254a6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}