{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.4.0-py3.8.egg/power_cogs/config/torch/torch_config.py:17: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "  class AdamConf:\n",
      "/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.4.0-py3.8.egg/power_cogs/config/torch/torch_config.py:27: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "  class LambdaLRConf:\n",
      "/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.4.0-py3.8.egg/power_cogs/config/torch/torch_config.py:35: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "  class MultiplicativeLRConf:\n",
      "/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.4.0-py3.8.egg/power_cogs/config/torch/torch_config.py:43: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "  class StepLRConf:\n",
      "/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.4.0-py3.8.egg/power_cogs/config/torch/torch_config.py:52: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "  class MultiStepLRConf:\n",
      "/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.4.0-py3.8.egg/power_cogs/config/torch/torch_config.py:61: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "  class ExponentialLRConf:\n",
      "/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.4.0-py3.8.egg/power_cogs/config/torch/torch_config.py:69: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "  class CosineAnnealingLRConf:\n",
      "/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.4.0-py3.8.egg/power_cogs/config/torch/torch_config.py:78: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "  class ReduceLROnPlateauConf:\n",
      "/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.4.0-py3.8.egg/power_cogs/config/torch/torch_config.py:93: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "  class CyclicLRConf:\n",
      "/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.4.0-py3.8.egg/power_cogs/config/torch/torch_config.py:111: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "  class CosineAnnealingWarmRestartsConf:\n",
      "/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.4.0-py3.8.egg/power_cogs/config/torch/torch_config.py:121: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "  class OneCycleLRConf:\n",
      "/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.4.0-py3.8.egg/power_cogs/config/torch/torch_config.py:141: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "  class DataLoaderConf:\n",
      "/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.4.0-py3.8.egg/power_cogs/config/torch/torch_config.py:158: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "  class DatasetConf:\n",
      "/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.4.0-py3.8.egg/power_cogs/config/torch/torch_config.py:163: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "  class ChainDatasetConf:\n",
      "/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.4.0-py3.8.egg/power_cogs/config/torch/torch_config.py:169: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "  class ConcatDatasetConf:\n",
      "/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.4.0-py3.8.egg/power_cogs/config/torch/torch_config.py:175: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "  class IterableDatasetConf:\n",
      "/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.4.0-py3.8.egg/power_cogs/config/torch/torch_config.py:180: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "  class TensorDatasetConf:\n",
      "/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.4.0-py3.8.egg/power_cogs/config/torch/torch_config.py:186: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "  class SubsetConf:\n"
     ]
    }
   ],
   "source": [
    "from power_cogs.callbacks.callback import *\n",
    "from power_cogs.callbacks.logger_callback import *\n",
    "from power_cogs.callbacks.class_method_callback import ClassMethodCallback\n",
    "from power_cogs.config.config import load_config\n",
    "from power_cogs.examples.reinforce.reinforce_trainer import ReinforceTrainer\n",
    "from typing import Dict, Any, Optional\n",
    "from omegaconf import OmegaConf\n",
    "from power_cogs.utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "rt = ReinforceTrainer.from_config_file(\"reinforce.yaml\",\n",
    "                                      overrides={\"logging_config\":{\"checkpoint_interval\":5}})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-09-09 15:06:19.579 | INFO     | power_cogs.callbacks.tensorboard_callback:setup:40 - Follow tensorboard logs with: tensorboard --logdir checkpoints/2021-09-09-15:06:19_ReinforceTrainer/tensorboard_logs\n",
      "average_summed_rewards:-47.07002354579193--max_summed_rewards:24.47283691540074--loss:-0.11703166998922825--max_loss:0.47992226481437683--sum_loss:-2.340633399784565--epoch:210--net.0.0.weight_grad:-0.09382223337888718--net.0.0.bias_grad:-0.9087388515472412--net.1.0.weight_grad:-1.1794319152832031--net.1.0.bias_grad:-0.29766330122947693--net.2.0.weight_grad:0.13967379927635193--net.2.0.bias_grad:0.038415104150772095--net.3.weight_grad:7.450580596923828e-08--net.3.bias_grad:-1.4901161193847656e-08:  21%|██        | 211/1000 [03:59<27:53,  2.12s/it]            "
     ]
    }
   ],
   "source": [
    "rt.train(epochs=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from power_cogs.wrappers.tune_wrapper import TuneWrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "wrapper = TuneWrapper(rt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-22 15:24:24,504\tINFO services.py:1267 -- View the Ray dashboard at \u001b[1m\u001b[32mhttp://127.0.0.1:8265\u001b[39m\u001b[22m\n",
      "2021-08-22 15:24:26,410\tWARNING tune.py:494 -- Tune detects GPUs, but no trials are using GPUs. To enable trials to use GPUs, set tune.run(resources_per_trial={'gpu': 1}...) which allows Tune to expose 1 GPU to each trial. You can also override `Trainable.default_resource_request` if using the Trainable API.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 6.5/7.4 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 0/8 CPUs, 0/1 GPUs, 0.0/1.07 GiB heap, 0.0/0.54 GiB objects (0.0/1.0 accelerator_type:T1000)<br>Result logdir: /home/shyam/ray_results/TuneTrainer_2021-08-22_15-24-26<br>Number of trials: 1/1 (1 PENDING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name             </th><th>status  </th><th>loc  </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>TuneTrainer_b58c9_00000</td><td>PENDING </td><td>     </td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/ray/tune/syncer.py:234: DeprecationWarning: ray.services.get_node_ip_address has been moved to ray.util.get_node_ip_address. 'ray.services' will be removed after Ray 1.4.\n",
      "  self.local_ip = services.get_node_ip_address()\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m /home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/config/torch/torch_config.py:17: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m /home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/config/torch/torch_config.py:27: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m /home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/config/torch/torch_config.py:35: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m /home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/config/torch/torch_config.py:43: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m /home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/config/torch/torch_config.py:52: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m /home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/config/torch/torch_config.py:61: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m /home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/config/torch/torch_config.py:69: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m /home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/config/torch/torch_config.py:78: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m /home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/config/torch/torch_config.py:93: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m /home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/config/torch/torch_config.py:111: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m /home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/config/torch/torch_config.py:121: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m /home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/config/torch/torch_config.py:141: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m /home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/config/torch/torch_config.py:158: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m /home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/config/torch/torch_config.py:163: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m /home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/config/torch/torch_config.py:169: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m /home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/config/torch/torch_config.py:175: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m /home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/config/torch/torch_config.py:180: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m /home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/config/torch/torch_config.py:186: RuntimeWarning: fields may not start with an underscore, ignoring \"_target_\"\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m 2021-08-22 15:24:27.633 | INFO     | power_cogs.callbacks.tensorboard_callback:setup:40 - Follow tensorboard logs with: tensorboard --logdir checkpoints/2021-08-22-15:24:27_ReinforceTrainer/tensorboard_logs\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for TuneTrainer_b58c9_00000:\n",
      "  average_summed_rewards: -213.09244011332777\n",
      "  date: 2021-08-22_15-24-27\n",
      "  done: false\n",
      "  epoch: 0\n",
      "  experiment_id: ea3e4da9a71247ddb0ad6c4bcb6d8084\n",
      "  hostname: shyam-ThinkPad-P53\n",
      "  iterations_since_restore: 1\n",
      "  loss: 0.00847292437683791\n",
      "  max_loss: 0.01459780428558588\n",
      "  max_summed_rewards: -60.69080595096608\n",
      "  net.0.0.bias_grad: -0.008909550495445728\n",
      "  net.0.0.weight_grad: -0.0012469281209632754\n",
      "  net.1.0.bias_grad: 0.01292765885591507\n",
      "  net.1.0.weight_grad: 0.04121093451976776\n",
      "  net.2.0.bias_grad: -0.005310457199811935\n",
      "  net.2.0.weight_grad: -0.006065706256777048\n",
      "  net.3.bias_grad: 0.0\n",
      "  net.3.weight_grad: -2.2992026060819626e-09\n",
      "  node_ip: 192.168.1.248\n",
      "  pid: 57240\n",
      "  sum_loss: 0.042364621884189546\n",
      "  time_since_restore: 0.20894694328308105\n",
      "  time_this_iter_s: 0.20894694328308105\n",
      "  time_total_s: 0.20894694328308105\n",
      "  timestamp: 1629671067\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 1\n",
      "  trial_id: b58c9_00000\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 6.6/7.4 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/8 CPUs, 0/1 GPUs, 0.0/1.07 GiB heap, 0.0/0.54 GiB objects (0.0/1.0 accelerator_type:T1000)<br>Current best trial: b58c9_00000 with loss=-0.0031783356331288814 and parameters={'trainer_class': <class 'power_cogs.examples.reinforce.reinforce_trainer.ReinforceTrainer'>, 'trainer_config': {'name': 'ReinforceTrainer', 'visualize': False, 'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'logging_config': {'mlflow': {'experiment_name': None, 'save_artifact': False}, 'wandb': {'project': None, 'api_key_file': None, 'log_config': False, 'reinit': False}, 'checkpoint_path': 'checkpoints', 'tensorboard_log_path': None, 'checkpoint_interval': 5}, 'stoppage_config': {'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'loss_threshold': -inf, 'stop': None, 'callbacks': []}, 'tune_config': {'metric': 'loss', 'mode': 'min', 'num_samples': 1, 'name': '', 'checkpoint_freq': 5, 'checkpoint_at_end': True, 'callbacks': [], 'additional_config': {}}, 'optimizer_config': {'_target_': 'torch.optim.adam.Adam', 'params': '???', 'lr': 0.002, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'scheduler_config': {'_target_': 'torch.optim.lr_scheduler.ExponentialLR', 'optimizer': '???', 'gamma': 0.9999, 'last_epoch': -1}, 'dataloader_config': {'_target_': 'torch.utils.data.dataloader.DataLoader', 'dataset': '???', 'batch_size': 1, 'shuffle': False, 'sampler': None, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None}, 'cluster_config': {'name': '', 'cluster_config_path': None, 'join_cluster': False}, 'dataset_config': {'env_name': 'LunarLander-v2', 'max_env_steps': 10000, 'gamma': 0.99, 'discount_rewards': True, 'normalize_rewards': True, 'random_policy': False, 'num_rollouts': 100, 'num_workers': 5}, 'model_config': {'input_dims': 8, 'hidden_dims': [32, 32], 'output_dims': 4, 'output_activation': None, 'use_normal_init': True, 'normal_std': 0.1, 'zero_bias': False}}}<br>Result logdir: /home/shyam/ray_results/TuneTrainer_2021-08-22_15-24-26<br>Number of trials: 1/1 (1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name             </th><th>status  </th><th>loc                </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  average_summed_rewards</th><th style=\"text-align: right;\">  max_summed_rewards</th><th style=\"text-align: right;\">       loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>TuneTrainer_b58c9_00000</td><td>RUNNING </td><td>192.168.1.248:57240</td><td style=\"text-align: right;\">    15</td><td style=\"text-align: right;\">         3.25453</td><td style=\"text-align: right;\">                -246.704</td><td style=\"text-align: right;\">            -181.916</td><td style=\"text-align: right;\">-0.00317834</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for TuneTrainer_b58c9_00000:\n",
      "  average_summed_rewards: -190.9828004871608\n",
      "  date: 2021-08-22_15-24-32\n",
      "  done: false\n",
      "  epoch: 3\n",
      "  experiment_id: ea3e4da9a71247ddb0ad6c4bcb6d8084\n",
      "  hostname: shyam-ThinkPad-P53\n",
      "  iterations_since_restore: 19\n",
      "  loss: -0.008966851350851356\n",
      "  max_loss: -0.0010110115399584174\n",
      "  max_summed_rewards: -88.38348606937365\n",
      "  net.0.0.bias_grad: -0.03103065863251686\n",
      "  net.0.0.weight_grad: -0.07617010921239853\n",
      "  net.1.0.bias_grad: -0.011701010167598724\n",
      "  net.1.0.weight_grad: -0.10763837397098541\n",
      "  net.2.0.bias_grad: -0.027567891404032707\n",
      "  net.2.0.weight_grad: -0.14361892640590668\n",
      "  net.3.bias_grad: -7.450580596923828e-09\n",
      "  net.3.weight_grad: -2.3435859475284815e-08\n",
      "  node_ip: 192.168.1.248\n",
      "  pid: 57240\n",
      "  sum_loss: -0.044834256754256785\n",
      "  time_since_restore: 4.209816217422485\n",
      "  time_this_iter_s: 0.26084065437316895\n",
      "  time_total_s: 4.209816217422485\n",
      "  timestamp: 1629671072\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 19\n",
      "  trial_id: b58c9_00000\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 6.7/7.4 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/8 CPUs, 0/1 GPUs, 0.0/1.07 GiB heap, 0.0/0.54 GiB objects (0.0/1.0 accelerator_type:T1000)<br>Current best trial: b58c9_00000 with loss=-0.01116545358672738 and parameters={'trainer_class': <class 'power_cogs.examples.reinforce.reinforce_trainer.ReinforceTrainer'>, 'trainer_config': {'name': 'ReinforceTrainer', 'visualize': False, 'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'logging_config': {'mlflow': {'experiment_name': None, 'save_artifact': False}, 'wandb': {'project': None, 'api_key_file': None, 'log_config': False, 'reinit': False}, 'checkpoint_path': 'checkpoints', 'tensorboard_log_path': None, 'checkpoint_interval': 5}, 'stoppage_config': {'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'loss_threshold': -inf, 'stop': None, 'callbacks': []}, 'tune_config': {'metric': 'loss', 'mode': 'min', 'num_samples': 1, 'name': '', 'checkpoint_freq': 5, 'checkpoint_at_end': True, 'callbacks': [], 'additional_config': {}}, 'optimizer_config': {'_target_': 'torch.optim.adam.Adam', 'params': '???', 'lr': 0.002, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'scheduler_config': {'_target_': 'torch.optim.lr_scheduler.ExponentialLR', 'optimizer': '???', 'gamma': 0.9999, 'last_epoch': -1}, 'dataloader_config': {'_target_': 'torch.utils.data.dataloader.DataLoader', 'dataset': '???', 'batch_size': 1, 'shuffle': False, 'sampler': None, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None}, 'cluster_config': {'name': '', 'cluster_config_path': None, 'join_cluster': False}, 'dataset_config': {'env_name': 'LunarLander-v2', 'max_env_steps': 10000, 'gamma': 0.99, 'discount_rewards': True, 'normalize_rewards': True, 'random_policy': False, 'num_rollouts': 100, 'num_workers': 5}, 'model_config': {'input_dims': 8, 'hidden_dims': [32, 32], 'output_dims': 4, 'output_activation': None, 'use_normal_init': True, 'normal_std': 0.1, 'zero_bias': False}}}<br>Result logdir: /home/shyam/ray_results/TuneTrainer_2021-08-22_15-24-26<br>Number of trials: 1/1 (1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name             </th><th>status  </th><th>loc                </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  average_summed_rewards</th><th style=\"text-align: right;\">  max_summed_rewards</th><th style=\"text-align: right;\">      loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>TuneTrainer_b58c9_00000</td><td>RUNNING </td><td>192.168.1.248:57240</td><td style=\"text-align: right;\">    33</td><td style=\"text-align: right;\">         7.37084</td><td style=\"text-align: right;\">                -158.844</td><td style=\"text-align: right;\">            -97.5364</td><td style=\"text-align: right;\">-0.0111655</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for TuneTrainer_b58c9_00000:\n",
      "  average_summed_rewards: -60.64452384208063\n",
      "  date: 2021-08-22_15-24-38\n",
      "  done: false\n",
      "  epoch: 3\n",
      "  experiment_id: ea3e4da9a71247ddb0ad6c4bcb6d8084\n",
      "  hostname: shyam-ThinkPad-P53\n",
      "  iterations_since_restore: 34\n",
      "  loss: -0.07386767175048589\n",
      "  max_loss: 0.011161514557898045\n",
      "  max_summed_rewards: 35.05662046930411\n",
      "  net.0.0.bias_grad: -0.07278364896774292\n",
      "  net.0.0.weight_grad: -0.007293734699487686\n",
      "  net.1.0.bias_grad: -0.05829687416553497\n",
      "  net.1.0.weight_grad: -0.264308899641037\n",
      "  net.2.0.bias_grad: -0.09066733717918396\n",
      "  net.2.0.weight_grad: -0.3606535792350769\n",
      "  net.3.bias_grad: -3.725290298461914e-09\n",
      "  net.3.weight_grad: -1.3242242857813835e-08\n",
      "  node_ip: 192.168.1.248\n",
      "  pid: 57240\n",
      "  sum_loss: -0.3693383587524295\n",
      "  time_since_restore: 8.817182064056396\n",
      "  time_this_iter_s: 1.446340560913086\n",
      "  time_total_s: 8.817182064056396\n",
      "  timestamp: 1629671078\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 34\n",
      "  trial_id: b58c9_00000\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 6.7/7.4 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/8 CPUs, 0/1 GPUs, 0.0/1.07 GiB heap, 0.0/0.54 GiB objects (0.0/1.0 accelerator_type:T1000)<br>Current best trial: b58c9_00000 with loss=0.001067112444434315 and parameters={'trainer_class': <class 'power_cogs.examples.reinforce.reinforce_trainer.ReinforceTrainer'>, 'trainer_config': {'name': 'ReinforceTrainer', 'visualize': False, 'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'logging_config': {'mlflow': {'experiment_name': None, 'save_artifact': False}, 'wandb': {'project': None, 'api_key_file': None, 'log_config': False, 'reinit': False}, 'checkpoint_path': 'checkpoints', 'tensorboard_log_path': None, 'checkpoint_interval': 5}, 'stoppage_config': {'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'loss_threshold': -inf, 'stop': None, 'callbacks': []}, 'tune_config': {'metric': 'loss', 'mode': 'min', 'num_samples': 1, 'name': '', 'checkpoint_freq': 5, 'checkpoint_at_end': True, 'callbacks': [], 'additional_config': {}}, 'optimizer_config': {'_target_': 'torch.optim.adam.Adam', 'params': '???', 'lr': 0.002, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'scheduler_config': {'_target_': 'torch.optim.lr_scheduler.ExponentialLR', 'optimizer': '???', 'gamma': 0.9999, 'last_epoch': -1}, 'dataloader_config': {'_target_': 'torch.utils.data.dataloader.DataLoader', 'dataset': '???', 'batch_size': 1, 'shuffle': False, 'sampler': None, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None}, 'cluster_config': {'name': '', 'cluster_config_path': None, 'join_cluster': False}, 'dataset_config': {'env_name': 'LunarLander-v2', 'max_env_steps': 10000, 'gamma': 0.99, 'discount_rewards': True, 'normalize_rewards': True, 'random_policy': False, 'num_rollouts': 100, 'num_workers': 5}, 'model_config': {'input_dims': 8, 'hidden_dims': [32, 32], 'output_dims': 4, 'output_activation': None, 'use_normal_init': True, 'normal_std': 0.1, 'zero_bias': False}}}<br>Result logdir: /home/shyam/ray_results/TuneTrainer_2021-08-22_15-24-26<br>Number of trials: 1/1 (1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name             </th><th>status  </th><th>loc                </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  average_summed_rewards</th><th style=\"text-align: right;\">  max_summed_rewards</th><th style=\"text-align: right;\">      loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>TuneTrainer_b58c9_00000</td><td>RUNNING </td><td>192.168.1.248:57240</td><td style=\"text-align: right;\">    43</td><td style=\"text-align: right;\">         12.0356</td><td style=\"text-align: right;\">                 -77.044</td><td style=\"text-align: right;\">            -6.74317</td><td style=\"text-align: right;\">0.00106711</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for TuneTrainer_b58c9_00000:\n",
      "  average_summed_rewards: -98.127718040161\n",
      "  date: 2021-08-22_15-24-43\n",
      "  done: false\n",
      "  epoch: 0\n",
      "  experiment_id: ea3e4da9a71247ddb0ad6c4bcb6d8084\n",
      "  hostname: shyam-ThinkPad-P53\n",
      "  iterations_since_restore: 46\n",
      "  loss: 0.022951717674732208\n",
      "  max_loss: 0.35660138726234436\n",
      "  max_summed_rewards: -8.252168003616703\n",
      "  net.0.0.bias_grad: -0.23580698668956757\n",
      "  net.0.0.weight_grad: 0.019238179549574852\n",
      "  net.1.0.bias_grad: -0.22507244348526\n",
      "  net.1.0.weight_grad: -0.8490992188453674\n",
      "  net.2.0.bias_grad: -0.21915411949157715\n",
      "  net.2.0.weight_grad: -1.0383830070495605\n",
      "  net.3.bias_grad: 6.984919309616089e-09\n",
      "  net.3.weight_grad: 2.6309862732887268e-08\n",
      "  node_ip: 192.168.1.248\n",
      "  pid: 57240\n",
      "  sum_loss: 0.11475858837366104\n",
      "  time_since_restore: 13.225189208984375\n",
      "  time_this_iter_s: 0.5983848571777344\n",
      "  time_total_s: 13.225189208984375\n",
      "  timestamp: 1629671083\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 46\n",
      "  trial_id: b58c9_00000\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 6.7/7.4 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/8 CPUs, 0/1 GPUs, 0.0/1.07 GiB heap, 0.0/0.54 GiB objects (0.0/1.0 accelerator_type:T1000)<br>Current best trial: b58c9_00000 with loss=0.07254198789596558 and parameters={'trainer_class': <class 'power_cogs.examples.reinforce.reinforce_trainer.ReinforceTrainer'>, 'trainer_config': {'name': 'ReinforceTrainer', 'visualize': False, 'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'logging_config': {'mlflow': {'experiment_name': None, 'save_artifact': False}, 'wandb': {'project': None, 'api_key_file': None, 'log_config': False, 'reinit': False}, 'checkpoint_path': 'checkpoints', 'tensorboard_log_path': None, 'checkpoint_interval': 5}, 'stoppage_config': {'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'loss_threshold': -inf, 'stop': None, 'callbacks': []}, 'tune_config': {'metric': 'loss', 'mode': 'min', 'num_samples': 1, 'name': '', 'checkpoint_freq': 5, 'checkpoint_at_end': True, 'callbacks': [], 'additional_config': {}}, 'optimizer_config': {'_target_': 'torch.optim.adam.Adam', 'params': '???', 'lr': 0.002, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'scheduler_config': {'_target_': 'torch.optim.lr_scheduler.ExponentialLR', 'optimizer': '???', 'gamma': 0.9999, 'last_epoch': -1}, 'dataloader_config': {'_target_': 'torch.utils.data.dataloader.DataLoader', 'dataset': '???', 'batch_size': 1, 'shuffle': False, 'sampler': None, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None}, 'cluster_config': {'name': '', 'cluster_config_path': None, 'join_cluster': False}, 'dataset_config': {'env_name': 'LunarLander-v2', 'max_env_steps': 10000, 'gamma': 0.99, 'discount_rewards': True, 'normalize_rewards': True, 'random_policy': False, 'num_rollouts': 100, 'num_workers': 5}, 'model_config': {'input_dims': 8, 'hidden_dims': [32, 32], 'output_dims': 4, 'output_activation': None, 'use_normal_init': True, 'normal_std': 0.1, 'zero_bias': False}}}<br>Result logdir: /home/shyam/ray_results/TuneTrainer_2021-08-22_15-24-26<br>Number of trials: 1/1 (1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name             </th><th>status  </th><th>loc                </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  average_summed_rewards</th><th style=\"text-align: right;\">  max_summed_rewards</th><th style=\"text-align: right;\">    loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>TuneTrainer_b58c9_00000</td><td>RUNNING </td><td>192.168.1.248:57240</td><td style=\"text-align: right;\">    54</td><td style=\"text-align: right;\">         16.7679</td><td style=\"text-align: right;\">                -98.4617</td><td style=\"text-align: right;\">            -27.5187</td><td style=\"text-align: right;\">0.072542</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for TuneTrainer_b58c9_00000:\n",
      "  average_summed_rewards: -54.72992026786208\n",
      "  date: 2021-08-22_15-24-48\n",
      "  done: false\n",
      "  epoch: 1\n",
      "  experiment_id: ea3e4da9a71247ddb0ad6c4bcb6d8084\n",
      "  hostname: shyam-ThinkPad-P53\n",
      "  iterations_since_restore: 57\n",
      "  loss: -0.05696168038994074\n",
      "  max_loss: -0.008246823213994503\n",
      "  max_summed_rewards: -15.50890274233356\n",
      "  net.0.0.bias_grad: 0.016566969454288483\n",
      "  net.0.0.weight_grad: -0.025129713118076324\n",
      "  net.1.0.bias_grad: 0.045474108308553696\n",
      "  net.1.0.weight_grad: 0.15784257650375366\n",
      "  net.2.0.bias_grad: -0.03635241836309433\n",
      "  net.2.0.weight_grad: -0.18968360126018524\n",
      "  net.3.bias_grad: 7.450580596923828e-09\n",
      "  net.3.weight_grad: 1.4206307241693139e-08\n",
      "  node_ip: 192.168.1.248\n",
      "  pid: 57240\n",
      "  sum_loss: -0.2848084019497037\n",
      "  time_since_restore: 17.745941400527954\n",
      "  time_this_iter_s: 0.27655506134033203\n",
      "  time_total_s: 17.745941400527954\n",
      "  timestamp: 1629671088\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 57\n",
      "  trial_id: b58c9_00000\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 6.7/7.4 GiB: ***LOW MEMORY*** less than 10% of the memory on this node is available for use. This can cause unexpected crashes. Consider reducing the memory used by your application or reducing the Ray object store size by setting `object_store_memory` when calling `ray.init`.<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/8 CPUs, 0/1 GPUs, 0.0/1.07 GiB heap, 0.0/0.54 GiB objects (0.0/1.0 accelerator_type:T1000)<br>Current best trial: b58c9_00000 with loss=-0.07639773190021515 and parameters={'trainer_class': <class 'power_cogs.examples.reinforce.reinforce_trainer.ReinforceTrainer'>, 'trainer_config': {'name': 'ReinforceTrainer', 'visualize': False, 'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'logging_config': {'mlflow': {'experiment_name': None, 'save_artifact': False}, 'wandb': {'project': None, 'api_key_file': None, 'log_config': False, 'reinit': False}, 'checkpoint_path': 'checkpoints', 'tensorboard_log_path': None, 'checkpoint_interval': 5}, 'stoppage_config': {'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'loss_threshold': -inf, 'stop': None, 'callbacks': []}, 'tune_config': {'metric': 'loss', 'mode': 'min', 'num_samples': 1, 'name': '', 'checkpoint_freq': 5, 'checkpoint_at_end': True, 'callbacks': [], 'additional_config': {}}, 'optimizer_config': {'_target_': 'torch.optim.adam.Adam', 'params': '???', 'lr': 0.002, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'scheduler_config': {'_target_': 'torch.optim.lr_scheduler.ExponentialLR', 'optimizer': '???', 'gamma': 0.9999, 'last_epoch': -1}, 'dataloader_config': {'_target_': 'torch.utils.data.dataloader.DataLoader', 'dataset': '???', 'batch_size': 1, 'shuffle': False, 'sampler': None, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None}, 'cluster_config': {'name': '', 'cluster_config_path': None, 'join_cluster': False}, 'dataset_config': {'env_name': 'LunarLander-v2', 'max_env_steps': 10000, 'gamma': 0.99, 'discount_rewards': True, 'normalize_rewards': True, 'random_policy': False, 'num_rollouts': 100, 'num_workers': 5}, 'model_config': {'input_dims': 8, 'hidden_dims': [32, 32], 'output_dims': 4, 'output_activation': None, 'use_normal_init': True, 'normal_std': 0.1, 'zero_bias': False}}}<br>Result logdir: /home/shyam/ray_results/TuneTrainer_2021-08-22_15-24-26<br>Number of trials: 1/1 (1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name             </th><th>status  </th><th>loc                </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  average_summed_rewards</th><th style=\"text-align: right;\">  max_summed_rewards</th><th style=\"text-align: right;\">      loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>TuneTrainer_b58c9_00000</td><td>RUNNING </td><td>192.168.1.248:57240</td><td style=\"text-align: right;\">    61</td><td style=\"text-align: right;\">         22.3525</td><td style=\"text-align: right;\">                -61.2355</td><td style=\"text-align: right;\">             14.6073</td><td style=\"text-align: right;\">-0.0763977</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for TuneTrainer_b58c9_00000:\n",
      "  average_summed_rewards: -44.83696435367154\n",
      "  date: 2021-08-22_15-24-54\n",
      "  done: false\n",
      "  epoch: 1\n",
      "  experiment_id: ea3e4da9a71247ddb0ad6c4bcb6d8084\n",
      "  hostname: shyam-ThinkPad-P53\n",
      "  iterations_since_restore: 62\n",
      "  loss: -0.0071395992301404474\n",
      "  max_loss: 0.08007071912288666\n",
      "  max_summed_rewards: -9.320450817623595\n",
      "  net.0.0.bias_grad: 0.20176205039024353\n",
      "  net.0.0.weight_grad: 0.054696522653102875\n",
      "  net.1.0.bias_grad: 0.07202540338039398\n",
      "  net.1.0.weight_grad: 0.42988088726997375\n",
      "  net.2.0.bias_grad: 0.020677750930190086\n",
      "  net.2.0.weight_grad: 0.1585598886013031\n",
      "  net.3.bias_grad: -2.9802322387695312e-08\n",
      "  net.3.weight_grad: -1.957523636519909e-07\n",
      "  node_ip: 192.168.1.248\n",
      "  pid: 57240\n",
      "  sum_loss: -0.03569799615070224\n",
      "  time_since_restore: 23.43412685394287\n",
      "  time_this_iter_s: 1.0816619396209717\n",
      "  time_total_s: 23.43412685394287\n",
      "  timestamp: 1629671094\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 62\n",
      "  trial_id: b58c9_00000\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 6.7/7.4 GiB: ***LOW MEMORY*** less than 10% of the memory on this node is available for use. This can cause unexpected crashes. Consider reducing the memory used by your application or reducing the Ray object store size by setting `object_store_memory` when calling `ray.init`.<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/8 CPUs, 0/1 GPUs, 0.0/1.07 GiB heap, 0.0/0.54 GiB objects (0.0/1.0 accelerator_type:T1000)<br>Current best trial: b58c9_00000 with loss=0.031391741381958126 and parameters={'trainer_class': <class 'power_cogs.examples.reinforce.reinforce_trainer.ReinforceTrainer'>, 'trainer_config': {'name': 'ReinforceTrainer', 'visualize': False, 'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'logging_config': {'mlflow': {'experiment_name': None, 'save_artifact': False}, 'wandb': {'project': None, 'api_key_file': None, 'log_config': False, 'reinit': False}, 'checkpoint_path': 'checkpoints', 'tensorboard_log_path': None, 'checkpoint_interval': 5}, 'stoppage_config': {'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'loss_threshold': -inf, 'stop': None, 'callbacks': []}, 'tune_config': {'metric': 'loss', 'mode': 'min', 'num_samples': 1, 'name': '', 'checkpoint_freq': 5, 'checkpoint_at_end': True, 'callbacks': [], 'additional_config': {}}, 'optimizer_config': {'_target_': 'torch.optim.adam.Adam', 'params': '???', 'lr': 0.002, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'scheduler_config': {'_target_': 'torch.optim.lr_scheduler.ExponentialLR', 'optimizer': '???', 'gamma': 0.9999, 'last_epoch': -1}, 'dataloader_config': {'_target_': 'torch.utils.data.dataloader.DataLoader', 'dataset': '???', 'batch_size': 1, 'shuffle': False, 'sampler': None, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None}, 'cluster_config': {'name': '', 'cluster_config_path': None, 'join_cluster': False}, 'dataset_config': {'env_name': 'LunarLander-v2', 'max_env_steps': 10000, 'gamma': 0.99, 'discount_rewards': True, 'normalize_rewards': True, 'random_policy': False, 'num_rollouts': 100, 'num_workers': 5}, 'model_config': {'input_dims': 8, 'hidden_dims': [32, 32], 'output_dims': 4, 'output_activation': None, 'use_normal_init': True, 'normal_std': 0.1, 'zero_bias': False}}}<br>Result logdir: /home/shyam/ray_results/TuneTrainer_2021-08-22_15-24-26<br>Number of trials: 1/1 (1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name             </th><th>status  </th><th>loc                </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  average_summed_rewards</th><th style=\"text-align: right;\">  max_summed_rewards</th><th style=\"text-align: right;\">     loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>TuneTrainer_b58c9_00000</td><td>RUNNING </td><td>192.168.1.248:57240</td><td style=\"text-align: right;\">    65</td><td style=\"text-align: right;\">         27.5204</td><td style=\"text-align: right;\">                -363.025</td><td style=\"text-align: right;\">            -27.3098</td><td style=\"text-align: right;\">0.0313917</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for TuneTrainer_b58c9_00000:\n",
      "  average_summed_rewards: -191.6666952282155\n",
      "  date: 2021-08-22_15-24-59\n",
      "  done: false\n",
      "  epoch: 1\n",
      "  experiment_id: ea3e4da9a71247ddb0ad6c4bcb6d8084\n",
      "  hostname: shyam-ThinkPad-P53\n",
      "  iterations_since_restore: 67\n",
      "  loss: -0.004161244444549084\n",
      "  max_loss: 0.07844746857881546\n",
      "  max_summed_rewards: -87.85377068228813\n",
      "  net.0.0.bias_grad: 0.21979758143424988\n",
      "  net.0.0.weight_grad: 0.6230391263961792\n",
      "  net.1.0.bias_grad: 0.22237932682037354\n",
      "  net.1.0.weight_grad: 1.3738114833831787\n",
      "  net.2.0.bias_grad: 0.1255839765071869\n",
      "  net.2.0.weight_grad: 0.8132373690605164\n",
      "  net.3.bias_grad: -5.587935447692871e-09\n",
      "  net.3.weight_grad: -1.7741695046424866e-07\n",
      "  node_ip: 192.168.1.248\n",
      "  pid: 57240\n",
      "  sum_loss: -0.02080622222274542\n",
      "  time_since_restore: 28.284371376037598\n",
      "  time_this_iter_s: 0.2813222408294678\n",
      "  time_total_s: 28.284371376037598\n",
      "  timestamp: 1629671099\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 67\n",
      "  trial_id: b58c9_00000\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 6.7/7.4 GiB: ***LOW MEMORY*** less than 10% of the memory on this node is available for use. This can cause unexpected crashes. Consider reducing the memory used by your application or reducing the Ray object store size by setting `object_store_memory` when calling `ray.init`.<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/8 CPUs, 0/1 GPUs, 0.0/1.07 GiB heap, 0.0/0.54 GiB objects (0.0/1.0 accelerator_type:T1000)<br>Current best trial: b58c9_00000 with loss=-0.06032639709301293 and parameters={'trainer_class': <class 'power_cogs.examples.reinforce.reinforce_trainer.ReinforceTrainer'>, 'trainer_config': {'name': 'ReinforceTrainer', 'visualize': False, 'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'logging_config': {'mlflow': {'experiment_name': None, 'save_artifact': False}, 'wandb': {'project': None, 'api_key_file': None, 'log_config': False, 'reinit': False}, 'checkpoint_path': 'checkpoints', 'tensorboard_log_path': None, 'checkpoint_interval': 5}, 'stoppage_config': {'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'loss_threshold': -inf, 'stop': None, 'callbacks': []}, 'tune_config': {'metric': 'loss', 'mode': 'min', 'num_samples': 1, 'name': '', 'checkpoint_freq': 5, 'checkpoint_at_end': True, 'callbacks': [], 'additional_config': {}}, 'optimizer_config': {'_target_': 'torch.optim.adam.Adam', 'params': '???', 'lr': 0.002, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'scheduler_config': {'_target_': 'torch.optim.lr_scheduler.ExponentialLR', 'optimizer': '???', 'gamma': 0.9999, 'last_epoch': -1}, 'dataloader_config': {'_target_': 'torch.utils.data.dataloader.DataLoader', 'dataset': '???', 'batch_size': 1, 'shuffle': False, 'sampler': None, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None}, 'cluster_config': {'name': '', 'cluster_config_path': None, 'join_cluster': False}, 'dataset_config': {'env_name': 'LunarLander-v2', 'max_env_steps': 10000, 'gamma': 0.99, 'discount_rewards': True, 'normalize_rewards': True, 'random_policy': False, 'num_rollouts': 100, 'num_workers': 5}, 'model_config': {'input_dims': 8, 'hidden_dims': [32, 32], 'output_dims': 4, 'output_activation': None, 'use_normal_init': True, 'normal_std': 0.1, 'zero_bias': False}}}<br>Result logdir: /home/shyam/ray_results/TuneTrainer_2021-08-22_15-24-26<br>Number of trials: 1/1 (1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name             </th><th>status  </th><th>loc                </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  average_summed_rewards</th><th style=\"text-align: right;\">  max_summed_rewards</th><th style=\"text-align: right;\">      loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>TuneTrainer_b58c9_00000</td><td>RUNNING </td><td>192.168.1.248:57240</td><td style=\"text-align: right;\">    79</td><td style=\"text-align: right;\">         32.0053</td><td style=\"text-align: right;\">                -98.6689</td><td style=\"text-align: right;\">            -62.3133</td><td style=\"text-align: right;\">-0.0603264</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for TuneTrainer_b58c9_00000:\n",
      "  average_summed_rewards: -21.543668223888552\n",
      "  date: 2021-08-22_15-25-05\n",
      "  done: false\n",
      "  epoch: 0\n",
      "  experiment_id: ea3e4da9a71247ddb0ad6c4bcb6d8084\n",
      "  hostname: shyam-ThinkPad-P53\n",
      "  iterations_since_restore: 81\n",
      "  loss: -0.04936741441488266\n",
      "  max_loss: 0.07514594495296478\n",
      "  max_summed_rewards: -14.489139981871062\n",
      "  net.0.0.bias_grad: -0.13169246912002563\n",
      "  net.0.0.weight_grad: -0.22426125407218933\n",
      "  net.1.0.bias_grad: -0.08497221767902374\n",
      "  net.1.0.weight_grad: -0.4230814576148987\n",
      "  net.2.0.bias_grad: -0.09666246175765991\n",
      "  net.2.0.weight_grad: -0.5980570316314697\n",
      "  net.3.bias_grad: -1.4901161193847656e-08\n",
      "  net.3.weight_grad: -1.2497184798121452e-07\n",
      "  node_ip: 192.168.1.248\n",
      "  pid: 57240\n",
      "  sum_loss: -0.2468370720744133\n",
      "  time_since_restore: 33.36522698402405\n",
      "  time_this_iter_s: 1.0213117599487305\n",
      "  time_total_s: 33.36522698402405\n",
      "  timestamp: 1629671105\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 81\n",
      "  trial_id: b58c9_00000\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 6.7/7.4 GiB: ***LOW MEMORY*** less than 10% of the memory on this node is available for use. This can cause unexpected crashes. Consider reducing the memory used by your application or reducing the Ray object store size by setting `object_store_memory` when calling `ray.init`.<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/8 CPUs, 0/1 GPUs, 0.0/1.07 GiB heap, 0.0/0.54 GiB objects (0.0/1.0 accelerator_type:T1000)<br>Current best trial: b58c9_00000 with loss=-0.018407701328396798 and parameters={'trainer_class': <class 'power_cogs.examples.reinforce.reinforce_trainer.ReinforceTrainer'>, 'trainer_config': {'name': 'ReinforceTrainer', 'visualize': False, 'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'logging_config': {'mlflow': {'experiment_name': None, 'save_artifact': False}, 'wandb': {'project': None, 'api_key_file': None, 'log_config': False, 'reinit': False}, 'checkpoint_path': 'checkpoints', 'tensorboard_log_path': None, 'checkpoint_interval': 5}, 'stoppage_config': {'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'loss_threshold': -inf, 'stop': None, 'callbacks': []}, 'tune_config': {'metric': 'loss', 'mode': 'min', 'num_samples': 1, 'name': '', 'checkpoint_freq': 5, 'checkpoint_at_end': True, 'callbacks': [], 'additional_config': {}}, 'optimizer_config': {'_target_': 'torch.optim.adam.Adam', 'params': '???', 'lr': 0.002, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'scheduler_config': {'_target_': 'torch.optim.lr_scheduler.ExponentialLR', 'optimizer': '???', 'gamma': 0.9999, 'last_epoch': -1}, 'dataloader_config': {'_target_': 'torch.utils.data.dataloader.DataLoader', 'dataset': '???', 'batch_size': 1, 'shuffle': False, 'sampler': None, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None}, 'cluster_config': {'name': '', 'cluster_config_path': None, 'join_cluster': False}, 'dataset_config': {'env_name': 'LunarLander-v2', 'max_env_steps': 10000, 'gamma': 0.99, 'discount_rewards': True, 'normalize_rewards': True, 'random_policy': False, 'num_rollouts': 100, 'num_workers': 5}, 'model_config': {'input_dims': 8, 'hidden_dims': [32, 32], 'output_dims': 4, 'output_activation': None, 'use_normal_init': True, 'normal_std': 0.1, 'zero_bias': False}}}<br>Result logdir: /home/shyam/ray_results/TuneTrainer_2021-08-22_15-24-26<br>Number of trials: 1/1 (1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name             </th><th>status  </th><th>loc                </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  average_summed_rewards</th><th style=\"text-align: right;\">  max_summed_rewards</th><th style=\"text-align: right;\">      loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>TuneTrainer_b58c9_00000</td><td>RUNNING </td><td>192.168.1.248:57240</td><td style=\"text-align: right;\">    85</td><td style=\"text-align: right;\">         38.0784</td><td style=\"text-align: right;\">                -127.681</td><td style=\"text-align: right;\">             16.8673</td><td style=\"text-align: right;\">-0.0184077</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for TuneTrainer_b58c9_00000:\n",
      "  average_summed_rewards: -108.92096580952595\n",
      "  date: 2021-08-22_15-25-11\n",
      "  done: false\n",
      "  epoch: 0\n",
      "  experiment_id: ea3e4da9a71247ddb0ad6c4bcb6d8084\n",
      "  hostname: shyam-ThinkPad-P53\n",
      "  iterations_since_restore: 86\n",
      "  loss: 0.0927922174334526\n",
      "  max_loss: 0.28555846214294434\n",
      "  max_summed_rewards: 27.28509252612793\n",
      "  net.0.0.bias_grad: -0.34247004985809326\n",
      "  net.0.0.weight_grad: -0.7383955121040344\n",
      "  net.1.0.bias_grad: -0.020899947732686996\n",
      "  net.1.0.weight_grad: -0.48582375049591064\n",
      "  net.2.0.bias_grad: -0.20391035079956055\n",
      "  net.2.0.weight_grad: -1.3007502555847168\n",
      "  net.3.bias_grad: 1.4901161193847656e-08\n",
      "  net.3.weight_grad: -4.5355409383773804e-07\n",
      "  node_ip: 192.168.1.248\n",
      "  pid: 57240\n",
      "  sum_loss: 0.46396108716726303\n",
      "  time_since_restore: 39.37305474281311\n",
      "  time_this_iter_s: 1.294673204421997\n",
      "  time_total_s: 39.37305474281311\n",
      "  timestamp: 1629671111\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 86\n",
      "  trial_id: b58c9_00000\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 6.7/7.4 GiB: ***LOW MEMORY*** less than 10% of the memory on this node is available for use. This can cause unexpected crashes. Consider reducing the memory used by your application or reducing the Ray object store size by setting `object_store_memory` when calling `ray.init`.<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/8 CPUs, 0/1 GPUs, 0.0/1.07 GiB heap, 0.0/0.54 GiB objects (0.0/1.0 accelerator_type:T1000)<br>Current best trial: b58c9_00000 with loss=-0.09801256954669953 and parameters={'trainer_class': <class 'power_cogs.examples.reinforce.reinforce_trainer.ReinforceTrainer'>, 'trainer_config': {'name': 'ReinforceTrainer', 'visualize': False, 'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'logging_config': {'mlflow': {'experiment_name': None, 'save_artifact': False}, 'wandb': {'project': None, 'api_key_file': None, 'log_config': False, 'reinit': False}, 'checkpoint_path': 'checkpoints', 'tensorboard_log_path': None, 'checkpoint_interval': 5}, 'stoppage_config': {'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'loss_threshold': -inf, 'stop': None, 'callbacks': []}, 'tune_config': {'metric': 'loss', 'mode': 'min', 'num_samples': 1, 'name': '', 'checkpoint_freq': 5, 'checkpoint_at_end': True, 'callbacks': [], 'additional_config': {}}, 'optimizer_config': {'_target_': 'torch.optim.adam.Adam', 'params': '???', 'lr': 0.002, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'scheduler_config': {'_target_': 'torch.optim.lr_scheduler.ExponentialLR', 'optimizer': '???', 'gamma': 0.9999, 'last_epoch': -1}, 'dataloader_config': {'_target_': 'torch.utils.data.dataloader.DataLoader', 'dataset': '???', 'batch_size': 1, 'shuffle': False, 'sampler': None, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None}, 'cluster_config': {'name': '', 'cluster_config_path': None, 'join_cluster': False}, 'dataset_config': {'env_name': 'LunarLander-v2', 'max_env_steps': 10000, 'gamma': 0.99, 'discount_rewards': True, 'normalize_rewards': True, 'random_policy': False, 'num_rollouts': 100, 'num_workers': 5}, 'model_config': {'input_dims': 8, 'hidden_dims': [32, 32], 'output_dims': 4, 'output_activation': None, 'use_normal_init': True, 'normal_std': 0.1, 'zero_bias': False}}}<br>Result logdir: /home/shyam/ray_results/TuneTrainer_2021-08-22_15-24-26<br>Number of trials: 1/1 (1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name             </th><th>status  </th><th>loc                </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  average_summed_rewards</th><th style=\"text-align: right;\">  max_summed_rewards</th><th style=\"text-align: right;\">      loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>TuneTrainer_b58c9_00000</td><td>RUNNING </td><td>192.168.1.248:57240</td><td style=\"text-align: right;\">    87</td><td style=\"text-align: right;\">          43.365</td><td style=\"text-align: right;\">                 -51.261</td><td style=\"text-align: right;\">             3.22556</td><td style=\"text-align: right;\">-0.0980126</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for TuneTrainer_b58c9_00000:\n",
      "  average_summed_rewards: 26.662359914254903\n",
      "  date: 2021-08-22_15-25-18\n",
      "  done: false\n",
      "  epoch: 3\n",
      "  experiment_id: ea3e4da9a71247ddb0ad6c4bcb6d8084\n",
      "  hostname: shyam-ThinkPad-P53\n",
      "  iterations_since_restore: 89\n",
      "  loss: -0.1627722304314375\n",
      "  max_loss: 0.08257459104061127\n",
      "  max_summed_rewards: 52.588957508380496\n",
      "  net.0.0.bias_grad: -0.14344406127929688\n",
      "  net.0.0.weight_grad: -0.041086770594120026\n",
      "  net.1.0.bias_grad: -0.03797777742147446\n",
      "  net.1.0.weight_grad: -0.12703120708465576\n",
      "  net.2.0.bias_grad: 0.037355225533246994\n",
      "  net.2.0.weight_grad: 0.1925843358039856\n",
      "  net.3.bias_grad: -2.7939677238464355e-09\n",
      "  net.3.weight_grad: -2.112938091158867e-08\n",
      "  node_ip: 192.168.1.248\n",
      "  pid: 57240\n",
      "  sum_loss: -0.8138611521571875\n",
      "  time_since_restore: 45.62113881111145\n",
      "  time_this_iter_s: 1.8787720203399658\n",
      "  time_total_s: 45.62113881111145\n",
      "  timestamp: 1629671118\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 89\n",
      "  trial_id: b58c9_00000\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 6.7/7.4 GiB: ***LOW MEMORY*** less than 10% of the memory on this node is available for use. This can cause unexpected crashes. Consider reducing the memory used by your application or reducing the Ray object store size by setting `object_store_memory` when calling `ray.init`.<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/8 CPUs, 0/1 GPUs, 0.0/1.07 GiB heap, 0.0/0.54 GiB objects (0.0/1.0 accelerator_type:T1000)<br>Current best trial: b58c9_00000 with loss=0.08174543716013431 and parameters={'trainer_class': <class 'power_cogs.examples.reinforce.reinforce_trainer.ReinforceTrainer'>, 'trainer_config': {'name': 'ReinforceTrainer', 'visualize': False, 'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'logging_config': {'mlflow': {'experiment_name': None, 'save_artifact': False}, 'wandb': {'project': None, 'api_key_file': None, 'log_config': False, 'reinit': False}, 'checkpoint_path': 'checkpoints', 'tensorboard_log_path': None, 'checkpoint_interval': 5}, 'stoppage_config': {'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'loss_threshold': -inf, 'stop': None, 'callbacks': []}, 'tune_config': {'metric': 'loss', 'mode': 'min', 'num_samples': 1, 'name': '', 'checkpoint_freq': 5, 'checkpoint_at_end': True, 'callbacks': [], 'additional_config': {}}, 'optimizer_config': {'_target_': 'torch.optim.adam.Adam', 'params': '???', 'lr': 0.002, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'scheduler_config': {'_target_': 'torch.optim.lr_scheduler.ExponentialLR', 'optimizer': '???', 'gamma': 0.9999, 'last_epoch': -1}, 'dataloader_config': {'_target_': 'torch.utils.data.dataloader.DataLoader', 'dataset': '???', 'batch_size': 1, 'shuffle': False, 'sampler': None, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None}, 'cluster_config': {'name': '', 'cluster_config_path': None, 'join_cluster': False}, 'dataset_config': {'env_name': 'LunarLander-v2', 'max_env_steps': 10000, 'gamma': 0.99, 'discount_rewards': True, 'normalize_rewards': True, 'random_policy': False, 'num_rollouts': 100, 'num_workers': 5}, 'model_config': {'input_dims': 8, 'hidden_dims': [32, 32], 'output_dims': 4, 'output_activation': None, 'use_normal_init': True, 'normal_std': 0.1, 'zero_bias': False}}}<br>Result logdir: /home/shyam/ray_results/TuneTrainer_2021-08-22_15-24-26<br>Number of trials: 1/1 (1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name             </th><th>status  </th><th>loc                </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  average_summed_rewards</th><th style=\"text-align: right;\">  max_summed_rewards</th><th style=\"text-align: right;\">     loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>TuneTrainer_b58c9_00000</td><td>RUNNING </td><td>192.168.1.248:57240</td><td style=\"text-align: right;\">    91</td><td style=\"text-align: right;\">         48.1774</td><td style=\"text-align: right;\">                -70.5558</td><td style=\"text-align: right;\">             53.4717</td><td style=\"text-align: right;\">0.0817454</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for TuneTrainer_b58c9_00000:\n",
      "  average_summed_rewards: -110.85914535334261\n",
      "  date: 2021-08-22_15-25-24\n",
      "  done: false\n",
      "  epoch: 1\n",
      "  experiment_id: ea3e4da9a71247ddb0ad6c4bcb6d8084\n",
      "  hostname: shyam-ThinkPad-P53\n",
      "  iterations_since_restore: 92\n",
      "  loss: 0.18169374577701092\n",
      "  max_loss: 0.44866278767585754\n",
      "  max_summed_rewards: 14.144157656401797\n",
      "  net.0.0.bias_grad: -0.8245294690132141\n",
      "  net.0.0.weight_grad: -0.7290650010108948\n",
      "  net.1.0.bias_grad: -0.6659967303276062\n",
      "  net.1.0.weight_grad: -2.712100028991699\n",
      "  net.2.0.bias_grad: 0.017761066555976868\n",
      "  net.2.0.weight_grad: 0.13339455425739288\n",
      "  net.3.bias_grad: -1.4901161193847656e-07\n",
      "  net.3.weight_grad: -7.656490197405219e-08\n",
      "  node_ip: 192.168.1.248\n",
      "  pid: 57240\n",
      "  sum_loss: 0.9084687288850546\n",
      "  time_since_restore: 52.185094118118286\n",
      "  time_this_iter_s: 4.007687091827393\n",
      "  time_total_s: 52.185094118118286\n",
      "  timestamp: 1629671124\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 92\n",
      "  trial_id: b58c9_00000\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 6.7/7.4 GiB: ***LOW MEMORY*** less than 10% of the memory on this node is available for use. This can cause unexpected crashes. Consider reducing the memory used by your application or reducing the Ray object store size by setting `object_store_memory` when calling `ray.init`.<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/8 CPUs, 0/1 GPUs, 0.0/1.07 GiB heap, 0.0/0.54 GiB objects (0.0/1.0 accelerator_type:T1000)<br>Current best trial: b58c9_00000 with loss=0.04078814648091793 and parameters={'trainer_class': <class 'power_cogs.examples.reinforce.reinforce_trainer.ReinforceTrainer'>, 'trainer_config': {'name': 'ReinforceTrainer', 'visualize': False, 'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'logging_config': {'mlflow': {'experiment_name': None, 'save_artifact': False}, 'wandb': {'project': None, 'api_key_file': None, 'log_config': False, 'reinit': False}, 'checkpoint_path': 'checkpoints', 'tensorboard_log_path': None, 'checkpoint_interval': 5}, 'stoppage_config': {'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'loss_threshold': -inf, 'stop': None, 'callbacks': []}, 'tune_config': {'metric': 'loss', 'mode': 'min', 'num_samples': 1, 'name': '', 'checkpoint_freq': 5, 'checkpoint_at_end': True, 'callbacks': [], 'additional_config': {}}, 'optimizer_config': {'_target_': 'torch.optim.adam.Adam', 'params': '???', 'lr': 0.002, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'scheduler_config': {'_target_': 'torch.optim.lr_scheduler.ExponentialLR', 'optimizer': '???', 'gamma': 0.9999, 'last_epoch': -1}, 'dataloader_config': {'_target_': 'torch.utils.data.dataloader.DataLoader', 'dataset': '???', 'batch_size': 1, 'shuffle': False, 'sampler': None, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None}, 'cluster_config': {'name': '', 'cluster_config_path': None, 'join_cluster': False}, 'dataset_config': {'env_name': 'LunarLander-v2', 'max_env_steps': 10000, 'gamma': 0.99, 'discount_rewards': True, 'normalize_rewards': True, 'random_policy': False, 'num_rollouts': 100, 'num_workers': 5}, 'model_config': {'input_dims': 8, 'hidden_dims': [32, 32], 'output_dims': 4, 'output_activation': None, 'use_normal_init': True, 'normal_std': 0.1, 'zero_bias': False}}}<br>Result logdir: /home/shyam/ray_results/TuneTrainer_2021-08-22_15-24-26<br>Number of trials: 1/1 (1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name             </th><th>status  </th><th>loc                </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  average_summed_rewards</th><th style=\"text-align: right;\">  max_summed_rewards</th><th style=\"text-align: right;\">     loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>TuneTrainer_b58c9_00000</td><td>RUNNING </td><td>192.168.1.248:57240</td><td style=\"text-align: right;\">    95</td><td style=\"text-align: right;\">         53.1195</td><td style=\"text-align: right;\">                -34.1411</td><td style=\"text-align: right;\">            -13.7534</td><td style=\"text-align: right;\">0.0407881</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for TuneTrainer_b58c9_00000:\n",
      "  average_summed_rewards: -42.73944989650491\n",
      "  date: 2021-08-22_15-25-30\n",
      "  done: false\n",
      "  epoch: 2\n",
      "  experiment_id: ea3e4da9a71247ddb0ad6c4bcb6d8084\n",
      "  hostname: shyam-ThinkPad-P53\n",
      "  iterations_since_restore: 108\n",
      "  loss: -0.01496085710823536\n",
      "  max_loss: 0.078209288418293\n",
      "  max_summed_rewards: -2.2042309226669943\n",
      "  net.0.0.bias_grad: 0.27922821044921875\n",
      "  net.0.0.weight_grad: 0.17943432927131653\n",
      "  net.1.0.bias_grad: 0.06205965578556061\n",
      "  net.1.0.weight_grad: 0.32254281640052795\n",
      "  net.2.0.bias_grad: 0.13970379531383514\n",
      "  net.2.0.weight_grad: 0.6385571956634521\n",
      "  net.3.bias_grad: 7.450580596923828e-09\n",
      "  net.3.weight_grad: -9.677023626863956e-10\n",
      "  node_ip: 192.168.1.248\n",
      "  pid: 57240\n",
      "  sum_loss: -0.0748042855411768\n",
      "  time_since_restore: 56.53879237174988\n",
      "  time_this_iter_s: 0.36773180961608887\n",
      "  time_total_s: 56.53879237174988\n",
      "  timestamp: 1629671130\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 108\n",
      "  trial_id: b58c9_00000\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 6.7/7.4 GiB: ***LOW MEMORY*** less than 10% of the memory on this node is available for use. This can cause unexpected crashes. Consider reducing the memory used by your application or reducing the Ray object store size by setting `object_store_memory` when calling `ray.init`.<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/8 CPUs, 0/1 GPUs, 0.0/1.07 GiB heap, 0.0/0.54 GiB objects (0.0/1.0 accelerator_type:T1000)<br>Current best trial: b58c9_00000 with loss=0.0061114267911762 and parameters={'trainer_class': <class 'power_cogs.examples.reinforce.reinforce_trainer.ReinforceTrainer'>, 'trainer_config': {'name': 'ReinforceTrainer', 'visualize': False, 'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'logging_config': {'mlflow': {'experiment_name': None, 'save_artifact': False}, 'wandb': {'project': None, 'api_key_file': None, 'log_config': False, 'reinit': False}, 'checkpoint_path': 'checkpoints', 'tensorboard_log_path': None, 'checkpoint_interval': 5}, 'stoppage_config': {'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'loss_threshold': -inf, 'stop': None, 'callbacks': []}, 'tune_config': {'metric': 'loss', 'mode': 'min', 'num_samples': 1, 'name': '', 'checkpoint_freq': 5, 'checkpoint_at_end': True, 'callbacks': [], 'additional_config': {}}, 'optimizer_config': {'_target_': 'torch.optim.adam.Adam', 'params': '???', 'lr': 0.002, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'scheduler_config': {'_target_': 'torch.optim.lr_scheduler.ExponentialLR', 'optimizer': '???', 'gamma': 0.9999, 'last_epoch': -1}, 'dataloader_config': {'_target_': 'torch.utils.data.dataloader.DataLoader', 'dataset': '???', 'batch_size': 1, 'shuffle': False, 'sampler': None, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None}, 'cluster_config': {'name': '', 'cluster_config_path': None, 'join_cluster': False}, 'dataset_config': {'env_name': 'LunarLander-v2', 'max_env_steps': 10000, 'gamma': 0.99, 'discount_rewards': True, 'normalize_rewards': True, 'random_policy': False, 'num_rollouts': 100, 'num_workers': 5}, 'model_config': {'input_dims': 8, 'hidden_dims': [32, 32], 'output_dims': 4, 'output_activation': None, 'use_normal_init': True, 'normal_std': 0.1, 'zero_bias': False}}}<br>Result logdir: /home/shyam/ray_results/TuneTrainer_2021-08-22_15-24-26<br>Number of trials: 1/1 (1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name             </th><th>status  </th><th>loc                </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  average_summed_rewards</th><th style=\"text-align: right;\">  max_summed_rewards</th><th style=\"text-align: right;\">      loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>TuneTrainer_b58c9_00000</td><td>RUNNING </td><td>192.168.1.248:57240</td><td style=\"text-align: right;\">   111</td><td style=\"text-align: right;\">         57.4701</td><td style=\"text-align: right;\">                -29.1848</td><td style=\"text-align: right;\">            -18.0449</td><td style=\"text-align: right;\">0.00611143</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for TuneTrainer_b58c9_00000:\n",
      "  average_summed_rewards: -136.24161910549472\n",
      "  date: 2021-08-22_15-25-35\n",
      "  done: false\n",
      "  epoch: 0\n",
      "  experiment_id: ea3e4da9a71247ddb0ad6c4bcb6d8084\n",
      "  hostname: shyam-ThinkPad-P53\n",
      "  iterations_since_restore: 121\n",
      "  loss: 0.05948927905410528\n",
      "  max_loss: 0.15825605392456055\n",
      "  max_summed_rewards: -56.474195785434155\n",
      "  net.0.0.bias_grad: -0.34145182371139526\n",
      "  net.0.0.weight_grad: -0.5060182213783264\n",
      "  net.1.0.bias_grad: -0.18337740004062653\n",
      "  net.1.0.weight_grad: -0.8270753622055054\n",
      "  net.2.0.bias_grad: 0.24638605117797852\n",
      "  net.2.0.weight_grad: 1.3166913986206055\n",
      "  net.3.bias_grad: 3.725290298461914e-09\n",
      "  net.3.weight_grad: 9.985160431824625e-08\n",
      "  node_ip: 192.168.1.248\n",
      "  pid: 57240\n",
      "  sum_loss: 0.2974463952705264\n",
      "  time_since_restore: 61.40763187408447\n",
      "  time_this_iter_s: 0.5116226673126221\n",
      "  time_total_s: 61.40763187408447\n",
      "  timestamp: 1629671135\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 121\n",
      "  trial_id: b58c9_00000\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 6.7/7.4 GiB: ***LOW MEMORY*** less than 10% of the memory on this node is available for use. This can cause unexpected crashes. Consider reducing the memory used by your application or reducing the Ray object store size by setting `object_store_memory` when calling `ray.init`.<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/8 CPUs, 0/1 GPUs, 0.0/1.07 GiB heap, 0.0/0.54 GiB objects (0.0/1.0 accelerator_type:T1000)<br>Current best trial: b58c9_00000 with loss=0.09625650718808174 and parameters={'trainer_class': <class 'power_cogs.examples.reinforce.reinforce_trainer.ReinforceTrainer'>, 'trainer_config': {'name': 'ReinforceTrainer', 'visualize': False, 'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'logging_config': {'mlflow': {'experiment_name': None, 'save_artifact': False}, 'wandb': {'project': None, 'api_key_file': None, 'log_config': False, 'reinit': False}, 'checkpoint_path': 'checkpoints', 'tensorboard_log_path': None, 'checkpoint_interval': 5}, 'stoppage_config': {'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'loss_threshold': -inf, 'stop': None, 'callbacks': []}, 'tune_config': {'metric': 'loss', 'mode': 'min', 'num_samples': 1, 'name': '', 'checkpoint_freq': 5, 'checkpoint_at_end': True, 'callbacks': [], 'additional_config': {}}, 'optimizer_config': {'_target_': 'torch.optim.adam.Adam', 'params': '???', 'lr': 0.002, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'scheduler_config': {'_target_': 'torch.optim.lr_scheduler.ExponentialLR', 'optimizer': '???', 'gamma': 0.9999, 'last_epoch': -1}, 'dataloader_config': {'_target_': 'torch.utils.data.dataloader.DataLoader', 'dataset': '???', 'batch_size': 1, 'shuffle': False, 'sampler': None, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None}, 'cluster_config': {'name': '', 'cluster_config_path': None, 'join_cluster': False}, 'dataset_config': {'env_name': 'LunarLander-v2', 'max_env_steps': 10000, 'gamma': 0.99, 'discount_rewards': True, 'normalize_rewards': True, 'random_policy': False, 'num_rollouts': 100, 'num_workers': 5}, 'model_config': {'input_dims': 8, 'hidden_dims': [32, 32], 'output_dims': 4, 'output_activation': None, 'use_normal_init': True, 'normal_std': 0.1, 'zero_bias': False}}}<br>Result logdir: /home/shyam/ray_results/TuneTrainer_2021-08-22_15-24-26<br>Number of trials: 1/1 (1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name             </th><th>status  </th><th>loc                </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  average_summed_rewards</th><th style=\"text-align: right;\">  max_summed_rewards</th><th style=\"text-align: right;\">     loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>TuneTrainer_b58c9_00000</td><td>RUNNING </td><td>192.168.1.248:57240</td><td style=\"text-align: right;\">   123</td><td style=\"text-align: right;\">         62.1678</td><td style=\"text-align: right;\">                -109.873</td><td style=\"text-align: right;\">            -86.4937</td><td style=\"text-align: right;\">0.0962565</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for TuneTrainer_b58c9_00000:\n",
      "  average_summed_rewards: -36.116588394717525\n",
      "  date: 2021-08-22_15-25-40\n",
      "  done: false\n",
      "  epoch: 4\n",
      "  experiment_id: ea3e4da9a71247ddb0ad6c4bcb6d8084\n",
      "  hostname: shyam-ThinkPad-P53\n",
      "  iterations_since_restore: 130\n",
      "  loss: 0.03036163914948702\n",
      "  max_loss: 0.29135048389434814\n",
      "  max_summed_rewards: -17.30832731815802\n",
      "  net.0.0.bias_grad: -0.38724949955940247\n",
      "  net.0.0.weight_grad: -0.5926477909088135\n",
      "  net.1.0.bias_grad: -0.2229461371898651\n",
      "  net.1.0.weight_grad: -0.6886385083198547\n",
      "  net.2.0.bias_grad: -0.2856287956237793\n",
      "  net.2.0.weight_grad: -1.209222435951233\n",
      "  net.3.bias_grad: 1.862645149230957e-08\n",
      "  net.3.weight_grad: -2.2584572434425354e-08\n",
      "  node_ip: 192.168.1.248\n",
      "  pid: 57240\n",
      "  sum_loss: 0.1518081957474351\n",
      "  time_since_restore: 66.12222337722778\n",
      "  time_this_iter_s: 1.3702208995819092\n",
      "  time_total_s: 66.12222337722778\n",
      "  timestamp: 1629671140\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 130\n",
      "  trial_id: b58c9_00000\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 6.7/7.4 GiB: ***LOW MEMORY*** less than 10% of the memory on this node is available for use. This can cause unexpected crashes. Consider reducing the memory used by your application or reducing the Ray object store size by setting `object_store_memory` when calling `ray.init`.<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/8 CPUs, 0/1 GPUs, 0.0/1.07 GiB heap, 0.0/0.54 GiB objects (0.0/1.0 accelerator_type:T1000)<br>Current best trial: b58c9_00000 with loss=0.4221898741554469 and parameters={'trainer_class': <class 'power_cogs.examples.reinforce.reinforce_trainer.ReinforceTrainer'>, 'trainer_config': {'name': 'ReinforceTrainer', 'visualize': False, 'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'logging_config': {'mlflow': {'experiment_name': None, 'save_artifact': False}, 'wandb': {'project': None, 'api_key_file': None, 'log_config': False, 'reinit': False}, 'checkpoint_path': 'checkpoints', 'tensorboard_log_path': None, 'checkpoint_interval': 5}, 'stoppage_config': {'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'loss_threshold': -inf, 'stop': None, 'callbacks': []}, 'tune_config': {'metric': 'loss', 'mode': 'min', 'num_samples': 1, 'name': '', 'checkpoint_freq': 5, 'checkpoint_at_end': True, 'callbacks': [], 'additional_config': {}}, 'optimizer_config': {'_target_': 'torch.optim.adam.Adam', 'params': '???', 'lr': 0.002, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'scheduler_config': {'_target_': 'torch.optim.lr_scheduler.ExponentialLR', 'optimizer': '???', 'gamma': 0.9999, 'last_epoch': -1}, 'dataloader_config': {'_target_': 'torch.utils.data.dataloader.DataLoader', 'dataset': '???', 'batch_size': 1, 'shuffle': False, 'sampler': None, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None}, 'cluster_config': {'name': '', 'cluster_config_path': None, 'join_cluster': False}, 'dataset_config': {'env_name': 'LunarLander-v2', 'max_env_steps': 10000, 'gamma': 0.99, 'discount_rewards': True, 'normalize_rewards': True, 'random_policy': False, 'num_rollouts': 100, 'num_workers': 5}, 'model_config': {'input_dims': 8, 'hidden_dims': [32, 32], 'output_dims': 4, 'output_activation': None, 'use_normal_init': True, 'normal_std': 0.1, 'zero_bias': False}}}<br>Result logdir: /home/shyam/ray_results/TuneTrainer_2021-08-22_15-24-26<br>Number of trials: 1/1 (1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name             </th><th>status  </th><th>loc                </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  average_summed_rewards</th><th style=\"text-align: right;\">  max_summed_rewards</th><th style=\"text-align: right;\">   loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>TuneTrainer_b58c9_00000</td><td>RUNNING </td><td>192.168.1.248:57240</td><td style=\"text-align: right;\">   131</td><td style=\"text-align: right;\">         67.6692</td><td style=\"text-align: right;\">                -13.8391</td><td style=\"text-align: right;\">             51.4787</td><td style=\"text-align: right;\">0.42219</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for TuneTrainer_b58c9_00000:\n",
      "  average_summed_rewards: -9.156826012005904\n",
      "  date: 2021-08-22_15-25-46\n",
      "  done: false\n",
      "  epoch: 2\n",
      "  experiment_id: ea3e4da9a71247ddb0ad6c4bcb6d8084\n",
      "  hostname: shyam-ThinkPad-P53\n",
      "  iterations_since_restore: 133\n",
      "  loss: 0.16460455134510993\n",
      "  max_loss: 1.315956711769104\n",
      "  max_summed_rewards: 29.769899365730804\n",
      "  net.0.0.bias_grad: -0.7119587659835815\n",
      "  net.0.0.weight_grad: -0.5120675563812256\n",
      "  net.1.0.bias_grad: -0.5121210217475891\n",
      "  net.1.0.weight_grad: -1.624690294265747\n",
      "  net.2.0.bias_grad: -0.16886964440345764\n",
      "  net.2.0.weight_grad: -0.5910794734954834\n",
      "  net.3.bias_grad: -4.0978193283081055e-08\n",
      "  net.3.weight_grad: -7.390917744487524e-08\n",
      "  node_ip: 192.168.1.248\n",
      "  pid: 57240\n",
      "  sum_loss: 0.8230227567255497\n",
      "  time_since_restore: 71.51898074150085\n",
      "  time_this_iter_s: 3.0012855529785156\n",
      "  time_total_s: 71.51898074150085\n",
      "  timestamp: 1629671146\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 133\n",
      "  trial_id: b58c9_00000\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 6.7/7.4 GiB: ***LOW MEMORY*** less than 10% of the memory on this node is available for use. This can cause unexpected crashes. Consider reducing the memory used by your application or reducing the Ray object store size by setting `object_store_memory` when calling `ray.init`.<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/8 CPUs, 0/1 GPUs, 0.0/1.07 GiB heap, 0.0/0.54 GiB objects (0.0/1.0 accelerator_type:T1000)<br>Current best trial: b58c9_00000 with loss=0.12037951201200485 and parameters={'trainer_class': <class 'power_cogs.examples.reinforce.reinforce_trainer.ReinforceTrainer'>, 'trainer_config': {'name': 'ReinforceTrainer', 'visualize': False, 'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'logging_config': {'mlflow': {'experiment_name': None, 'save_artifact': False}, 'wandb': {'project': None, 'api_key_file': None, 'log_config': False, 'reinit': False}, 'checkpoint_path': 'checkpoints', 'tensorboard_log_path': None, 'checkpoint_interval': 5}, 'stoppage_config': {'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'loss_threshold': -inf, 'stop': None, 'callbacks': []}, 'tune_config': {'metric': 'loss', 'mode': 'min', 'num_samples': 1, 'name': '', 'checkpoint_freq': 5, 'checkpoint_at_end': True, 'callbacks': [], 'additional_config': {}}, 'optimizer_config': {'_target_': 'torch.optim.adam.Adam', 'params': '???', 'lr': 0.002, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'scheduler_config': {'_target_': 'torch.optim.lr_scheduler.ExponentialLR', 'optimizer': '???', 'gamma': 0.9999, 'last_epoch': -1}, 'dataloader_config': {'_target_': 'torch.utils.data.dataloader.DataLoader', 'dataset': '???', 'batch_size': 1, 'shuffle': False, 'sampler': None, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None}, 'cluster_config': {'name': '', 'cluster_config_path': None, 'join_cluster': False}, 'dataset_config': {'env_name': 'LunarLander-v2', 'max_env_steps': 10000, 'gamma': 0.99, 'discount_rewards': True, 'normalize_rewards': True, 'random_policy': False, 'num_rollouts': 100, 'num_workers': 5}, 'model_config': {'input_dims': 8, 'hidden_dims': [32, 32], 'output_dims': 4, 'output_activation': None, 'use_normal_init': True, 'normal_std': 0.1, 'zero_bias': False}}}<br>Result logdir: /home/shyam/ray_results/TuneTrainer_2021-08-22_15-24-26<br>Number of trials: 1/1 (1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name             </th><th>status  </th><th>loc                </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  average_summed_rewards</th><th style=\"text-align: right;\">  max_summed_rewards</th><th style=\"text-align: right;\">   loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>TuneTrainer_b58c9_00000</td><td>RUNNING </td><td>192.168.1.248:57240</td><td style=\"text-align: right;\">   134</td><td style=\"text-align: right;\">         72.9892</td><td style=\"text-align: right;\">                 -12.812</td><td style=\"text-align: right;\">             18.0516</td><td style=\"text-align: right;\">0.12038</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for TuneTrainer_b58c9_00000:\n",
      "  average_summed_rewards: 36.47284433496553\n",
      "  date: 2021-08-22_15-25-52\n",
      "  done: false\n",
      "  epoch: 0\n",
      "  experiment_id: ea3e4da9a71247ddb0ad6c4bcb6d8084\n",
      "  hostname: shyam-ThinkPad-P53\n",
      "  iterations_since_restore: 136\n",
      "  loss: -0.11529331579804421\n",
      "  max_loss: 0.736659586429596\n",
      "  max_summed_rewards: 191.20955970271865\n",
      "  net.0.0.bias_grad: -0.4268302917480469\n",
      "  net.0.0.weight_grad: -0.3760391175746918\n",
      "  net.1.0.bias_grad: -0.3534277081489563\n",
      "  net.1.0.weight_grad: -1.2130563259124756\n",
      "  net.2.0.bias_grad: -0.19524508714675903\n",
      "  net.2.0.weight_grad: -0.8931980729103088\n",
      "  net.3.bias_grad: 2.7939677238464355e-08\n",
      "  net.3.weight_grad: 1.1750671546906233e-08\n",
      "  node_ip: 192.168.1.248\n",
      "  pid: 57240\n",
      "  sum_loss: -0.576466578990221\n",
      "  time_since_restore: 77.47113084793091\n",
      "  time_this_iter_s: 2.875709056854248\n",
      "  time_total_s: 77.47113084793091\n",
      "  timestamp: 1629671152\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 136\n",
      "  trial_id: b58c9_00000\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 6.7/7.4 GiB: ***LOW MEMORY*** less than 10% of the memory on this node is available for use. This can cause unexpected crashes. Consider reducing the memory used by your application or reducing the Ray object store size by setting `object_store_memory` when calling `ray.init`.<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/8 CPUs, 0/1 GPUs, 0.0/1.07 GiB heap, 0.0/0.54 GiB objects (0.0/1.0 accelerator_type:T1000)<br>Current best trial: b58c9_00000 with loss=-0.22278135344386102 and parameters={'trainer_class': <class 'power_cogs.examples.reinforce.reinforce_trainer.ReinforceTrainer'>, 'trainer_config': {'name': 'ReinforceTrainer', 'visualize': False, 'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'logging_config': {'mlflow': {'experiment_name': None, 'save_artifact': False}, 'wandb': {'project': None, 'api_key_file': None, 'log_config': False, 'reinit': False}, 'checkpoint_path': 'checkpoints', 'tensorboard_log_path': None, 'checkpoint_interval': 5}, 'stoppage_config': {'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'loss_threshold': -inf, 'stop': None, 'callbacks': []}, 'tune_config': {'metric': 'loss', 'mode': 'min', 'num_samples': 1, 'name': '', 'checkpoint_freq': 5, 'checkpoint_at_end': True, 'callbacks': [], 'additional_config': {}}, 'optimizer_config': {'_target_': 'torch.optim.adam.Adam', 'params': '???', 'lr': 0.002, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'scheduler_config': {'_target_': 'torch.optim.lr_scheduler.ExponentialLR', 'optimizer': '???', 'gamma': 0.9999, 'last_epoch': -1}, 'dataloader_config': {'_target_': 'torch.utils.data.dataloader.DataLoader', 'dataset': '???', 'batch_size': 1, 'shuffle': False, 'sampler': None, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None}, 'cluster_config': {'name': '', 'cluster_config_path': None, 'join_cluster': False}, 'dataset_config': {'env_name': 'LunarLander-v2', 'max_env_steps': 10000, 'gamma': 0.99, 'discount_rewards': True, 'normalize_rewards': True, 'random_policy': False, 'num_rollouts': 100, 'num_workers': 5}, 'model_config': {'input_dims': 8, 'hidden_dims': [32, 32], 'output_dims': 4, 'output_activation': None, 'use_normal_init': True, 'normal_std': 0.1, 'zero_bias': False}}}<br>Result logdir: /home/shyam/ray_results/TuneTrainer_2021-08-22_15-24-26<br>Number of trials: 1/1 (1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name             </th><th>status  </th><th>loc                </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  average_summed_rewards</th><th style=\"text-align: right;\">  max_summed_rewards</th><th style=\"text-align: right;\">     loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>TuneTrainer_b58c9_00000</td><td>RUNNING </td><td>192.168.1.248:57240</td><td style=\"text-align: right;\">   137</td><td style=\"text-align: right;\">         78.8915</td><td style=\"text-align: right;\">                 56.7254</td><td style=\"text-align: right;\">             183.825</td><td style=\"text-align: right;\">-0.222781</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for TuneTrainer_b58c9_00000:\n",
      "  average_summed_rewards: -21.256167421535828\n",
      "  date: 2021-08-22_15-25-57\n",
      "  done: false\n",
      "  epoch: 0\n",
      "  experiment_id: ea3e4da9a71247ddb0ad6c4bcb6d8084\n",
      "  hostname: shyam-ThinkPad-P53\n",
      "  iterations_since_restore: 141\n",
      "  loss: 0.06300178915262222\n",
      "  max_loss: 0.29963141679763794\n",
      "  max_summed_rewards: 26.640889379227488\n",
      "  net.0.0.bias_grad: -0.2721911072731018\n",
      "  net.0.0.weight_grad: -0.05940985679626465\n",
      "  net.1.0.bias_grad: -0.07925589382648468\n",
      "  net.1.0.weight_grad: -0.2752135992050171\n",
      "  net.2.0.bias_grad: -0.10192270576953888\n",
      "  net.2.0.weight_grad: -0.658330500125885\n",
      "  net.3.bias_grad: 1.4901161193847656e-08\n",
      "  net.3.weight_grad: 3.2887328416109085e-07\n",
      "  node_ip: 192.168.1.248\n",
      "  pid: 57240\n",
      "  sum_loss: 0.3150089457631111\n",
      "  time_since_restore: 82.42850613594055\n",
      "  time_this_iter_s: 0.9029805660247803\n",
      "  time_total_s: 82.42850613594055\n",
      "  timestamp: 1629671157\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 141\n",
      "  trial_id: b58c9_00000\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 6.7/7.4 GiB: ***LOW MEMORY*** less than 10% of the memory on this node is available for use. This can cause unexpected crashes. Consider reducing the memory used by your application or reducing the Ray object store size by setting `object_store_memory` when calling `ray.init`.<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/8 CPUs, 0/1 GPUs, 0.0/1.07 GiB heap, 0.0/0.54 GiB objects (0.0/1.0 accelerator_type:T1000)<br>Current best trial: b58c9_00000 with loss=0.09458951503038407 and parameters={'trainer_class': <class 'power_cogs.examples.reinforce.reinforce_trainer.ReinforceTrainer'>, 'trainer_config': {'name': 'ReinforceTrainer', 'visualize': False, 'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'logging_config': {'mlflow': {'experiment_name': None, 'save_artifact': False}, 'wandb': {'project': None, 'api_key_file': None, 'log_config': False, 'reinit': False}, 'checkpoint_path': 'checkpoints', 'tensorboard_log_path': None, 'checkpoint_interval': 5}, 'stoppage_config': {'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'loss_threshold': -inf, 'stop': None, 'callbacks': []}, 'tune_config': {'metric': 'loss', 'mode': 'min', 'num_samples': 1, 'name': '', 'checkpoint_freq': 5, 'checkpoint_at_end': True, 'callbacks': [], 'additional_config': {}}, 'optimizer_config': {'_target_': 'torch.optim.adam.Adam', 'params': '???', 'lr': 0.002, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'scheduler_config': {'_target_': 'torch.optim.lr_scheduler.ExponentialLR', 'optimizer': '???', 'gamma': 0.9999, 'last_epoch': -1}, 'dataloader_config': {'_target_': 'torch.utils.data.dataloader.DataLoader', 'dataset': '???', 'batch_size': 1, 'shuffle': False, 'sampler': None, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None}, 'cluster_config': {'name': '', 'cluster_config_path': None, 'join_cluster': False}, 'dataset_config': {'env_name': 'LunarLander-v2', 'max_env_steps': 10000, 'gamma': 0.99, 'discount_rewards': True, 'normalize_rewards': True, 'random_policy': False, 'num_rollouts': 100, 'num_workers': 5}, 'model_config': {'input_dims': 8, 'hidden_dims': [32, 32], 'output_dims': 4, 'output_activation': None, 'use_normal_init': True, 'normal_std': 0.1, 'zero_bias': False}}}<br>Result logdir: /home/shyam/ray_results/TuneTrainer_2021-08-22_15-24-26<br>Number of trials: 1/1 (1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name             </th><th>status  </th><th>loc                </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  average_summed_rewards</th><th style=\"text-align: right;\">  max_summed_rewards</th><th style=\"text-align: right;\">     loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>TuneTrainer_b58c9_00000</td><td>RUNNING </td><td>192.168.1.248:57240</td><td style=\"text-align: right;\">   143</td><td style=\"text-align: right;\">         84.0822</td><td style=\"text-align: right;\">                 52.9215</td><td style=\"text-align: right;\">             242.316</td><td style=\"text-align: right;\">0.0945895</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for TuneTrainer_b58c9_00000:\n",
      "  average_summed_rewards: 0.514566304120369\n",
      "  date: 2021-08-22_15-26-03\n",
      "  done: false\n",
      "  epoch: 2\n",
      "  experiment_id: ea3e4da9a71247ddb0ad6c4bcb6d8084\n",
      "  hostname: shyam-ThinkPad-P53\n",
      "  iterations_since_restore: 148\n",
      "  loss: -0.05698042921721935\n",
      "  max_loss: 0.10752548277378082\n",
      "  max_summed_rewards: 36.56473061776636\n",
      "  net.0.0.bias_grad: -0.8335903882980347\n",
      "  net.0.0.weight_grad: -0.6318800449371338\n",
      "  net.1.0.bias_grad: -0.5043700337409973\n",
      "  net.1.0.weight_grad: -1.7512030601501465\n",
      "  net.2.0.bias_grad: -0.4513086974620819\n",
      "  net.2.0.weight_grad: -2.2835004329681396\n",
      "  net.3.bias_grad: -7.450580596923828e-09\n",
      "  net.3.weight_grad: -5.223046173341572e-08\n",
      "  node_ip: 192.168.1.248\n",
      "  pid: 57240\n",
      "  sum_loss: -0.28490214608609676\n",
      "  time_since_restore: 87.5851182937622\n",
      "  time_this_iter_s: 0.5483489036560059\n",
      "  time_total_s: 87.5851182937622\n",
      "  timestamp: 1629671163\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 148\n",
      "  trial_id: b58c9_00000\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 6.7/7.4 GiB: ***LOW MEMORY*** less than 10% of the memory on this node is available for use. This can cause unexpected crashes. Consider reducing the memory used by your application or reducing the Ray object store size by setting `object_store_memory` when calling `ray.init`.<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/8 CPUs, 0/1 GPUs, 0.0/1.07 GiB heap, 0.0/0.54 GiB objects (0.0/1.0 accelerator_type:T1000)<br>Current best trial: b58c9_00000 with loss=-0.1583890736103058 and parameters={'trainer_class': <class 'power_cogs.examples.reinforce.reinforce_trainer.ReinforceTrainer'>, 'trainer_config': {'name': 'ReinforceTrainer', 'visualize': False, 'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'logging_config': {'mlflow': {'experiment_name': None, 'save_artifact': False}, 'wandb': {'project': None, 'api_key_file': None, 'log_config': False, 'reinit': False}, 'checkpoint_path': 'checkpoints', 'tensorboard_log_path': None, 'checkpoint_interval': 5}, 'stoppage_config': {'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'loss_threshold': -inf, 'stop': None, 'callbacks': []}, 'tune_config': {'metric': 'loss', 'mode': 'min', 'num_samples': 1, 'name': '', 'checkpoint_freq': 5, 'checkpoint_at_end': True, 'callbacks': [], 'additional_config': {}}, 'optimizer_config': {'_target_': 'torch.optim.adam.Adam', 'params': '???', 'lr': 0.002, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'scheduler_config': {'_target_': 'torch.optim.lr_scheduler.ExponentialLR', 'optimizer': '???', 'gamma': 0.9999, 'last_epoch': -1}, 'dataloader_config': {'_target_': 'torch.utils.data.dataloader.DataLoader', 'dataset': '???', 'batch_size': 1, 'shuffle': False, 'sampler': None, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None}, 'cluster_config': {'name': '', 'cluster_config_path': None, 'join_cluster': False}, 'dataset_config': {'env_name': 'LunarLander-v2', 'max_env_steps': 10000, 'gamma': 0.99, 'discount_rewards': True, 'normalize_rewards': True, 'random_policy': False, 'num_rollouts': 100, 'num_workers': 5}, 'model_config': {'input_dims': 8, 'hidden_dims': [32, 32], 'output_dims': 4, 'output_activation': None, 'use_normal_init': True, 'normal_std': 0.1, 'zero_bias': False}}}<br>Result logdir: /home/shyam/ray_results/TuneTrainer_2021-08-22_15-24-26<br>Number of trials: 1/1 (1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name             </th><th>status  </th><th>loc                </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  average_summed_rewards</th><th style=\"text-align: right;\">  max_summed_rewards</th><th style=\"text-align: right;\">     loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>TuneTrainer_b58c9_00000</td><td>RUNNING </td><td>192.168.1.248:57240</td><td style=\"text-align: right;\">   150</td><td style=\"text-align: right;\">          89.587</td><td style=\"text-align: right;\">                 25.4457</td><td style=\"text-align: right;\">             180.009</td><td style=\"text-align: right;\">-0.158389</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for TuneTrainer_b58c9_00000:\n",
      "  average_summed_rewards: -316.747152211184\n",
      "  date: 2021-08-22_15-26-09\n",
      "  done: false\n",
      "  epoch: 1\n",
      "  experiment_id: ea3e4da9a71247ddb0ad6c4bcb6d8084\n",
      "  hostname: shyam-ThinkPad-P53\n",
      "  iterations_since_restore: 152\n",
      "  loss: 0.2415801152586937\n",
      "  max_loss: 0.6546870470046997\n",
      "  max_summed_rewards: 29.791074049919047\n",
      "  net.0.0.bias_grad: -0.0745801255106926\n",
      "  net.0.0.weight_grad: 0.33335015177726746\n",
      "  net.1.0.bias_grad: -0.04605284333229065\n",
      "  net.1.0.weight_grad: -0.23019945621490479\n",
      "  net.2.0.bias_grad: -0.03844859451055527\n",
      "  net.2.0.weight_grad: -0.34263914823532104\n",
      "  net.3.bias_grad: 7.450580596923828e-09\n",
      "  net.3.weight_grad: 6.811296771047637e-08\n",
      "  node_ip: 192.168.1.248\n",
      "  pid: 57240\n",
      "  sum_loss: 1.2079005762934685\n",
      "  time_since_restore: 93.84615445137024\n",
      "  time_this_iter_s: 3.362436294555664\n",
      "  time_total_s: 93.84615445137024\n",
      "  timestamp: 1629671169\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 152\n",
      "  trial_id: b58c9_00000\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 6.7/7.4 GiB: ***LOW MEMORY*** less than 10% of the memory on this node is available for use. This can cause unexpected crashes. Consider reducing the memory used by your application or reducing the Ray object store size by setting `object_store_memory` when calling `ray.init`.<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/8 CPUs, 0/1 GPUs, 0.0/1.07 GiB heap, 0.0/0.54 GiB objects (0.0/1.0 accelerator_type:T1000)<br>Current best trial: b58c9_00000 with loss=-0.031343826092779636 and parameters={'trainer_class': <class 'power_cogs.examples.reinforce.reinforce_trainer.ReinforceTrainer'>, 'trainer_config': {'name': 'ReinforceTrainer', 'visualize': False, 'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'logging_config': {'mlflow': {'experiment_name': None, 'save_artifact': False}, 'wandb': {'project': None, 'api_key_file': None, 'log_config': False, 'reinit': False}, 'checkpoint_path': 'checkpoints', 'tensorboard_log_path': None, 'checkpoint_interval': 5}, 'stoppage_config': {'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'loss_threshold': -inf, 'stop': None, 'callbacks': []}, 'tune_config': {'metric': 'loss', 'mode': 'min', 'num_samples': 1, 'name': '', 'checkpoint_freq': 5, 'checkpoint_at_end': True, 'callbacks': [], 'additional_config': {}}, 'optimizer_config': {'_target_': 'torch.optim.adam.Adam', 'params': '???', 'lr': 0.002, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'scheduler_config': {'_target_': 'torch.optim.lr_scheduler.ExponentialLR', 'optimizer': '???', 'gamma': 0.9999, 'last_epoch': -1}, 'dataloader_config': {'_target_': 'torch.utils.data.dataloader.DataLoader', 'dataset': '???', 'batch_size': 1, 'shuffle': False, 'sampler': None, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None}, 'cluster_config': {'name': '', 'cluster_config_path': None, 'join_cluster': False}, 'dataset_config': {'env_name': 'LunarLander-v2', 'max_env_steps': 10000, 'gamma': 0.99, 'discount_rewards': True, 'normalize_rewards': True, 'random_policy': False, 'num_rollouts': 100, 'num_workers': 5}, 'model_config': {'input_dims': 8, 'hidden_dims': [32, 32], 'output_dims': 4, 'output_activation': None, 'use_normal_init': True, 'normal_std': 0.1, 'zero_bias': False}}}<br>Result logdir: /home/shyam/ray_results/TuneTrainer_2021-08-22_15-24-26<br>Number of trials: 1/1 (1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name             </th><th>status  </th><th>loc                </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  average_summed_rewards</th><th style=\"text-align: right;\">  max_summed_rewards</th><th style=\"text-align: right;\">      loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>TuneTrainer_b58c9_00000</td><td>RUNNING </td><td>192.168.1.248:57240</td><td style=\"text-align: right;\">   154</td><td style=\"text-align: right;\">         94.7106</td><td style=\"text-align: right;\">                -34.7735</td><td style=\"text-align: right;\">             19.8773</td><td style=\"text-align: right;\">-0.0313438</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for TuneTrainer_b58c9_00000:\n",
      "  average_summed_rewards: -19.025576483562034\n",
      "  date: 2021-08-22_15-26-15\n",
      "  done: false\n",
      "  epoch: 2\n",
      "  experiment_id: ea3e4da9a71247ddb0ad6c4bcb6d8084\n",
      "  hostname: shyam-ThinkPad-P53\n",
      "  iterations_since_restore: 168\n",
      "  loss: 0.02368834726512432\n",
      "  max_loss: 0.10662181675434113\n",
      "  max_summed_rewards: -3.850999624260666\n",
      "  net.0.0.bias_grad: -0.3562157154083252\n",
      "  net.0.0.weight_grad: 0.12907971441745758\n",
      "  net.1.0.bias_grad: -0.10701024532318115\n",
      "  net.1.0.weight_grad: -0.40509048104286194\n",
      "  net.2.0.bias_grad: -0.2717684507369995\n",
      "  net.2.0.weight_grad: -2.126588821411133\n",
      "  net.3.bias_grad: -3.3527612686157227e-08\n",
      "  net.3.weight_grad: -7.62993295211345e-07\n",
      "  node_ip: 192.168.1.248\n",
      "  pid: 57240\n",
      "  sum_loss: 0.1184417363256216\n",
      "  time_since_restore: 98.1438422203064\n",
      "  time_this_iter_s: 0.2768266201019287\n",
      "  time_total_s: 98.1438422203064\n",
      "  timestamp: 1629671175\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 168\n",
      "  trial_id: b58c9_00000\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 6.7/7.4 GiB: ***LOW MEMORY*** less than 10% of the memory on this node is available for use. This can cause unexpected crashes. Consider reducing the memory used by your application or reducing the Ray object store size by setting `object_store_memory` when calling `ray.init`.<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/8 CPUs, 0/1 GPUs, 0.0/1.07 GiB heap, 0.0/0.54 GiB objects (0.0/1.0 accelerator_type:T1000)<br>Current best trial: b58c9_00000 with loss=-0.07174286507070064 and parameters={'trainer_class': <class 'power_cogs.examples.reinforce.reinforce_trainer.ReinforceTrainer'>, 'trainer_config': {'name': 'ReinforceTrainer', 'visualize': False, 'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'logging_config': {'mlflow': {'experiment_name': None, 'save_artifact': False}, 'wandb': {'project': None, 'api_key_file': None, 'log_config': False, 'reinit': False}, 'checkpoint_path': 'checkpoints', 'tensorboard_log_path': None, 'checkpoint_interval': 5}, 'stoppage_config': {'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'loss_threshold': -inf, 'stop': None, 'callbacks': []}, 'tune_config': {'metric': 'loss', 'mode': 'min', 'num_samples': 1, 'name': '', 'checkpoint_freq': 5, 'checkpoint_at_end': True, 'callbacks': [], 'additional_config': {}}, 'optimizer_config': {'_target_': 'torch.optim.adam.Adam', 'params': '???', 'lr': 0.002, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'scheduler_config': {'_target_': 'torch.optim.lr_scheduler.ExponentialLR', 'optimizer': '???', 'gamma': 0.9999, 'last_epoch': -1}, 'dataloader_config': {'_target_': 'torch.utils.data.dataloader.DataLoader', 'dataset': '???', 'batch_size': 1, 'shuffle': False, 'sampler': None, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None}, 'cluster_config': {'name': '', 'cluster_config_path': None, 'join_cluster': False}, 'dataset_config': {'env_name': 'LunarLander-v2', 'max_env_steps': 10000, 'gamma': 0.99, 'discount_rewards': True, 'normalize_rewards': True, 'random_policy': False, 'num_rollouts': 100, 'num_workers': 5}, 'model_config': {'input_dims': 8, 'hidden_dims': [32, 32], 'output_dims': 4, 'output_activation': None, 'use_normal_init': True, 'normal_std': 0.1, 'zero_bias': False}}}<br>Result logdir: /home/shyam/ray_results/TuneTrainer_2021-08-22_15-24-26<br>Number of trials: 1/1 (1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name             </th><th>status  </th><th>loc                </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  average_summed_rewards</th><th style=\"text-align: right;\">  max_summed_rewards</th><th style=\"text-align: right;\">      loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>TuneTrainer_b58c9_00000</td><td>RUNNING </td><td>192.168.1.248:57240</td><td style=\"text-align: right;\">   171</td><td style=\"text-align: right;\">         99.1223</td><td style=\"text-align: right;\">                -26.2303</td><td style=\"text-align: right;\">             6.64059</td><td style=\"text-align: right;\">-0.0717429</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for TuneTrainer_b58c9_00000:\n",
      "  average_summed_rewards: -532.9827585442388\n",
      "  date: 2021-08-22_15-26-20\n",
      "  done: false\n",
      "  epoch: 0\n",
      "  experiment_id: ea3e4da9a71247ddb0ad6c4bcb6d8084\n",
      "  hostname: shyam-ThinkPad-P53\n",
      "  iterations_since_restore: 181\n",
      "  loss: -0.019374039582908155\n",
      "  max_loss: 0.1305990219116211\n",
      "  max_summed_rewards: -355.23745144116117\n",
      "  net.0.0.bias_grad: -0.4155457019805908\n",
      "  net.0.0.weight_grad: -0.4816843867301941\n",
      "  net.1.0.bias_grad: -0.3923611640930176\n",
      "  net.1.0.weight_grad: 1.8464077711105347\n",
      "  net.2.0.bias_grad: -0.013621211051940918\n",
      "  net.2.0.weight_grad: -1.4926823377609253\n",
      "  net.3.bias_grad: 1.862645149230957e-08\n",
      "  net.3.weight_grad: 7.099006325006485e-07\n",
      "  node_ip: 192.168.1.248\n",
      "  pid: 57240\n",
      "  sum_loss: -0.09687019791454077\n",
      "  time_since_restore: 102.62427616119385\n",
      "  time_this_iter_s: 0.17107725143432617\n",
      "  time_total_s: 102.62427616119385\n",
      "  timestamp: 1629671180\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 181\n",
      "  trial_id: b58c9_00000\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 6.7/7.4 GiB: ***LOW MEMORY*** less than 10% of the memory on this node is available for use. This can cause unexpected crashes. Consider reducing the memory used by your application or reducing the Ray object store size by setting `object_store_memory` when calling `ray.init`.<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/8 CPUs, 0/1 GPUs, 0.0/1.07 GiB heap, 0.0/0.54 GiB objects (0.0/1.0 accelerator_type:T1000)<br>Current best trial: b58c9_00000 with loss=-0.015460842475295066 and parameters={'trainer_class': <class 'power_cogs.examples.reinforce.reinforce_trainer.ReinforceTrainer'>, 'trainer_config': {'name': 'ReinforceTrainer', 'visualize': False, 'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'logging_config': {'mlflow': {'experiment_name': None, 'save_artifact': False}, 'wandb': {'project': None, 'api_key_file': None, 'log_config': False, 'reinit': False}, 'checkpoint_path': 'checkpoints', 'tensorboard_log_path': None, 'checkpoint_interval': 5}, 'stoppage_config': {'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'loss_threshold': -inf, 'stop': None, 'callbacks': []}, 'tune_config': {'metric': 'loss', 'mode': 'min', 'num_samples': 1, 'name': '', 'checkpoint_freq': 5, 'checkpoint_at_end': True, 'callbacks': [], 'additional_config': {}}, 'optimizer_config': {'_target_': 'torch.optim.adam.Adam', 'params': '???', 'lr': 0.002, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'scheduler_config': {'_target_': 'torch.optim.lr_scheduler.ExponentialLR', 'optimizer': '???', 'gamma': 0.9999, 'last_epoch': -1}, 'dataloader_config': {'_target_': 'torch.utils.data.dataloader.DataLoader', 'dataset': '???', 'batch_size': 1, 'shuffle': False, 'sampler': None, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None}, 'cluster_config': {'name': '', 'cluster_config_path': None, 'join_cluster': False}, 'dataset_config': {'env_name': 'LunarLander-v2', 'max_env_steps': 10000, 'gamma': 0.99, 'discount_rewards': True, 'normalize_rewards': True, 'random_policy': False, 'num_rollouts': 100, 'num_workers': 5}, 'model_config': {'input_dims': 8, 'hidden_dims': [32, 32], 'output_dims': 4, 'output_activation': None, 'use_normal_init': True, 'normal_std': 0.1, 'zero_bias': False}}}<br>Result logdir: /home/shyam/ray_results/TuneTrainer_2021-08-22_15-24-26<br>Number of trials: 1/1 (1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name             </th><th>status  </th><th>loc                </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  average_summed_rewards</th><th style=\"text-align: right;\">  max_summed_rewards</th><th style=\"text-align: right;\">      loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>TuneTrainer_b58c9_00000</td><td>RUNNING </td><td>192.168.1.248:57240</td><td style=\"text-align: right;\">   185</td><td style=\"text-align: right;\">         103.355</td><td style=\"text-align: right;\">                -546.697</td><td style=\"text-align: right;\">            -350.981</td><td style=\"text-align: right;\">-0.0154608</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for TuneTrainer_b58c9_00000:\n",
      "  average_summed_rewards: -490.3116113280157\n",
      "  date: 2021-08-22_15-26-25\n",
      "  done: false\n",
      "  epoch: 4\n",
      "  experiment_id: ea3e4da9a71247ddb0ad6c4bcb6d8084\n",
      "  hostname: shyam-ThinkPad-P53\n",
      "  iterations_since_restore: 200\n",
      "  loss: 0.0012571800500154496\n",
      "  max_loss: 0.05693366751074791\n",
      "  max_summed_rewards: -430.8120690796608\n",
      "  net.0.0.bias_grad: 0.07945168763399124\n",
      "  net.0.0.weight_grad: -0.07487381994724274\n",
      "  net.1.0.bias_grad: -0.0198203232139349\n",
      "  net.1.0.weight_grad: -1.4400324821472168\n",
      "  net.2.0.bias_grad: 0.039744287729263306\n",
      "  net.2.0.weight_grad: 1.3527132272720337\n",
      "  net.3.bias_grad: 5.587935447692871e-09\n",
      "  net.3.weight_grad: 5.3783878684043884e-08\n",
      "  node_ip: 192.168.1.248\n",
      "  pid: 57240\n",
      "  sum_loss: 0.006285900250077248\n",
      "  time_since_restore: 106.5813102722168\n",
      "  time_this_iter_s: 0.2474803924560547\n",
      "  time_total_s: 106.5813102722168\n",
      "  timestamp: 1629671185\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 200\n",
      "  trial_id: b58c9_00000\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 6.8/7.4 GiB: ***LOW MEMORY*** less than 10% of the memory on this node is available for use. This can cause unexpected crashes. Consider reducing the memory used by your application or reducing the Ray object store size by setting `object_store_memory` when calling `ray.init`.<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/8 CPUs, 0/1 GPUs, 0.0/1.07 GiB heap, 0.0/0.54 GiB objects (0.0/1.0 accelerator_type:T1000)<br>Current best trial: b58c9_00000 with loss=0.037879892019554974 and parameters={'trainer_class': <class 'power_cogs.examples.reinforce.reinforce_trainer.ReinforceTrainer'>, 'trainer_config': {'name': 'ReinforceTrainer', 'visualize': False, 'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'logging_config': {'mlflow': {'experiment_name': None, 'save_artifact': False}, 'wandb': {'project': None, 'api_key_file': None, 'log_config': False, 'reinit': False}, 'checkpoint_path': 'checkpoints', 'tensorboard_log_path': None, 'checkpoint_interval': 5}, 'stoppage_config': {'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'loss_threshold': -inf, 'stop': None, 'callbacks': []}, 'tune_config': {'metric': 'loss', 'mode': 'min', 'num_samples': 1, 'name': '', 'checkpoint_freq': 5, 'checkpoint_at_end': True, 'callbacks': [], 'additional_config': {}}, 'optimizer_config': {'_target_': 'torch.optim.adam.Adam', 'params': '???', 'lr': 0.002, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'scheduler_config': {'_target_': 'torch.optim.lr_scheduler.ExponentialLR', 'optimizer': '???', 'gamma': 0.9999, 'last_epoch': -1}, 'dataloader_config': {'_target_': 'torch.utils.data.dataloader.DataLoader', 'dataset': '???', 'batch_size': 1, 'shuffle': False, 'sampler': None, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None}, 'cluster_config': {'name': '', 'cluster_config_path': None, 'join_cluster': False}, 'dataset_config': {'env_name': 'LunarLander-v2', 'max_env_steps': 10000, 'gamma': 0.99, 'discount_rewards': True, 'normalize_rewards': True, 'random_policy': False, 'num_rollouts': 100, 'num_workers': 5}, 'model_config': {'input_dims': 8, 'hidden_dims': [32, 32], 'output_dims': 4, 'output_activation': None, 'use_normal_init': True, 'normal_std': 0.1, 'zero_bias': False}}}<br>Result logdir: /home/shyam/ray_results/TuneTrainer_2021-08-22_15-24-26<br>Number of trials: 1/1 (1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name             </th><th>status  </th><th>loc                </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  average_summed_rewards</th><th style=\"text-align: right;\">  max_summed_rewards</th><th style=\"text-align: right;\">     loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>TuneTrainer_b58c9_00000</td><td>RUNNING </td><td>192.168.1.248:57240</td><td style=\"text-align: right;\">   203</td><td style=\"text-align: right;\">         107.322</td><td style=\"text-align: right;\">                -532.789</td><td style=\"text-align: right;\">            -414.269</td><td style=\"text-align: right;\">0.0378799</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for TuneTrainer_b58c9_00000:\n",
      "  average_summed_rewards: -111.15747108092435\n",
      "  date: 2021-08-22_15-26-30\n",
      "  done: false\n",
      "  epoch: 4\n",
      "  experiment_id: ea3e4da9a71247ddb0ad6c4bcb6d8084\n",
      "  hostname: shyam-ThinkPad-P53\n",
      "  iterations_since_restore: 215\n",
      "  loss: -0.4349615551531315\n",
      "  max_loss: 0.027331020683050156\n",
      "  max_summed_rewards: -92.82670963241412\n",
      "  net.0.0.bias_grad: 0.9231418371200562\n",
      "  net.0.0.weight_grad: -1.1540212631225586\n",
      "  net.1.0.bias_grad: -0.49275749921798706\n",
      "  net.1.0.weight_grad: -1.7708582878112793\n",
      "  net.2.0.bias_grad: -1.0167486667633057\n",
      "  net.2.0.weight_grad: -4.673501491546631\n",
      "  net.3.bias_grad: 4.0978193283081055e-08\n",
      "  net.3.weight_grad: 5.830079317092896e-07\n",
      "  node_ip: 192.168.1.248\n",
      "  pid: 57240\n",
      "  sum_loss: -2.1748077757656574\n",
      "  time_since_restore: 110.91324043273926\n",
      "  time_this_iter_s: 0.24834346771240234\n",
      "  time_total_s: 110.91324043273926\n",
      "  timestamp: 1629671190\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 215\n",
      "  trial_id: b58c9_00000\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 6.8/7.4 GiB: ***LOW MEMORY*** less than 10% of the memory on this node is available for use. This can cause unexpected crashes. Consider reducing the memory used by your application or reducing the Ray object store size by setting `object_store_memory` when calling `ray.init`.<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/8 CPUs, 0/1 GPUs, 0.0/1.07 GiB heap, 0.0/0.54 GiB objects (0.0/1.0 accelerator_type:T1000)<br>Current best trial: b58c9_00000 with loss=-1.1153701219707728 and parameters={'trainer_class': <class 'power_cogs.examples.reinforce.reinforce_trainer.ReinforceTrainer'>, 'trainer_config': {'name': 'ReinforceTrainer', 'visualize': False, 'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'logging_config': {'mlflow': {'experiment_name': None, 'save_artifact': False}, 'wandb': {'project': None, 'api_key_file': None, 'log_config': False, 'reinit': False}, 'checkpoint_path': 'checkpoints', 'tensorboard_log_path': None, 'checkpoint_interval': 5}, 'stoppage_config': {'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'loss_threshold': -inf, 'stop': None, 'callbacks': []}, 'tune_config': {'metric': 'loss', 'mode': 'min', 'num_samples': 1, 'name': '', 'checkpoint_freq': 5, 'checkpoint_at_end': True, 'callbacks': [], 'additional_config': {}}, 'optimizer_config': {'_target_': 'torch.optim.adam.Adam', 'params': '???', 'lr': 0.002, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'scheduler_config': {'_target_': 'torch.optim.lr_scheduler.ExponentialLR', 'optimizer': '???', 'gamma': 0.9999, 'last_epoch': -1}, 'dataloader_config': {'_target_': 'torch.utils.data.dataloader.DataLoader', 'dataset': '???', 'batch_size': 1, 'shuffle': False, 'sampler': None, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None}, 'cluster_config': {'name': '', 'cluster_config_path': None, 'join_cluster': False}, 'dataset_config': {'env_name': 'LunarLander-v2', 'max_env_steps': 10000, 'gamma': 0.99, 'discount_rewards': True, 'normalize_rewards': True, 'random_policy': False, 'num_rollouts': 100, 'num_workers': 5}, 'model_config': {'input_dims': 8, 'hidden_dims': [32, 32], 'output_dims': 4, 'output_activation': None, 'use_normal_init': True, 'normal_std': 0.1, 'zero_bias': False}}}<br>Result logdir: /home/shyam/ray_results/TuneTrainer_2021-08-22_15-24-26<br>Number of trials: 1/1 (1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name             </th><th>status  </th><th>loc                </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  average_summed_rewards</th><th style=\"text-align: right;\">  max_summed_rewards</th><th style=\"text-align: right;\">    loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>TuneTrainer_b58c9_00000</td><td>RUNNING </td><td>192.168.1.248:57240</td><td style=\"text-align: right;\">   218</td><td style=\"text-align: right;\">         111.718</td><td style=\"text-align: right;\">                -85.4603</td><td style=\"text-align: right;\">            -23.9525</td><td style=\"text-align: right;\">-1.11537</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for TuneTrainer_b58c9_00000:\n",
      "  average_summed_rewards: -160.89230637140355\n",
      "  date: 2021-08-22_15-26-35\n",
      "  done: false\n",
      "  epoch: 2\n",
      "  experiment_id: ea3e4da9a71247ddb0ad6c4bcb6d8084\n",
      "  hostname: shyam-ThinkPad-P53\n",
      "  iterations_since_restore: 233\n",
      "  loss: 0.0798736265860498\n",
      "  max_loss: 0.16760839521884918\n",
      "  max_summed_rewards: -15.851657195379829\n",
      "  net.0.0.bias_grad: -1.3328814506530762\n",
      "  net.0.0.weight_grad: 1.1208488941192627\n",
      "  net.1.0.bias_grad: -0.33264949917793274\n",
      "  net.1.0.weight_grad: -1.7026145458221436\n",
      "  net.2.0.bias_grad: -0.10360058397054672\n",
      "  net.2.0.weight_grad: -0.9071844816207886\n",
      "  net.3.bias_grad: 7.450580596923828e-09\n",
      "  net.3.weight_grad: 8.860297384671867e-08\n",
      "  node_ip: 192.168.1.248\n",
      "  pid: 57240\n",
      "  sum_loss: 0.399368132930249\n",
      "  time_since_restore: 115.07994627952576\n",
      "  time_this_iter_s: 0.23847246170043945\n",
      "  time_total_s: 115.07994627952576\n",
      "  timestamp: 1629671195\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 233\n",
      "  trial_id: b58c9_00000\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 6.8/7.4 GiB: ***LOW MEMORY*** less than 10% of the memory on this node is available for use. This can cause unexpected crashes. Consider reducing the memory used by your application or reducing the Ray object store size by setting `object_store_memory` when calling `ray.init`.<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/8 CPUs, 0/1 GPUs, 0.0/1.07 GiB heap, 0.0/0.54 GiB objects (0.0/1.0 accelerator_type:T1000)<br>Current best trial: b58c9_00000 with loss=0.13986790850758551 and parameters={'trainer_class': <class 'power_cogs.examples.reinforce.reinforce_trainer.ReinforceTrainer'>, 'trainer_config': {'name': 'ReinforceTrainer', 'visualize': False, 'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'logging_config': {'mlflow': {'experiment_name': None, 'save_artifact': False}, 'wandb': {'project': None, 'api_key_file': None, 'log_config': False, 'reinit': False}, 'checkpoint_path': 'checkpoints', 'tensorboard_log_path': None, 'checkpoint_interval': 5}, 'stoppage_config': {'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'loss_threshold': -inf, 'stop': None, 'callbacks': []}, 'tune_config': {'metric': 'loss', 'mode': 'min', 'num_samples': 1, 'name': '', 'checkpoint_freq': 5, 'checkpoint_at_end': True, 'callbacks': [], 'additional_config': {}}, 'optimizer_config': {'_target_': 'torch.optim.adam.Adam', 'params': '???', 'lr': 0.002, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'scheduler_config': {'_target_': 'torch.optim.lr_scheduler.ExponentialLR', 'optimizer': '???', 'gamma': 0.9999, 'last_epoch': -1}, 'dataloader_config': {'_target_': 'torch.utils.data.dataloader.DataLoader', 'dataset': '???', 'batch_size': 1, 'shuffle': False, 'sampler': None, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None}, 'cluster_config': {'name': '', 'cluster_config_path': None, 'join_cluster': False}, 'dataset_config': {'env_name': 'LunarLander-v2', 'max_env_steps': 10000, 'gamma': 0.99, 'discount_rewards': True, 'normalize_rewards': True, 'random_policy': False, 'num_rollouts': 100, 'num_workers': 5}, 'model_config': {'input_dims': 8, 'hidden_dims': [32, 32], 'output_dims': 4, 'output_activation': None, 'use_normal_init': True, 'normal_std': 0.1, 'zero_bias': False}}}<br>Result logdir: /home/shyam/ray_results/TuneTrainer_2021-08-22_15-24-26<br>Number of trials: 1/1 (1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name             </th><th>status  </th><th>loc                </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  average_summed_rewards</th><th style=\"text-align: right;\">  max_summed_rewards</th><th style=\"text-align: right;\">    loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>TuneTrainer_b58c9_00000</td><td>RUNNING </td><td>192.168.1.248:57240</td><td style=\"text-align: right;\">   236</td><td style=\"text-align: right;\">         115.805</td><td style=\"text-align: right;\">                 -213.55</td><td style=\"text-align: right;\">              -47.98</td><td style=\"text-align: right;\">0.139868</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for TuneTrainer_b58c9_00000:\n",
      "  average_summed_rewards: -179.45643455427077\n",
      "  date: 2021-08-22_15-26-41\n",
      "  done: false\n",
      "  epoch: 4\n",
      "  experiment_id: ea3e4da9a71247ddb0ad6c4bcb6d8084\n",
      "  hostname: shyam-ThinkPad-P53\n",
      "  iterations_since_restore: 250\n",
      "  loss: 0.06284772865474224\n",
      "  max_loss: 0.2565726935863495\n",
      "  max_summed_rewards: -2.587687775453645\n",
      "  net.0.0.bias_grad: -3.7148351669311523\n",
      "  net.0.0.weight_grad: 1.2450095415115356\n",
      "  net.1.0.bias_grad: -0.3055557906627655\n",
      "  net.1.0.weight_grad: -2.091036558151245\n",
      "  net.2.0.bias_grad: -0.11779462546110153\n",
      "  net.2.0.weight_grad: -1.355387806892395\n",
      "  net.3.bias_grad: -7.450580596923828e-09\n",
      "  net.3.weight_grad: -2.7531132218427956e-07\n",
      "  node_ip: 192.168.1.248\n",
      "  pid: 57240\n",
      "  sum_loss: 0.3142386432737112\n",
      "  time_since_restore: 119.1708447933197\n",
      "  time_this_iter_s: 0.24797344207763672\n",
      "  time_total_s: 119.1708447933197\n",
      "  timestamp: 1629671201\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 250\n",
      "  trial_id: b58c9_00000\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 6.8/7.4 GiB: ***LOW MEMORY*** less than 10% of the memory on this node is available for use. This can cause unexpected crashes. Consider reducing the memory used by your application or reducing the Ray object store size by setting `object_store_memory` when calling `ray.init`.<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/8 CPUs, 0/1 GPUs, 0.0/1.07 GiB heap, 0.0/0.54 GiB objects (0.0/1.0 accelerator_type:T1000)<br>Current best trial: b58c9_00000 with loss=0.14884055331349372 and parameters={'trainer_class': <class 'power_cogs.examples.reinforce.reinforce_trainer.ReinforceTrainer'>, 'trainer_config': {'name': 'ReinforceTrainer', 'visualize': False, 'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'logging_config': {'mlflow': {'experiment_name': None, 'save_artifact': False}, 'wandb': {'project': None, 'api_key_file': None, 'log_config': False, 'reinit': False}, 'checkpoint_path': 'checkpoints', 'tensorboard_log_path': None, 'checkpoint_interval': 5}, 'stoppage_config': {'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'loss_threshold': -inf, 'stop': None, 'callbacks': []}, 'tune_config': {'metric': 'loss', 'mode': 'min', 'num_samples': 1, 'name': '', 'checkpoint_freq': 5, 'checkpoint_at_end': True, 'callbacks': [], 'additional_config': {}}, 'optimizer_config': {'_target_': 'torch.optim.adam.Adam', 'params': '???', 'lr': 0.002, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'scheduler_config': {'_target_': 'torch.optim.lr_scheduler.ExponentialLR', 'optimizer': '???', 'gamma': 0.9999, 'last_epoch': -1}, 'dataloader_config': {'_target_': 'torch.utils.data.dataloader.DataLoader', 'dataset': '???', 'batch_size': 1, 'shuffle': False, 'sampler': None, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None}, 'cluster_config': {'name': '', 'cluster_config_path': None, 'join_cluster': False}, 'dataset_config': {'env_name': 'LunarLander-v2', 'max_env_steps': 10000, 'gamma': 0.99, 'discount_rewards': True, 'normalize_rewards': True, 'random_policy': False, 'num_rollouts': 100, 'num_workers': 5}, 'model_config': {'input_dims': 8, 'hidden_dims': [32, 32], 'output_dims': 4, 'output_activation': None, 'use_normal_init': True, 'normal_std': 0.1, 'zero_bias': False}}}<br>Result logdir: /home/shyam/ray_results/TuneTrainer_2021-08-22_15-24-26<br>Number of trials: 1/1 (1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name             </th><th>status  </th><th>loc                </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  average_summed_rewards</th><th style=\"text-align: right;\">  max_summed_rewards</th><th style=\"text-align: right;\">    loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>TuneTrainer_b58c9_00000</td><td>RUNNING </td><td>192.168.1.248:57240</td><td style=\"text-align: right;\">   253</td><td style=\"text-align: right;\">          119.81</td><td style=\"text-align: right;\">                -164.023</td><td style=\"text-align: right;\">            -102.617</td><td style=\"text-align: right;\">0.148841</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-22 15:26:43,019\tWARNING tune.py:506 -- SIGINT received (e.g. via Ctrl+C), ending Ray Tune run. This will try to checkpoint the experiment state one last time. Press CTRL+C one more time (or send SIGINT/SIGKILL/SIGTERM) to skip. \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 6.9/7.4 GiB: ***LOW MEMORY*** less than 10% of the memory on this node is available for use. This can cause unexpected crashes. Consider reducing the memory used by your application or reducing the Ray object store size by setting `object_store_memory` when calling `ray.init`.<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/8 CPUs, 0/1 GPUs, 0.0/1.07 GiB heap, 0.0/0.54 GiB objects (0.0/1.0 accelerator_type:T1000)<br>Current best trial: b58c9_00000 with loss=-0.0039087474346160885 and parameters={'trainer_class': <class 'power_cogs.examples.reinforce.reinforce_trainer.ReinforceTrainer'>, 'trainer_config': {'name': 'ReinforceTrainer', 'visualize': False, 'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'logging_config': {'mlflow': {'experiment_name': None, 'save_artifact': False}, 'wandb': {'project': None, 'api_key_file': None, 'log_config': False, 'reinit': False}, 'checkpoint_path': 'checkpoints', 'tensorboard_log_path': None, 'checkpoint_interval': 5}, 'stoppage_config': {'batch_size': 32, 'epochs': 1000, 'early_stoppage': False, 'loss_threshold': -inf, 'stop': None, 'callbacks': []}, 'tune_config': {'metric': 'loss', 'mode': 'min', 'num_samples': 1, 'name': '', 'checkpoint_freq': 5, 'checkpoint_at_end': True, 'callbacks': [], 'additional_config': {}}, 'optimizer_config': {'_target_': 'torch.optim.adam.Adam', 'params': '???', 'lr': 0.002, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'scheduler_config': {'_target_': 'torch.optim.lr_scheduler.ExponentialLR', 'optimizer': '???', 'gamma': 0.9999, 'last_epoch': -1}, 'dataloader_config': {'_target_': 'torch.utils.data.dataloader.DataLoader', 'dataset': '???', 'batch_size': 1, 'shuffle': False, 'sampler': None, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None}, 'cluster_config': {'name': '', 'cluster_config_path': None, 'join_cluster': False}, 'dataset_config': {'env_name': 'LunarLander-v2', 'max_env_steps': 10000, 'gamma': 0.99, 'discount_rewards': True, 'normalize_rewards': True, 'random_policy': False, 'num_rollouts': 100, 'num_workers': 5}, 'model_config': {'input_dims': 8, 'hidden_dims': [32, 32], 'output_dims': 4, 'output_activation': None, 'use_normal_init': True, 'normal_std': 0.1, 'zero_bias': False}}}<br>Result logdir: /home/shyam/ray_results/TuneTrainer_2021-08-22_15-24-26<br>Number of trials: 1/1 (1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name             </th><th>status  </th><th>loc                </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  average_summed_rewards</th><th style=\"text-align: right;\">  max_summed_rewards</th><th style=\"text-align: right;\">       loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>TuneTrainer_b58c9_00000</td><td>RUNNING </td><td>192.168.1.248:57240</td><td style=\"text-align: right;\">   257</td><td style=\"text-align: right;\">         120.701</td><td style=\"text-align: right;\">                -193.017</td><td style=\"text-align: right;\">            -58.8936</td><td style=\"text-align: right;\">-0.00390875</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m 2021-08-22 15:26:43,088\tERROR worker.py:382 -- SystemExit was raised from the worker\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m Traceback (most recent call last):\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m   File \"python/ray/_raylet.pyx\", line 599, in ray._raylet.task_execution_handler\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m   File \"python/ray/_raylet.pyx\", line 451, in ray._raylet.execute_task\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m   File \"python/ray/_raylet.pyx\", line 488, in ray._raylet.execute_task\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m   File \"python/ray/_raylet.pyx\", line 495, in ray._raylet.execute_task\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m   File \"python/ray/_raylet.pyx\", line 505, in ray._raylet.execute_task\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m   File \"python/ray/_raylet.pyx\", line 449, in ray._raylet.execute_task.function_executor\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m   File \"/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/ray/_private/function_manager.py\", line 556, in actor_method_executor\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m     return method(__ray_actor, *args, **kwargs)\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m   File \"/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/ray/tune/trainable.py\", line 173, in train_buffered\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m     result = self.train()\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m   File \"/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/ray/tune/trainable.py\", line 232, in train\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m     result = self.step()\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m   File \"/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/wrappers/tune_wrapper.py\", line 154, in step\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m     out = self.trainer.train_iter(\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m   File \"/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/callbacks/callback.py\", line 122, in _inner\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m     return wrapped(*args, **kwargs)\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m   File \"/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/callbacks/callback.py\", line 91, in _inner\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m     out = f(*args, **kwargs)\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m   File \"/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/examples/reinforce/reinforce_trainer.py\", line 57, in train_iter\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m     rollouts = self.dataset.get_rollouts(self.cluster, self.model)\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m   File \"/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/examples/reinforce/rollout_dataset.py\", line 77, in get_rollouts\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m     return cluster.execute_list(rollout_workers, rollout_args)\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m   File \"/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/cluster/cluster.py\", line 63, in execute_list\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m     return [\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m   File \"/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/cluster/cluster.py\", line 64, in <listcomp>\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m     self.execute(execution, **execution_arg)\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m   File \"/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/cluster/local_cluster.py\", line 20, in execute\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m     return [execution.execute(*args, **kwargs)]\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m   File \"/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/cluster/execution/rollout_execution.py\", line 143, in execute\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m     return self.rollout(*args, **kwargs)\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m   File \"/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/cluster/execution/rollout_execution.py\", line 101, in rollout\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m     observation, reward, done, info = self.env.step(action)\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m   File \"/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/gym/wrappers/time_limit.py\", line 16, in step\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m     observation, reward, done, info = self.env.step(action)\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m   File \"/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/gym/envs/box2d/lunar_lander.py\", line 269, in step\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m     p = self._create_particle(3.5,  # 3.5 is here to make particle speed adequate\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m   File \"/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/gym/envs/box2d/lunar_lander.py\", line 229, in _create_particle\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m     fixtures = fixtureDef(\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m   File \"/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/Box2D/Box2D.py\", line 2669, in __init__\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m     _init_kwargs(self, **kwargs)\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m   File \"/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/Box2D/Box2D.py\", line 111, in _init_kwargs\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m     setattr(self, key, value)\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m   File \"/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/Box2D/Box2D.py\", line 2696, in __SetCategoryBits\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m     self.filter.categoryBits=value\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m   File \"/home/shyam/anaconda3/envs/py38/lib/python3.8/site-packages/ray/worker.py\", line 379, in sigterm_handler\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m     sys.exit(1)\n",
      "\u001b[2m\u001b[36m(pid=57240)\u001b[0m SystemExit: 1\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-abc03024d3b1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtune\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/envs/py38/lib/python3.8/site-packages/power_cogs-0.1.0-py3.8.egg/power_cogs/wrappers/tune_wrapper.py\u001b[0m in \u001b[0;36mtune\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    219\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;34m\"stop\"\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mstoppage_config\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    220\u001b[0m             \u001b[0mtune_config\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"stop\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcreate_stopper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstoppage_config\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 221\u001b[0;31m         return tune.run(\n\u001b[0m\u001b[1;32m    222\u001b[0m             \u001b[0mTuneTrainer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    223\u001b[0m             config={\n",
      "\u001b[0;32m~/anaconda3/envs/py38/lib/python3.8/site-packages/ray/tune/tune.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(run_or_experiment, name, metric, mode, stop, time_budget_s, config, resources_per_trial, num_samples, local_dir, search_alg, scheduler, keep_checkpoints_num, checkpoint_score_attr, checkpoint_freq, checkpoint_at_end, verbose, progress_reporter, log_to_file, trial_name_creator, trial_dirname_creator, sync_config, export_formats, max_failures, fail_fast, restore, server_port, resume, queue_trials, reuse_actors, trial_executor, raise_on_failed_trial, callbacks, loggers, ray_auto_init, run_errored_only, global_checkpoint_period, with_server, upload_dir, sync_to_cloud, sync_to_driver, sync_on_checkpoint, _remote)\u001b[0m\n\u001b[1;32m    532\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    533\u001b[0m     \u001b[0mwait_for_sync\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 534\u001b[0;31m     \u001b[0mrunner\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcleanup_trials\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    535\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    536\u001b[0m     \u001b[0mincomplete_trials\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/py38/lib/python3.8/site-packages/ray/tune/trial_runner.py\u001b[0m in \u001b[0;36mcleanup_trials\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1145\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1146\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mcleanup_trials\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1147\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrial_executor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcleanup\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1148\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1149\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__getstate__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/py38/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py\u001b[0m in \u001b[0;36mcleanup\u001b[0;34m(self, trial_runner)\u001b[0m\n\u001b[1;32m   1010\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pg_manager\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreconcile_placement_groups\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial_runner\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_trials\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1011\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pg_manager\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcleanup\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mforce\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1012\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pg_manager\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcleanup_existing_pg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mblock\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1013\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1014\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mcontextmanager\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/py38/lib/python3.8/site-packages/ray/tune/utils/placement_groups.py\u001b[0m in \u001b[0;36mcleanup_existing_pg\u001b[0;34m(self, block)\u001b[0m\n\u001b[1;32m    315\u001b[0m                     \u001b[0mpg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_placement_group\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"name\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    316\u001b[0m                     \u001b[0mremove_placement_group\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 317\u001b[0;31m                 \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0.1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    318\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    319\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mstage_trial_pg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrial\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m\"Trial\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "wrapper.tune()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:py38] *",
   "language": "python",
   "name": "conda-env-py38-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
