notebooks/Robustness.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('../')\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.ticker import (MultipleLocator, FormatStrFormatter,\n",
    "                               AutoMinorLocator, LinearLocator)\n",
    "from matplotlib.ticker import ScalarFormatter\n",
    "import matplotlib.ticker as mtick\n",
    "from collections import OrderedDict\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "# import dill\n",
    "import argparse\n",
    "import pyrep.backend.sim as sim\n",
    "from networks.structures import PolicyNetwork, ValueNetwork, SoftQNetwork\n",
    "import torch\n",
    "\n",
    "from sim_framework.envs.drone_env import DroneEnv\n",
    "import time\n",
    "import itertools\n",
    "import random\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def custom_reset(env, variant):\n",
    "#     env.pr.stop()\n",
    "    env.current_action = np.array([0,0,0,0])\n",
    "    \n",
    "#     env.agent.set_position(np.round([0,0,1.7],2).tolist())\n",
    "#     env.agent.set_orientation(np.round([0,0,0],2).tolist())\n",
    "\n",
    "    env.agent.set_orientation(np.round(variant[3:],2).tolist())\n",
    "    env.agent.set_position(np.round(variant[:3],2).tolist())\n",
    "\n",
    "    env.agent.set_thrust_and_torque(np.asarray([0.] * 4), force_zero=True)\n",
    "    env.agent.set_joint_positions(env.initial_joint_positions)\n",
    "    env.agent.set_joint_target_velocities(env.initial_joint_velocities)\n",
    "    env.agent.set_joint_target_positions(env.initial_joint_target_positions)\n",
    "    \n",
    "#     env.agent.set_orientation(np.round([0,0,0],2).tolist())\n",
    "   \n",
    "\n",
    "    env.first_obs=True\n",
    "    env._make_observation()\n",
    "    env.last_state = env.observation[:18]    \n",
    "#     env.pr.start()\n",
    "    \n",
    "    return env.observation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def state_to_tensor(state, device):\n",
    "    \"\"\"Transform numpy array to torch tensor\"\"\"\n",
    "    if args.use_double:\n",
    "        return torch.DoubleTensor(state).unsqueeze(0).to(device)\n",
    "    else:\n",
    "        return torch.FloatTensor(state).unsqueeze(0).to(device)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def rollouts(\n",
    "        variant,\n",
    "        env,\n",
    "        policy,\n",
    "        action_range,\n",
    "        device,\n",
    "        max_timesteps=1000,\n",
    "        time_horizon=250):\n",
    "    \"\"\"\n",
    "    Perform policy rollouts until a max given number of steps\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    env :\n",
    "        A larocs_sim environment\n",
    "    policy :\n",
    "        An actor-policy for the agent act in the environment\n",
    "    action_range : list\n",
    "        Range of possible float values for the action\n",
    "    max_timesteps : int, optional\n",
    "        Number of timesteps to perform while interacting with the environment, by default 1000\n",
    "    time_horizon : int, optional\n",
    "        The number of steps for each episode, by default 250\n",
    "\n",
    "    \"\"\"\n",
    "    count = 0\n",
    "    dones = False\n",
    "    set_of_obs, set_of_next_obs, set_of_rewards, set_of_actions, set_of_dones, set_of_infos = [], [], [], [], [], []\n",
    "\n",
    "    rollout = -1\n",
    "\n",
    "    mb_obs, mb_next_obs, mb_rewards, mb_actions, mb_dones, mb_infos = [], [], [], [], [], []\n",
    "    rollout += 1\n",
    "\n",
    "#         obs0 = env.reset()\n",
    "    obs0 = custom_reset(env,variant)\n",
    "\n",
    "\n",
    "    for j in range(time_horizon):\n",
    "        dones = False\n",
    "        try:\n",
    "            actions, agent_info = policy.deterministic_action(\n",
    "                state_to_tensor(obs0, device))\n",
    "        except:\n",
    "            actions = policy.deterministic_action(\n",
    "                state_to_tensor(obs0, device))\n",
    "\n",
    "        # Take actions in env and look the results\n",
    "        obs1, rewards, dones, infos = env.step(actions * action_range[1])\n",
    "        # Append on the experience buffers\n",
    "        mb_obs.append(obs0)\n",
    "        # mb_obs.append(obs0)\n",
    "        mb_next_obs.append(obs1)\n",
    "        mb_actions.append(actions)\n",
    "        mb_dones.append(dones)\n",
    "        mb_rewards.append(rewards)\n",
    "        mb_infos.append(infos)\n",
    "\n",
    "        count += 1\n",
    "        if dones:\n",
    "            break\n",
    "\n",
    "        obs0 = obs1\n",
    "#         print()\n",
    "#         print('rewards: mean = {0}'.format(np.mean(mb_rewards)))\n",
    "    print('rewards: sum = {0}'.format(np.sum(mb_rewards)))\n",
    "\n",
    "    set_of_obs.append(mb_obs)\n",
    "    set_of_next_obs.append(mb_next_obs)\n",
    "    set_of_rewards.append(mb_rewards)\n",
    "    set_of_actions.append(mb_actions)\n",
    "    set_of_dones.append(mb_dones)\n",
    "    set_of_infos.append(mb_infos)\n",
    "\n",
    "    set_tau = {'obs': set_of_obs,\n",
    "                           'next_obs': set_of_next_obs,\n",
    "                           'rewards': set_of_rewards,\n",
    "                           'actions': set_of_actions,\n",
    "                           'dones': set_of_dones,\n",
    "                           'infos': set_of_infos}\n",
    "    return set_tau"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Args():\n",
    "    def __init__(self):\n",
    "        pass\n",
    "\n",
    "\n",
    "args = Args()\n",
    "\n",
    "args.H = 250\n",
    "\n",
    "args.max_timesteps=250\n",
    "\n",
    "\n",
    "env_reset_mode = \"Discretized_Uniform\"\n",
    "seed = 42\n",
    "headless = True\n",
    "# headless = False\n",
    "\n",
    "state='New_action'\n",
    "reward='Normal'\n",
    "try:\n",
    "    env.shutdown()\n",
    "except:\n",
    "    pass\n",
    "env = DroneEnv(random=env_reset_mode,seed=seed, headless = headless, state=state,\\\n",
    "               reward_function_name=reward)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Permutation List"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of different variants\n",
      "216\n"
     ]
    }
   ],
   "source": [
    "## Ticks for Discretized_Uniform initialization setup\n",
    "xy_ticks = env.x_y_ticks\n",
    "z_ticks = env.z_ticks\n",
    "ang_ticks = env.ang_ticks\n",
    "\n",
    "extreme_angles = ang_ticks[[0,-1]]\n",
    "extreme_angles = np.append(extreme_angles, 0)\n",
    "extreme_xyticks = xy_ticks[[0,-1]]\n",
    "extreme_zticks = z_ticks[[0,-1]]\n",
    "\n",
    "\n",
    "\n",
    "all_list = [extreme_xyticks,extreme_xyticks, np.round( extreme_zticks,2), extreme_angles,extreme_angles,extreme_angles]\n",
    "\n",
    "\n",
    "res = list(itertools.product(*all_list)) \n",
    "random.shuffle(res)\n",
    "print('Number of different variants')\n",
    "print(len(res))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Running"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Loading\n"
     ]
    }
   ],
   "source": [
    "args.use_double=False\n",
    "args.use_cuda=False\n",
    "\n",
    "use_cuda = torch.cuda.is_available()\n",
    "\n",
    "if use_cuda and (args.use_cuda):\n",
    "    device = torch.device(\"cuda\")\n",
    "else:\n",
    "    device = torch.device(\"cpu\")\n",
    "\n",
    "\n",
    "restore_path =  '../saved_policies/sac_optimal_policy_2.pt'\n",
    "\n",
    "try:\n",
    "    checkpoint = torch.load(restore_path, map_location='cpu')\n",
    "except BaseException:\n",
    "    checkpoint = torch.load(restore_path, map_location=torch.device('cpu'))\n",
    "print('Finished Loading')\n",
    "\n",
    "# Neural network parameters\n",
    "try:\n",
    "    state_dim = env.observation_space.shape[0]\n",
    "except BaseException:\n",
    "    state_dim = env.observation_space\n",
    "action_dim = env.action_space.shape[0]\n",
    "hidden_dim = checkpoint['linear1.weight'].data.shape[0]\n",
    "action_range = [env.agent.action_space.low.min(\n",
    "), env.agent.action_space.high.max()]\n",
    "size_obs = checkpoint['linear1.weight'].data.shape[1]\n",
    "\n",
    "assert size_obs == state_dim, 'Checkpoint state must be the same as the env'\n",
    "\n",
    "\n",
    "policy_net = PolicyNetwork(state_dim, action_dim, hidden_dim).to(device)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Loading the weights\n"
     ]
    }
   ],
   "source": [
    "# Loading  Models\n",
    "policy_net.load_state_dict(checkpoint)\n",
    "print('Finished Loading the weights')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "rewards: sum = 877.1200845534663\n",
      "1\n",
      "rewards: sum = 869.9280922522692\n",
      "2\n",
      "rewards: sum = 891.6576752246817\n",
      "3\n",
      "rewards: sum = 883.0101529152173\n",
      "4\n",
      "rewards: sum = 896.3264992912492\n",
      "5\n",
      "rewards: sum = 874.0152384834564\n",
      "6\n",
      "rewards: sum = 900.5373236322093\n",
      "7\n",
      "rewards: sum = 899.9096966483784\n",
      "8\n",
      "rewards: sum = 894.9760864179252\n",
      "9\n",
      "rewards: sum = 886.492229917679\n",
      "10\n",
      "rewards: sum = 901.0729021115104\n",
      "11\n",
      "rewards: sum = 887.2421559703803\n",
      "12\n",
      "rewards: sum = 889.0111172262613\n",
      "13\n",
      "rewards: sum = 860.6523525027959\n",
      "14\n",
      "rewards: sum = 862.3917533988092\n",
      "15\n",
      "rewards: sum = 888.6664420551124\n",
      "16\n",
      "rewards: sum = 904.3786044462072\n",
      "17\n",
      "rewards: sum = 897.1422578799277\n",
      "18\n",
      "rewards: sum = 904.8975850526614\n",
      "19\n",
      "rewards: sum = 878.4370341487543\n",
      "20\n",
      "rewards: sum = 896.7519630918687\n",
      "21\n",
      "rewards: sum = 871.8436541283222\n",
      "22\n",
      "rewards: sum = 878.558593293236\n",
      "23\n",
      "rewards: sum = 904.699504927172\n",
      "24\n",
      "rewards: sum = 880.1194983128975\n",
      "25\n",
      "rewards: sum = 878.523675280956\n",
      "26\n",
      "rewards: sum = 882.9461452790712\n",
      "27\n",
      "rewards: sum = 863.4715386745926\n",
      "28\n",
      "rewards: sum = 899.7397506765387\n",
      "29\n",
      "rewards: sum = 875.8037820009756\n",
      "30\n",
      "rewards: sum = 898.0817699817233\n",
      "31\n",
      "rewards: sum = 885.0298235842963\n",
      "32\n",
      "rewards: sum = 860.603878460849\n",
      "33\n",
      "rewards: sum = 899.1610794915347\n",
      "34\n",
      "rewards: sum = 907.2296450942133\n",
      "35\n",
      "rewards: sum = 884.4922844950675\n",
      "36\n",
      "rewards: sum = 887.118100912087\n",
      "37\n",
      "rewards: sum = 899.6497850689009\n",
      "38\n",
      "rewards: sum = 874.040763531865\n",
      "39\n",
      "rewards: sum = 877.4780777957269\n",
      "40\n",
      "rewards: sum = 865.3159603220156\n",
      "41\n",
      "rewards: sum = 890.8162485466335\n",
      "42\n",
      "rewards: sum = 894.398685175023\n",
      "43\n",
      "rewards: sum = 896.7292923725229\n",
      "44\n",
      "rewards: sum = 874.3684217745333\n",
      "45\n",
      "rewards: sum = 890.0497513913326\n",
      "46\n",
      "rewards: sum = 888.3588113783471\n",
      "47\n",
      "rewards: sum = 897.3101298101333\n",
      "48\n",
      "rewards: sum = 903.720273688921\n",
      "49\n",
      "rewards: sum = 878.1758214536835\n",
      "50\n",
      "rewards: sum = 891.54218835386\n",
      "51\n",
      "rewards: sum = 887.3333606750956\n",
      "52\n",
      "rewards: sum = 896.4733332860587\n",
      "53\n",
      "rewards: sum = 896.2428940994746\n",
      "54\n",
      "rewards: sum = 872.0980191229746\n",
      "55\n",
      "rewards: sum = 895.2377687164615\n",
      "56\n",
      "rewards: sum = 900.2512609809758\n",
      "57\n",
      "rewards: sum = 883.9459429163135\n",
      "58\n",
      "rewards: sum = 881.1159313309353\n",
      "59\n",
      "rewards: sum = 905.6991006539256\n",
      "60\n",
      "rewards: sum = 892.1217530806359\n",
      "61\n",
      "rewards: sum = 897.8970661355372\n",
      "62\n",
      "rewards: sum = 898.0981787532521\n",
      "63\n",
      "rewards: sum = 880.1850117657287\n",
      "64\n",
      "rewards: sum = 877.8151583108748\n",
      "65\n",
      "rewards: sum = 888.6726378395499\n",
      "66\n",
      "rewards: sum = 889.7889131081004\n",
      "67\n",
      "rewards: sum = 875.4889509668368\n",
      "68\n",
      "rewards: sum = 874.8845105642113\n",
      "69\n",
      "rewards: sum = 908.5999890390235\n",
      "70\n",
      "rewards: sum = 880.1802331306792\n",
      "71\n",
      "rewards: sum = 864.0250776596363\n",
      "72\n",
      "rewards: sum = 882.3160873384968\n",
      "73\n",
      "rewards: sum = 900.0099752908798\n",
      "74\n",
      "rewards: sum = 859.1669384718923\n",
      "75\n",
      "rewards: sum = 870.4063612971073\n",
      "76\n",
      "rewards: sum = 892.0235208870704\n",
      "77\n",
      "rewards: sum = 864.4731059839928\n",
      "78\n",
      "rewards: sum = 892.9096124655371\n",
      "79\n",
      "rewards: sum = 898.7552933317618\n",
      "80\n",
      "rewards: sum = 879.8477111625366\n",
      "81\n",
      "rewards: sum = 874.9483665867407\n",
      "82\n",
      "rewards: sum = 890.1741272239042\n",
      "83\n",
      "rewards: sum = 888.5680966989888\n",
      "84\n",
      "rewards: sum = 887.394871781398\n",
      "85\n",
      "rewards: sum = 869.6367722934792\n",
      "86\n",
      "rewards: sum = 907.0537672727554\n",
      "87\n",
      "rewards: sum = 903.4620615204284\n",
      "88\n",
      "rewards: sum = 895.4471779789939\n",
      "89\n",
      "rewards: sum = 905.2972526552121\n",
      "90\n",
      "rewards: sum = 888.7805656164828\n",
      "91\n",
      "rewards: sum = 889.2913439626157\n",
      "92\n",
      "rewards: sum = 889.3228466167005\n",
      "93\n",
      "rewards: sum = 891.6299172011553\n",
      "94\n",
      "rewards: sum = 877.0072429058387\n",
      "95\n",
      "rewards: sum = 886.1487450326917\n",
      "96\n",
      "rewards: sum = 904.2764247031805\n",
      "97\n",
      "rewards: sum = 867.9149661258944\n",
      "98\n",
      "rewards: sum = 861.3714723129583\n",
      "99\n",
      "rewards: sum = 886.2550257673836\n",
      "100\n",
      "rewards: sum = 905.1098989330309\n",
      "101\n",
      "rewards: sum = 868.7850588059113\n",
      "102\n",
      "rewards: sum = 896.1657750931473\n",
      "103\n",
      "rewards: sum = 901.9382654078967\n",
      "104\n",
      "rewards: sum = 905.9356892658475\n",
      "105\n",
      "rewards: sum = 892.8438508915797\n",
      "106\n",
      "rewards: sum = 865.3958650772488\n",
      "107\n",
      "rewards: sum = 880.1571674100767\n",
      "108\n",
      "rewards: sum = 887.6338052695636\n",
      "109\n",
      "rewards: sum = 869.3132076379745\n",
      "110\n",
      "rewards: sum = 885.605521167952\n",
      "111\n",
      "rewards: sum = 904.648925306919\n",
      "112\n",
      "rewards: sum = 876.8297294798848\n",
      "113\n",
      "rewards: sum = 875.9425116335061\n",
      "114\n",
      "rewards: sum = 859.3138985533194\n",
      "115\n",
      "rewards: sum = 896.0121033288492\n",
      "116\n",
      "rewards: sum = 883.3270857664729\n",
      "117\n",
      "rewards: sum = 885.2384869879169\n",
      "118\n",
      "rewards: sum = 905.4902121118921\n",
      "119\n",
      "rewards: sum = 874.5613911275354\n",
      "120\n",
      "rewards: sum = 900.173096402854\n",
      "121\n",
      "rewards: sum = 906.6125687178132\n",
      "122\n",
      "rewards: sum = 869.8399541657919\n",
      "123\n",
      "rewards: sum = 897.3525655404424\n",
      "124\n",
      "rewards: sum = 882.7358544631047\n",
      "125\n",
      "rewards: sum = 880.1140476267295\n",
      "126\n",
      "rewards: sum = 867.8751980722742\n",
      "127\n",
      "rewards: sum = 886.3557792963463\n",
      "128\n",
      "rewards: sum = 888.3169191519439\n",
      "129\n",
      "rewards: sum = 895.0608496324099\n",
      "130\n",
      "rewards: sum = 874.2996137935713\n",
      "131\n",
      "rewards: sum = 866.5415474741133\n",
      "132\n",
      "rewards: sum = 896.9772065911468\n",
      "133\n",
      "rewards: sum = 908.1953438831422\n",
      "134\n",
      "rewards: sum = 877.4911076870802\n",
      "135\n",
      "rewards: sum = 871.8277994381117\n",
      "136\n",
      "rewards: sum = 897.7919084211632\n",
      "137\n",
      "rewards: sum = 882.0196374989864\n",
      "138\n",
      "rewards: sum = 901.1230401159547\n",
      "139\n",
      "rewards: sum = 894.7401410247085\n",
      "140\n",
      "rewards: sum = 875.4262999174857\n",
      "141\n",
      "rewards: sum = 868.940856199061\n",
      "142\n",
      "rewards: sum = 873.8985469960013\n",
      "143\n",
      "rewards: sum = 877.176272228336\n",
      "144\n",
      "rewards: sum = 874.6807737344345\n",
      "145\n",
      "rewards: sum = 880.6095389342806\n",
      "146\n",
      "rewards: sum = 899.5998276260364\n",
      "147\n",
      "rewards: sum = 904.6259297211284\n",
      "148\n",
      "rewards: sum = 891.537499132633\n",
      "149\n",
      "rewards: sum = 885.4084181786037\n",
      "150\n",
      "rewards: sum = 878.3407669313056\n",
      "151\n",
      "rewards: sum = 884.2035327490833\n",
      "152\n",
      "rewards: sum = 880.3855661686064\n",
      "153\n",
      "rewards: sum = 873.3097975933238\n",
      "154\n",
      "rewards: sum = 886.8109559181175\n",
      "155\n",
      "rewards: sum = 880.4762756123328\n",
      "156\n",
      "rewards: sum = 882.7655801814049\n",
      "157\n",
      "rewards: sum = 877.2979047255722\n",
      "158\n",
      "rewards: sum = 892.6267489700444\n",
      "159\n",
      "rewards: sum = 894.4956217047243\n",
      "160\n",
      "rewards: sum = 899.271683824795\n",
      "161\n",
      "rewards: sum = 903.3672494783788\n",
      "162\n",
      "rewards: sum = 900.2672554117383\n",
      "163\n",
      "rewards: sum = 896.8892139879791\n",
      "164\n",
      "rewards: sum = 880.6624677497364\n",
      "165\n",
      "rewards: sum = 880.0700071427483\n",
      "166\n",
      "rewards: sum = 889.988926414548\n",
      "167\n",
      "rewards: sum = 897.9832567074013\n",
      "168\n",
      "rewards: sum = 891.6839192903972\n",
      "169\n",
      "rewards: sum = 889.0628301191881\n",
      "170\n",
      "rewards: sum = 871.0577941564181\n",
      "171\n",
      "rewards: sum = 871.8166329747819\n",
      "172\n",
      "rewards: sum = 907.5437227034581\n",
      "173\n",
      "rewards: sum = 881.4289658824617\n",
      "174\n",
      "rewards: sum = 879.0560142258798\n",
      "175\n",
      "rewards: sum = 889.5663122836002\n",
      "176\n",
      "rewards: sum = 903.6944064497113\n",
      "177\n",
      "rewards: sum = 885.3211531556096\n",
      "178\n",
      "rewards: sum = 905.7183268704994\n",
      "179\n",
      "rewards: sum = 880.6149473825969\n",
      "180\n",
      "rewards: sum = 896.3333512810068\n",
      "181\n",
      "rewards: sum = 886.8225148284394\n",
      "182\n",
      "rewards: sum = 897.9333795506757\n",
      "183\n",
      "rewards: sum = 896.8336241145724\n",
      "184\n",
      "rewards: sum = 892.4037890477107\n",
      "185\n",
      "rewards: sum = 873.0821395156672\n",
      "186\n",
      "rewards: sum = 858.7041658727724\n",
      "187\n",
      "rewards: sum = 864.8203602415979\n",
      "188\n",
      "rewards: sum = 884.2697529649645\n",
      "189\n",
      "rewards: sum = 881.4868743841479\n",
      "190\n",
      "rewards: sum = 894.3075156396611\n",
      "191\n",
      "rewards: sum = 882.8933127460814\n",
      "192\n",
      "rewards: sum = 857.7496610625499\n",
      "193\n",
      "rewards: sum = 870.3293932168983\n",
      "194\n",
      "rewards: sum = 908.3577337424588\n",
      "195\n",
      "rewards: sum = 899.3688506135675\n",
      "196\n",
      "rewards: sum = 873.5455914068026\n",
      "197\n",
      "rewards: sum = 869.7594461471712\n",
      "198\n",
      "rewards: sum = 897.6000464546166\n",
      "199\n",
      "rewards: sum = 882.9467139045704\n",
      "200\n",
      "rewards: sum = 895.7135742065373\n",
      "201\n",
      "rewards: sum = 874.6181235762783\n",
      "202\n",
      "rewards: sum = 882.1051917323938\n",
      "203\n",
      "rewards: sum = 871.6068109441206\n",
      "204\n",
      "rewards: sum = 887.7014972571806\n",
      "205\n",
      "rewards: sum = 898.7713085530722\n",
      "206\n",
      "rewards: sum = 865.0594020336572\n",
      "207\n",
      "rewards: sum = 879.7226768549124\n",
      "208\n",
      "rewards: sum = 858.2652145960723\n",
      "209\n",
      "rewards: sum = 883.5465473106078\n",
      "210\n",
      "rewards: sum = 889.4173072025488\n",
      "211\n",
      "rewards: sum = 897.0758127364392\n",
      "212\n",
      "rewards: sum = 874.7689052086174\n",
      "213\n",
      "rewards: sum = 906.4591840207333\n",
      "214\n",
      "rewards: sum = 888.1710793771435\n",
      "215\n",
      "rewards: sum = 875.3350440186033\n",
      "Time = 325.42\n"
     ]
    }
   ],
   "source": [
    "list_of_numsteps=[]\n",
    "list_of_rewards = []\n",
    "list_of_variants = []\n",
    "begin = time.time()\n",
    "for k, variant in enumerate(res):\n",
    "    print(k)\n",
    "    set_tau = rollouts(\n",
    "        variant,\n",
    "        env,\n",
    "        policy_net,\n",
    "        action_range,\n",
    "        device,\n",
    "        max_timesteps=args.max_timesteps,\n",
    "        time_horizon=args.H)\n",
    "    list_of_numsteps.append(len(set_tau['obs'][0]))\n",
    "    list_of_rewards.append(np.sum(set_tau['rewards'][0]))\n",
    "    list_of_variants.append(variant)\n",
    "end = time.time()\n",
    "print(\"Time = {0:.2f}\".format(end-begin))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Percentage of successful runs\n",
      "100.0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Reward</th>\n",
       "      <td>216.0</td>\n",
       "      <td>886.135645</td>\n",
       "      <td>12.723838</td>\n",
       "      <td>857.749661</td>\n",
       "      <td>876.962865</td>\n",
       "      <td>886.816735</td>\n",
       "      <td>896.847522</td>\n",
       "      <td>908.599989</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        count        mean        std         min         25%         50%  \\\n",
       "Reward  216.0  886.135645  12.723838  857.749661  876.962865  886.816735   \n",
       "\n",
       "               75%         max  \n",
       "Reward  896.847522  908.599989  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reward    886.816735\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "df = pd.DataFrame({'Variant': list_of_variants, \"Reward\" : list_of_rewards, 'Len' : list_of_numsteps})\n",
    "\n",
    "print('Percentage of successful runs')\n",
    "print((1 - len(df[df['Len'] < 250])/len(df))*100)\n",
    "\n",
    "display(df[['Reward']].describe().T)\n",
    "\n",
    "print(df[['Reward']].median())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Anedoctal Run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rewards: sum = 842.9171796620125\n"
     ]
    }
   ],
   "source": [
    "## Remember to chance headless flag!\n",
    "env.pr.stop()\n",
    "env.pr.start()\n",
    "\n",
    "for variant in [[0,0,6,0,1.7,0]]:\n",
    "     set_tau = rollouts(\n",
    "        variant,\n",
    "        env,\n",
    "        policy_net,\n",
    "        action_range,\n",
    "        device,\n",
    "        max_timesteps=args.max_timesteps,\n",
    "        time_horizon=args.H)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "289.097px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}