Frozen Lake QL base code.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import gym\n",
    "import random\n",
    "\n",
    "env = gym.make(\"FrozenLake-v0\")\n",
    "\n",
    "action_size = env.action_space.n\n",
    "state_size = env.observation_space.n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Feel free to play with these hyperparameters\n",
    "\n",
    "total_episodes = 15000        # Total episodes\n",
    "test_episodes = 10            # Test episodes\n",
    "learning_rate = 0.8           # Learning rate\n",
    "max_steps = 99                # Max steps per episode\n",
    "gamma = 0.95                  # Discounting rate\n",
    "\n",
    "# Exploration parameters\n",
    "epsilon = 1.0                 # Exploration rate\n",
    "max_epsilon = 1.0             # Exploration probability at start\n",
    "min_epsilon = 0.01            # Minimum exploration probability \n",
    "decay_rate = 0.005            # Exponential decay rate for exploration prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initializations\n",
    "qtable = np.zeros((state_size, action_size))\n",
    "rewards = []\n",
    "\n",
    "for episode in range(total_episodes):\n",
    "    state = env.reset()\n",
    "    step = 0\n",
    "    done = False\n",
    "    total_rewards = 0\n",
    "    \n",
    "    for step in range(max_steps):\n",
    "        # Choose an action in the current state (greedy or explore)\n",
    "        \n",
    "        exp_exp_tradeoff = random.uniform(0, 1)  \n",
    "        # exploitation (taking the max Q value for this state)\n",
    "        if exp_exp_tradeoff > epsilon:\n",
    "            # Enter code here\n",
    "            ## Hint: Greedily choose an action according to Q value\n",
    "\n",
    "        # exploration\n",
    "        else:\n",
    "            # Enter code here\n",
    "            ## Hint: Randomly choose an action\n",
    "\n",
    "        # Take this action and observe\n",
    "        new_state, reward, done, info = env.step(action)\n",
    "\n",
    "        # Do a Q update\n",
    "        # Enter code here\n",
    "        ## Hint: One line update equation convert to one line code, start with \"qtable[state, action] = ...\"\n",
    "        \n",
    "        total_rewards += reward\n",
    "        \n",
    "        state = new_state\n",
    "        \n",
    "        if done == True: \n",
    "            break\n",
    "        \n",
    "    # Decay epsilon to reduce exploration as time progresses\n",
    "    \n",
    "    # Enter code here to assign a decay value to \"decay_parameter\"\n",
    "    \n",
    "    ## Hint: \n",
    "    ## 1. Use inbuilt polynomial, exponential(, or whatever works) functions to decay epsilon\n",
    "    ## 2. \"decay_parameter\" is a function of \"decay_rate\" and \"episode\"\n",
    "    \n",
    "    epsilon = min_epsilon + (max_epsilon - min_epsilon)*decay_parameter\n",
    "    rewards.append(total_rewards)\n",
    "\n",
    "print(\"Score over time: \" +  str(sum(rewards)/total_episodes))\n",
    "print(\"Q values:\")\n",
    "print(qtable)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(5 points) Q1. In short, explain why fixed \"epsilon\" above isn't the best choice? (Hint: You can keep epsilon fixed and see whether your reasoning explains the behavior)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################################################################\n",
    "#################### Final policy animation ############################\n",
    "########################################################################\n",
    "\n",
    "print(\"We only print the last state in each episode, to see if our agent has reached the destination or fallen into a hole\")\n",
    "env.reset()\n",
    "\n",
    "for episode in range(test_episodes):\n",
    "    state = env.reset()\n",
    "    step = 0\n",
    "    done = False\n",
    "    print(\"****************************************************\")\n",
    "    print(\"EPISODE \", episode)\n",
    "\n",
    "    for step in range(max_steps):\n",
    "        # Taking action with Q learning\n",
    "        action = np.argmax(qtable[state,:])\n",
    "        \n",
    "        new_state, reward, done, info = env.step(action)\n",
    "        \n",
    "        if done:\n",
    "            env.render()\n",
    "            \n",
    "            print(\"Number of steps\", step)\n",
    "            break\n",
    "        state = new_state\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(5 points) Q2. In some episodes above, the policy isn't reaching the goal, why?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}