Project-3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### EECE 5698\n",
    "# Project-3\n",
    "##### Devesh Tarasia (001537213)\n",
    "tarasia.dev@northeastern.edu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import math as maths\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import random as rnd\n",
    "from matplotlib.patches import Rectangle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Maze():\n",
    "    def __init__(self,nRows,nCols,actionList,prob,startPosn,goalPosn,oilPosn,bumpPosn,wallPosn,oilR,bumpR,goalR,emptyR,actionR):\n",
    "        self.nRows = nRows\n",
    "        self.nCols = nCols\n",
    "        self.actionList = actionList\n",
    "        self.startPosn = startPosn\n",
    "        self.goalPosn = goalPosn\n",
    "        self.oilPosn = oilPosn\n",
    "        self.bumpPosn = bumpPosn\n",
    "        self.wallPosn = wallPosn\n",
    "        self.oilR = oilR\n",
    "        self.bumpR = bumpR\n",
    "        self.goalR = goalR\n",
    "        self.emptyR = emptyR\n",
    "        self.actionR = actionR\n",
    "        self.p = prob\n",
    "        self.maze = np.zeros((self.nRows,self.nCols))\n",
    "        self.values = np.zeros((self.nRows,self.nCols))\n",
    "        self.policy = np.random.choice(self.actionList,(self.nRows,self.nCols))\n",
    "        self.currentI = 1\n",
    "        self.currentJ = 1\n",
    "\n",
    "        self.maze[self.goalPosn] = self.goalR\n",
    "        self.values[self.goalPosn] = self.goalR\n",
    "        for i,j in self.oilPosn:\n",
    "            self.maze[i,j] = self.oilR\n",
    "        for i,j in self.bumpPosn:\n",
    "            self.maze[i,j] = self.bumpR\n",
    "        for i,j in self.wallPosn:\n",
    "            self.maze[i,j] = 0.0\n",
    "\n",
    "    def getActionList(self):\n",
    "        return self.actionList\n",
    "\n",
    "    def getStateValue(self,currentState): #May not be needing it\n",
    "        return self.values[currentState]\n",
    "\n",
    "    def getValue(self):\n",
    "        return self.values\n",
    "    \n",
    "    def setValue(self,currentState,value):\n",
    "        self.values[currentState] = value\n",
    "    \n",
    "    def getStatePolicy(self,currentState):\n",
    "        return self.policy[currentState]\n",
    "\n",
    "    def getPolicy(self):\n",
    "        return self.policy\n",
    "\n",
    "    def setStatePolicy(self,currentState,action):\n",
    "        self.policy[currentState] = action\n",
    "\n",
    "    def stateGenerator(self):\n",
    "        i = 1\n",
    "        j = 1\n",
    "        while i < self.nRows - 1:\n",
    "            if (i,j) not in self.wallPosn:\n",
    "                yield (i,j)\n",
    "            j = j + 1\n",
    "            if j == self.nCols - 1:\n",
    "                j = 1\n",
    "                i = i + 1\n",
    "\n",
    "    def reward(self,currentState,action=None):\n",
    "        i,j = currentState\n",
    "        if action == None:\n",
    "            action = self.policy[currentState]\n",
    "        probabilities = []\n",
    "        futureStates = [currentState,(i,j+1),(i,j-1),(i+1,j),(i-1,j)] #current,R,L,D,U\n",
    "        rewards = np.array([self.maze[k]+self.actionR for k in futureStates])\n",
    "        \n",
    "        if action == 'R':\n",
    "            probabilities = [0,1-self.p,self.p/3,self.p/3,self.p/3]\n",
    "        if action == 'L':\n",
    "            probabilities = [0,self.p/3,1-self.p,self.p/3,self.p/3]\n",
    "        if action == 'D':\n",
    "            probabilities = [0,self.p/3,self.p/3,1-self.p,self.p/3]\n",
    "        if action == 'U':\n",
    "            probabilities = [0,self.p/3,self.p/3,self.p/3,1-self.p]\n",
    "        \n",
    "        for k in range(1,len(futureStates)): #to add probabilities if wall is in side\n",
    "            if futureStates[k] in wallPosn: \n",
    "                probabilities[0] += probabilities[k]\n",
    "                probabilities[k] = 0\n",
    "\n",
    "        xaxis = [i[0] for i in futureStates]\n",
    "        yaxis = [i[1] for i in futureStates]\n",
    "        adjacentStates = (xaxis,yaxis)\n",
    "        probs = np.array(probabilities)\n",
    "\n",
    "        return probs, rewards, adjacentStates\n",
    "\n",
    "\n",
    "gamma = 0.55\n",
    "theta = 0.01\n",
    "nrows = 20\n",
    "ncols = 20\n",
    "prob = 0.02\n",
    "actionList = ['R','L','U','D']\n",
    "oilRew = -5\n",
    "bumpRew = -10\n",
    "goalRew = 200\n",
    "actionRew = -1\n",
    "emptyRew = 0\n",
    "goalPosn = (3,13)\n",
    "startPosn = (15,4)\n",
    "oilPosn = [(2,8),(2,16),(4,2),(5,6),(9,18),(15,10),(16,10),(17,14),(17,17),(18,7)]\n",
    "bumpPosn = [(1,11),(1,12),(2,1),(2,2),(2,3),(5,1),(5,9),(5,17),(6,17),(7,17),(8,17),(7,10),(7,11),(7,2),(12,11),(12,12),(14,1),(14,2),(15,17),(15,18),(16,7)]\n",
    "wallPosn = [(0,i) for i in range(ncols)] + [(i,0) for i in range(nrows)] + [(i,ncols-1) for i in range(nrows)] + [(nrows-1,i) for i in range(ncols)]\n",
    "wallPosn = wallPosn + [(2,5),(3,5)] + [(4,i) for i in range(3,17)] + [(i,6) for i in range(6,13)]\n",
    "wallPosn = wallPosn + [(7,12),(7,13),(7,14),(15,14),(15,15),(15,16),(11,16),(11,17),(12,17),(13,17),(17,1),(17,2),(10,1),(10,2),(10,3),(10,4),(5,3),(6,3),(7,3),(12,3),(12,4),(12,5),(12,7),(13,7),(14,7),(15,7)] + [(17,i) for i in range(7,13)] + [(i,9) for i in range(6,11)] + [(i,10) for i in range(10,15)] +\\\n",
    "            [(i,15) for i in range(6,12)] + [(i,13) for i in range(11,16)]\n",
    "\n",
    "mazeq1 = Maze(nrows,ncols,actionList,prob,startPosn,goalPosn,oilPosn,bumpPosn,wallPosn,oilRew,bumpRew,goalRew,emptyRew,actionRew)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "State_Matrix = \\\n",
    "    np.array([[0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0, 0],\n",
    "              [0, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 0],\n",
    "              [0, 214, 215, 216, 217,   0, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 0],\n",
    "              [0, 197, 198, 199, 200,   0, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 0],\n",
    "              [0, 193, 194,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0, 195, 196, 0],\n",
    "              [0, 176, 177,   0, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 0],\n",
    "              [0, 162, 163,   0, 164, 165,   0, 166, 167,   0, 168, 169, 170, 171, 172,   0, 173, 174, 175, 0],\n",
    "              [0, 151, 152,   0, 153, 154,   0, 155, 156,   0, 157, 158,   0,   0,   0,   0, 159, 160, 161, 0],\n",
    "              [0, 136, 137, 138, 139, 140,   0, 141, 142,   0, 143, 144, 145, 146, 147,   0, 148, 149, 150, 0],\n",
    "              [0, 121, 122, 123, 124, 125,   0, 126, 127,   0, 128, 129, 130, 131, 132,   0, 133, 134, 135, 0],\n",
    "              [0,   0,   0,   0,   0, 111,   0, 112, 113,   0,   0, 114, 115, 116, 117,   0, 118, 119, 120, 0],\n",
    "              [0,  99, 100, 101, 102, 103,   0, 104, 105, 106,   0, 107, 108,   0, 109,   0,   0,   0, 110, 0],\n",
    "              [0,  89,  90,   np.NAN,   np.NAN,   np.NAN,  np.NAN,   np.NAN,  91,  92,   0,  93,  94,   0,  95,  96,  97,   0,  98, 0],\n",
    "              [0,  75,  76,  77,  78,  79,  80,   0,  81,  82,   0,  83,  84,   0,  85,  86,  87,   0,  88, 0],\n",
    "              [0,  60,  61,  62,  63,  64,  65,   0,  66,  67,   0,  68,  69,   0,  70,  71,  72,  73,  74, 0],\n",
    "              [0,  47,  48,  49,  50,  51,  52,   0,  53,  54,  55,  56,  57,   0,   0,   0,   0,  58,  59, 0],\n",
    "              [0,  29,  30,  31,  32,  33,  34,  35,  36,  37,  38,  39,  40,  41,  42,  43,  44,  45,  46, 0],\n",
    "              [0,   0,   0,  19,  20,  21,  22,   0,   0,   0,   0,   0,   0,  23,  24,  25,  26,  27,  28, 0],\n",
    "              [0,   1,   2,   3,   4,   5,   6,   7,   8,   9,  10,  11,  12,  13,  14,  15,  16,  17,  18, 0],\n",
    "              [0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0, 0]])\n",
    "        \n",
    "\n",
    "plt.subplots(figsize=(10,7.5))\n",
    "heatmap = sns.heatmap(State_Matrix, fmt=\".2f\", linewidths=0.25, linecolor='black',\n",
    "                      cbar= False, cmap= 'rocket_r')\n",
    "heatmap.set_facecolor('black') # Color for the NaN cells in the state matrix\n",
    "plt.title('Maze Problem')\n",
    "plt.show()\n",
    "\n",
    "def coloring_blocks(heatmap, oil_states, bump_states, start_state, end_state):\n",
    "    # Adding red oil blocks\n",
    "    for i in range(len(oil_states)):\n",
    "        heatmap.add_patch(Rectangle((oil_states[i][1], oil_states[i][0]), 1, 1,\n",
    "                                    fill=True, facecolor='red', edgecolor='red', lw=0.25))\n",
    "    # Adding salmon bump blocks\n",
    "    for i in range(len(bump_states)):\n",
    "        heatmap.add_patch(Rectangle((bump_states[i][1], bump_states[i][0]), 1, 1,\n",
    "                                    fill=True, facecolor='lightsalmon', edgecolor='lightsalmon', lw=0.25))\n",
    "    # Adding start block (Blue)\n",
    "    heatmap.add_patch(Rectangle((start_state[1], start_state[0]), 1, 1,\n",
    "                                fill=True, facecolor='lightblue', edgecolor='lightblue', lw=0.25))\n",
    "\n",
    "    # Adding end block (Green)\n",
    "    heatmap.add_patch(Rectangle((end_state[1], end_state[0]), 1, 1,\n",
    "                                fill=True, facecolor='lightgreen', edgecolor='lightgreen', lw=0.25))\n",
    "\n",
    "# Example Use\n",
    "plt.subplots(figsize=(10,7.5))    \n",
    "heatmap = sns.heatmap(State_Matrix, fmt=\".2f\", linewidths=0.25, linecolor='black',\n",
    "                      cbar= False, cmap= 'rocket_r')\n",
    "heatmap.set_facecolor('black') # Color for the NaN cells in the state matrix\n",
    "coloring_blocks(heatmap, oil_states=[(3,5),(4,6)], bump_states=[(13,15),(14,16)], \\\n",
    "                start_state=(2,2),end_state=(12,12))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Q2\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "def XOR(a, b):\n",
    "    if a != b:\n",
    "        return 1\n",
    "    else:\n",
    "        return 0\n",
    "\n",
    "def to_binary(n):\n",
    "  bin_arr = [0,0,0,0]\n",
    "  i = 0\n",
    "  while (n>0):\n",
    "      bin_arr[i] = n%2\n",
    "      n = int(n/2)\n",
    "      i = i+1\n",
    "  bin_arr.reverse()\n",
    "  return np.array(bin_arr)\n",
    "\n",
    "def to_decimal(arr):\n",
    "  bin_arr = np.flip(arr)\n",
    "  mulArr = np.array([2**x for x in range(4)])\n",
    "  return np.sum(bin_arr*mulArr)\n",
    "\n",
    "def getnk(p):\n",
    "  q = np.random.binomial(1,p,size=4)\n",
    "  return q\n",
    "\n",
    "def vNormalize(value):\n",
    "  if value <= 0:\n",
    "    return 0\n",
    "  else:\n",
    "    return 1\n",
    "\n",
    "def epGreedy(classObj,s,p):\n",
    "  ep = np.random.uniform(0.0,1.0)\n",
    "  if ep < p:\n",
    "    return classObj.randomAction()\n",
    "  else:\n",
    "    return classObj.greedy(s)\n",
    "\n",
    "def SARSA(classObj,alpha,gamma):\n",
    "  numEpCount = 0\n",
    "  maxEpCount = 1000\n",
    "  EpLength = 0\n",
    "  maxEpLength = 1000\n",
    "  avgReward = np.zeros(maxEpLength)\n",
    "  while numEpCount < maxEpCount:\n",
    "    s = classObj.randomState()\n",
    "    a = epGreedy(classObj,s,p)\n",
    "    EpLength = 0\n",
    "    while EpLength < maxEpLength:\n",
    "      s_ = classObj.nextState(s,a)\n",
    "      r = classObj.getR(s,s_,a)\n",
    "      avgReward[EpLength] += r \n",
    "      a_ = epGreedy(classObj,s_,p)\n",
    "      qsa = classObj.getQ(s,a)\n",
    "      qsa_ = classObj.getQ(s_,a_)\n",
    "      delta = r + gamma*qsa_ - qsa\n",
    "      qsa = qsa + alpha*delta\n",
    "      classObj.setQ(s,a,qsa)\n",
    "      #print(\"S:\",s,\"A:\",a,\"R:\",r,\"S-:\",s_,\"A-:\",a_,\"val:\",val)\n",
    "      s = s_\n",
    "      a = a_\n",
    "      EpLength += 1\n",
    "    \n",
    "    numEpCount += 1\n",
    "  avgReward = avgReward/maxEpCount\n",
    "\n",
    "def QLearn(classObj,alpha,gamma):\n",
    "  numEpCount = 0\n",
    "  maxEpCount = 1000\n",
    "  EpLength = 0\n",
    "  maxEpLength = 1000\n",
    "  avgReward = np.zeros(maxEpLength)\n",
    "  while numEpCount < maxEpCount:\n",
    "    s = classObj.randomState()\n",
    "    EpLength = 0\n",
    "    while EpLength < maxEpLength:\n",
    "      a = epGreedy(classObj,s,p)\n",
    "      s_ = classObj.nextState(s,a)\n",
    "      r = classObj.getR(s,s_,a)\n",
    "      avgReward[EpLength] += r \n",
    "      qsa = classObj.getQ(s,a)\n",
    "      a_ = classObj.greedy(s_)\n",
    "      qsa_ = classObj.getQ(s_,a_)\n",
    "      delta = r + gamma*qsa_ - qsa\n",
    "      qsa = qsa + alpha*delta\n",
    "      classObj.setQ(s,a,qsa)\n",
    "      #print(\"S:\",s,\"A:\",a,\"R:\",r,\"S-:\",s_,\"A-:\",a_,\"val:\",val)\n",
    "      s = s_\n",
    "      EpLength += 1\n",
    "    \n",
    "    numEpCount += 1\n",
    "  avgReward = avgReward/maxEpCount\n",
    "\n",
    "\n",
    "def SARSA_l(classObj,alpha,gamma,lamb):\n",
    "  numEpCount = 0\n",
    "  maxEpCount = 1000\n",
    "  EpLength = 0\n",
    "  maxEpLength = 1000\n",
    "  avgReward = np.zeros(maxEpLength)\n",
    "  while numEpCount < maxEpCount:\n",
    "    s = classObj.randomState()\n",
    "    a = classObj.randomAction()\n",
    "    classObj.resetE()\n",
    "    EpLength = 0\n",
    "    while EpLength < maxEpLength:\n",
    "      s_ = classObj.nextState(s,a)\n",
    "      r = classObj.getR(s,s_,a)\n",
    "      a_ = epGreedy(classObj,s_,p)\n",
    "      qsa = classObj.getQ(s,a)\n",
    "      qsa_ = classObj.getQ(s_,a_)\n",
    "      delta = r + gamma*qsa_ - qsa\n",
    "      classObj.Eincrement(s,a)\n",
    "      avgReward[EpLength] += r \n",
    "      for pair in classObj.stateActionIter():\n",
    "        s,a = pair\n",
    "        e = classObj.getE(s,a)\n",
    "        qsa = qsa + alpha*delta*e\n",
    "        classObj.setQ(s,a,qsa)\n",
    "        classObj.changeE(s,a,gamma,lamb)\n",
    "      #print(\"S:\",s,\"A:\",a,\"R:\",r,\"S-:\",s_,\"A-:\",a_,\"val:\",val)\n",
    "      s = s_\n",
    "      a = a_\n",
    "      EpLength += 1\n",
    "    \n",
    "    numEpCount += 1\n",
    "  avgReward = avgReward/maxEpCount\n",
    "\n",
    "def AC(classObj,alpha,gamma,beta):\n",
    "  numEpCount = 0\n",
    "  maxEpCount = 2\n",
    "  EpLength = 0\n",
    "  maxEpLength = 10\n",
    "  while numEpCount < maxEpCount:\n",
    "    s = classObj.randomState()\n",
    "    while EpLength < maxEpLength:\n",
    "      classObj.updateP()\n",
    "      a = np.random.choice([x for x in range(len(classObj.actions))],p=classObj.getPS(s))\n",
    "      s_ = classObj.nextState(s,a)\n",
    "      r = classObj.getR(s,s_,a)\n",
    "      vs = classObj.getValue(s)\n",
    "      vs_ = classObj.getValue(s_)\n",
    "      delta = r + gamma*vs_ - vs\n",
    "      vs = vs + alpha*delta\n",
    "      classObj.updateH(beta,delta,s,a)\n",
    "      s = s_\n",
    "      EpLength += 1\n",
    "    \n",
    "    numEpCount += 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Genes():\n",
    "  def __init__(self,actionList,prob,connectivityMatrix):\n",
    "    self.actions = actionList #a1,a2,a3,a4 in that order\n",
    "    self.p = prob\n",
    "    self.C = connectivityMatrix\n",
    "    #self.Rsas = np.array([np.zeros((16,16))] * len(self.actions))\n",
    "    #self.transistion = np.array([np.zeros((16,16))] * len(self.actions))\n",
    "    #self.Ras = np.zeros((len(self.actions),16))\n",
    "    #self.values = np.zeros((16,1))\n",
    "    self.Q = np.zeros((16,len(self.actions))) #Row is states, columns is actions\n",
    "    self.e = np.zeros((16,len(self.actions))) #Row is states, columns is actions\n",
    "    self.policy = np.zeros((16,)) #will store action values at 1,2,3,4\n",
    "    self.values = np.zeros((16,))\n",
    "    self.H = np.zeros((16,len(self.actions))) #Row is states, columns is actions\n",
    "    self.P = np.zeros((16,len(self.actions))) #Row is states, columns is actions \n",
    "    \n",
    "  def setACPolicy(self):\n",
    "    for i in range(16):\n",
    "      self.policy[i] = np.argmax(self.P[i])\n",
    "  \n",
    "  def getValue(self,s):\n",
    "    return self.values[s]\n",
    "    \n",
    "  def getPS(self,s):\n",
    "    return self.P[s,:]\n",
    "\n",
    "  def updateP(self):\n",
    "    for i in range(16):\n",
    "      for j in range(len(actionList)):\n",
    "        self.P[i,j] = np.exp(self.H[i,j])/np.sum(np.exp(self.H[i,:]))\n",
    "\n",
    "  def updateH(self,beta,delta,s,a):\n",
    "    self.H[s,a] += beta*delta*(1-self.P[s,a])\n",
    "\n",
    "  def resetE(self):\n",
    "    self.e = np.zeros((16,len(self.actions)))\n",
    "\n",
    "  def setQ(self,s,a,val):\n",
    "    self.Q[s,a] = val\n",
    "\n",
    "  def Eincrement(self,s,a):\n",
    "    self.e[s,a] += 1\n",
    "\n",
    "  def changeE(self,s,a,gamma,l):\n",
    "    self.e[s,a] = self.e[s,a]*gamma*l\n",
    "  \n",
    "  def getE(self,s,a):\n",
    "    return self.e[s,a]\n",
    "  \n",
    "  def setPolicy(self):\n",
    "    for i in range(16):\n",
    "      self.policy[i] = np.argmax(self.Q[i,:])\n",
    "\n",
    "  def getQ(self,s,a):\n",
    "    return self.Q[s,a]\n",
    "\n",
    "  def randomAction(self):\n",
    "    return np.random.randint(0,len(self.actions))\n",
    "\n",
    "  def randomState(self):\n",
    "    return np.random.randint(0,16)\n",
    "\n",
    "  def greedy(self,s):\n",
    "    slice = self.Q[s,:]\n",
    "    return np.argmax(slice)\n",
    "\n",
    "  def getR(self,stateCurrent,stateNext,action): #State number\n",
    "    stateC = to_binary(stateCurrent)\n",
    "    stateN = to_binary(stateNext)\n",
    "    a = self.actions[action]\n",
    "    reward = 5*stateN - a\n",
    "    return np.sum(reward)\n",
    "\n",
    "  def stateIter(self):\n",
    "    for i in range(16):\n",
    "      yield i\n",
    "  \n",
    "  def stateActionIter(self):\n",
    "    for i in range(16):\n",
    "      for j in range(len(self.actions)):\n",
    "        yield (i,j)\n",
    "\n",
    "  def nextState(self,currentState,action):\n",
    "    thisState = to_binary(currentState).reshape((4,1))\n",
    "    a = self.actions[action]\n",
    "    val = np.matmul(self.C,thisState).reshape((4,))\n",
    "    val = np.array(list(map(vNormalize,val)))\n",
    "    val = np.array(list(map(XOR,val,a)))\n",
    "    nk = getnk(self.p)\n",
    "    val = np.array(list(map(XOR,val,nk)))\n",
    "    nextS = to_decimal(val)\n",
    "    return nextS \n",
    "    \n",
    "\n",
    "p = 0.05\n",
    "C = np.array([[0,0,-1,0],[1,0,-1,-1],[0,1,0,0],[-1,1,1,0]])\n",
    "actionList = [np.array([0,0,0,0]),np.array([0,1,0,0]),np.array([0,0,1,0]),np.array([0,0,0,1])]\n",
    "alpha = 0.2\n",
    "gamma = 0.95\n",
    "beta = 0.05\n",
    "l = 0.95\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1. 1. 1. 1. 1. 1. 1. 1. 0. 1. 2. 1. 3. 1. 1. 1.]\n",
      "[1. 1. 1. 1. 1. 1. 1. 1. 0. 0. 1. 1. 0. 1. 1. 1.]\n",
      "[1. 1. 1. 1. 1. 1. 1. 1. 0. 3. 0. 2. 0. 1. 1. 1.]\n",
      "[1. 1. 1. 1. 1. 1. 1. 1. 0. 1. 0. 1. 3. 1. 1. 1.]\n",
      "[1. 2. 1. 1. 1. 1. 1. 1. 0. 1. 2. 1. 3. 1. 1. 1.]\n",
      "[1. 1. 1. 1. 1. 1. 1. 1. 0. 2. 0. 1. 0. 1. 1. 1.]\n",
      "[1. 1. 1. 1. 1. 1. 1. 1. 0. 2. 3. 1. 0. 1. 1. 1.]\n",
      "[1. 1. 1. 1. 1. 1. 1. 1. 0. 1. 0. 1. 0. 1. 1. 1.]\n",
      "[1. 1. 1. 1. 1. 1. 1. 1. 0. 1. 2. 1. 3. 1. 1. 1.]\n",
      "[1. 1. 1. 1. 1. 1. 1. 1. 0. 1. 0. 2. 0. 1. 1. 1.]\n"
     ]
    }
   ],
   "source": [
    "for i in range(10):\n",
    "    gene1 = Genes(actionList,p,C)\n",
    "    SARSA(gene1,alpha,gamma)\n",
    "    gene1.setPolicy()\n",
    "    print(gene1.policy)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1. 1. 1. 1. 1. 1. 1. 1. 0. 0. 0. 1. 0. 1. 1. 1.]\n",
      "[1. 1. 1. 1. 1. 1. 1. 1. 0. 1. 0. 1. 3. 1. 1. 1.]\n",
      "[2. 1. 1. 1. 1. 1. 1. 1. 0. 1. 1. 1. 0. 1. 1. 1.]\n",
      "[1. 1. 1. 1. 1. 1. 1. 1. 0. 1. 1. 1. 0. 1. 1. 1.]\n",
      "[1. 1. 1. 1. 1. 1. 1. 1. 0. 0. 2. 1. 3. 1. 1. 1.]\n",
      "[1. 1. 1. 1. 1. 1. 1. 1. 0. 0. 1. 1. 0. 1. 1. 1.]\n",
      "[1. 1. 1. 1. 1. 1. 1. 1. 0. 0. 0. 1. 3. 1. 1. 1.]\n",
      "[1. 1. 1. 1. 1. 1. 1. 1. 0. 1. 0. 1. 3. 1. 1. 1.]\n",
      "[1. 1. 1. 1. 1. 1. 1. 1. 0. 1. 0. 1. 0. 1. 1. 1.]\n",
      "[1. 1. 1. 1. 1. 1. 1. 1. 0. 2. 2. 1. 0. 1. 1. 1.]\n"
     ]
    }
   ],
   "source": [
    "for i in range(10):\n",
    "    gene2 = Genes(actionList,p,C)\n",
    "    QLearn(gene2,alpha,gamma)\n",
    "    gene2.setPolicy()\n",
    "    print(gene2.policy)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      "[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      "[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      "[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      "[3. 2. 3. 3. 2. 3. 0. 0. 0. 0. 0. 0. 2. 0. 0. 0.]\n",
      "[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      "[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m/home/devesh/EECE5698/Project-3.ipynb Cell 10'\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      <a href='vscode-notebook-cell://wsl%2Bubuntu/home/devesh/EECE5698/Project-3.ipynb#ch0000014vscode-remote?line=0'>1</a>\u001b[0m \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(\u001b[39m10\u001b[39m):\n\u001b[1;32m      <a href='vscode-notebook-cell://wsl%2Bubuntu/home/devesh/EECE5698/Project-3.ipynb#ch0000014vscode-remote?line=1'>2</a>\u001b[0m     gene3 \u001b[39m=\u001b[39m Genes(actionList,p,C)\n\u001b[0;32m----> <a href='vscode-notebook-cell://wsl%2Bubuntu/home/devesh/EECE5698/Project-3.ipynb#ch0000014vscode-remote?line=2'>3</a>\u001b[0m     SARSA_l(gene3,alpha,gamma,l)\n\u001b[1;32m      <a href='vscode-notebook-cell://wsl%2Bubuntu/home/devesh/EECE5698/Project-3.ipynb#ch0000014vscode-remote?line=3'>4</a>\u001b[0m     gene3\u001b[39m.\u001b[39msetPolicy()\n\u001b[1;32m      <a href='vscode-notebook-cell://wsl%2Bubuntu/home/devesh/EECE5698/Project-3.ipynb#ch0000014vscode-remote?line=4'>5</a>\u001b[0m     \u001b[39mprint\u001b[39m(gene3\u001b[39m.\u001b[39mpolicy)\n",
      "\u001b[1;32m/home/devesh/EECE5698/Project-3.ipynb Cell 6'\u001b[0m in \u001b[0;36mSARSA_l\u001b[0;34m(classObj, alpha, gamma, lamb)\u001b[0m\n\u001b[1;32m    <a href='vscode-notebook-cell://wsl%2Bubuntu/home/devesh/EECE5698/Project-3.ipynb#ch0000005vscode-remote?line=117'>118</a>\u001b[0m   qsa \u001b[39m=\u001b[39m qsa \u001b[39m+\u001b[39m alpha\u001b[39m*\u001b[39mdelta\u001b[39m*\u001b[39me\n\u001b[1;32m    <a href='vscode-notebook-cell://wsl%2Bubuntu/home/devesh/EECE5698/Project-3.ipynb#ch0000005vscode-remote?line=118'>119</a>\u001b[0m   classObj\u001b[39m.\u001b[39msetQ(s,a,qsa)\n\u001b[0;32m--> <a href='vscode-notebook-cell://wsl%2Bubuntu/home/devesh/EECE5698/Project-3.ipynb#ch0000005vscode-remote?line=119'>120</a>\u001b[0m   classObj\u001b[39m.\u001b[39;49mchangeE(s,a,gamma,lamb)\n\u001b[1;32m    <a href='vscode-notebook-cell://wsl%2Bubuntu/home/devesh/EECE5698/Project-3.ipynb#ch0000005vscode-remote?line=120'>121</a>\u001b[0m \u001b[39m#print(\"S:\",s,\"A:\",a,\"R:\",r,\"S-:\",s_,\"A-:\",a_,\"val:\",val)\u001b[39;00m\n\u001b[1;32m    <a href='vscode-notebook-cell://wsl%2Bubuntu/home/devesh/EECE5698/Project-3.ipynb#ch0000005vscode-remote?line=121'>122</a>\u001b[0m s \u001b[39m=\u001b[39m s_\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for i in range(10):\n",
    "    gene3 = Genes(actionList,p,C)\n",
    "    SARSA_l(gene3,alpha,gamma,l)\n",
    "    gene3.setPolicy()\n",
    "    print(gene3.policy)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1. 3. 0. 0. 0. 2. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      "[1. 2. 0. 0. 2. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      "[1. 0. 2. 2. 3. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      "[3. 0. 1. 0. 1. 2. 1. 3. 0. 1. 0. 0. 0. 0. 0. 3.]\n",
      "[0. 1. 1. 0. 0. 2. 0. 0. 0. 3. 0. 0. 0. 0. 0. 0.]\n",
      "[3. 0. 1. 0. 0. 1. 3. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      "[0. 2. 0. 1. 0. 2. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      "[2. 2. 2. 2. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      "[3. 1. 0. 0. 0. 2. 0. 0. 0. 2. 0. 0. 0. 0. 0. 0.]\n",
      "[1. 2. 0. 1. 2. 2. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n"
     ]
    }
   ],
   "source": [
    "#for i in range(10):\n",
    "for i in range(10):\n",
    "    gene4 = Genes(actionList,p,C)\n",
    "    AC(gene4,alpha,gamma,beta)\n",
    "    gene4.setACPolicy()\n",
    "    print(gene4.policy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "#    for k in range(len(self.actions)):\n",
    "#      for i in range(16):\n",
    "#        for j in range(16):\n",
    "#          statei = to_binary(i).reshape(4,1)\n",
    "#          statej = to_binary(j).reshape(4,1)\n",
    "#          v = np.matmul(self.C,statei)\n",
    "#          v = np.array(list(map(XOR,v,self.actions[k]))).reshape(4,1)\n",
    "#          v = statej - v\n",
    "#          value = abs(int(sum(v)))\n",
    "#          self.transistion[k,i,j] = (self.p**value)*((1-self.p)**(4-value))\n",
    "#          self.Rsas[k,i,j] = 5*sum(statej) - sum(self.actions[k])\n",
    "#    \n",
    "#    for k in range(len(self.actions)):\n",
    "#      ans = np.multiply(self.transistion[k],self.Rsas[k])\n",
    "#      self.Ras[k] = np.sum(ans,axis=1)\n",
    "#    \n",
    "#    self.Ras = np.reshape(self.Ras,(len(self.actions),16,1))\n",
    "#\n",
    "#  def getTransition(self):\n",
    "#    return self.transistion      \n",
    "#\n",
    "#  def getReward(self):\n",
    "#    return self.Ras\n",
    "#\n",
    "#  def getActions(self):\n",
    "#    return self.actions\n",
    "#  \n",
    "#  def getValues(self):\n",
    "#    return self.values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}