EmotionDetect_landmarks_CNN_Dynamic_RealTime.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#dependencies\n",
    "import numpy as np\n",
    "import cv2\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from keras.utils import np_utils\n",
    "from keras.models import Sequential, Model, model_from_json\n",
    "from keras.layers import Conv2D, Activation, MaxPool2D, Dropout, Dense, BatchNormalization, Flatten\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "import tarfile\n",
    "import dlib\n",
    "import time\n",
    "import os\n",
    "import re\n",
    "import math\n",
    "import itertools as it\n",
    "from PIL import Image\n",
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#0=Angry, 1=Disgust, 2=Fear, 3=Happy, 4=Sad, 5=Surprise, 6=Neutral"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def optimise_pts(x,y):\n",
    "    subset=[]\n",
    "    i=0\n",
    "    nose_tip=np.asarray((np.mean(x[30:36]),np.mean(y[30:36])))\n",
    "    leye=np.asarray((np.mean(x[36:42]),np.mean(y[36:42])))\n",
    "    reye=np.asarray((np.mean(x[42:48]),np.mean(y[42:48])))\n",
    "    scale_dist=np.mean([np.linalg.norm(nose_tip-leye),np.linalg.norm(nose_tip-reye)])\n",
    "    for lip_x,lip_y in zip(it.combinations(x[48:68],2),it.combinations(y[48:68],2)):#171\n",
    "        x0=lip_x[0]-lip_x[1]\n",
    "        y0=lip_y[0]-lip_y[1]\n",
    "        dist=np.linalg.norm([x0,y0])\n",
    "        subset.append(dist)\n",
    "    for leyebr_x,leyebr_y in zip(x[17:22],y[17:22]):#30\n",
    "        for leye_x, leye_y in zip(x[36:42],y[36:42]):\n",
    "            x1= leyebr_x - leye_x\n",
    "            y1= leyebr_y - leye_y\n",
    "            dist=np.linalg.norm([x1,y1])\n",
    "            subset.append(dist)\n",
    "    for reyebr_x,reyebr_y in zip(x[22:27],y[22:27]):#30\n",
    "        for reye_x, reye_y in zip(x[42:48],y[42:48]):\n",
    "            x2= reyebr_x - reye_x\n",
    "            y2= reyebr_y - reye_y\n",
    "            dist=np.linalg.norm([x2,y2])\n",
    "            subset.append(dist)\n",
    "    for jaw_x, jaw_y in zip(x[5:12],y[5:12]):#7\n",
    "        x3 = jaw_x-nose_tip[0]\n",
    "        y3 = jaw_y-nose_tip[1]\n",
    "        dist = np.linalg.norm([x3,y3])\n",
    "        subset.append(dist)\n",
    "    for nl_x,nl_y in zip(x[48:60],y[48:60]):#12\n",
    "        x4 = nl_x-nose_tip[0]\n",
    "        y4 = nl_y-nose_tip[1]\n",
    "        dist = np.linalg.norm([x4,y4])\n",
    "        subset.append(dist)\n",
    "    for leyebr_x,leyebr_y, reyebr_x,reyebr_y in zip(x[17:22],y[17:22],x[22:27],y[22:27]):#==255\n",
    "        x5 = leyebr_x - nose_tip[0]\n",
    "        y5 = leyebr_y - nose_tip[1]\n",
    "        dist1=np.linalg.norm([x5,y5])\n",
    "        x6 = reyebr_x - nose_tip[0]\n",
    "        y6 = reyebr_y - nose_tip[1]\n",
    "        dist2=np.linalg.norm([x6,y6])\n",
    "        dist= (dist1+dist2)/2\n",
    "        subset.append(dist)\n",
    "    subset.append(0)\n",
    "    subset=np.array(subset)\n",
    "    subset = subset/scale_dist*100\n",
    "    return subset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_landmarks(frame):\n",
    "    predictor = dlib.shape_predictor(\"shape_predictor_68_face_landmarks.dat\")\n",
    "    detector = dlib.get_frontal_face_detector()\n",
    "    clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8))\n",
    "    clahe_image = clahe.apply(frame)\n",
    "    detections = detector(clahe_image,1)\n",
    "    x=[]\n",
    "    y=[]\n",
    "    result_data=[]\n",
    "    noi=-1\n",
    "    for k,d in enumerate(detections): \n",
    "        noi=k\n",
    "        shape = predictor(clahe_image, d)\n",
    "        for i in range(1,68):\n",
    "            x.append(float(shape.part(i).x))\n",
    "            y.append(float(shape.part(i).y))\n",
    "            cv2.circle(frame, (shape.part(i).x, shape.part(i).y), 1, (0,0,255), thickness=2) \n",
    "        result_data.append(optimise_pts(x,y))\n",
    "    return result_data, noi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def classify_jaffe(filename):\n",
    "    if(re.search(r'(..\\.AN)', filename)):\n",
    "        return 0\n",
    "    if(re.search(r'(..\\.DI)', filename)):\n",
    "        return 1\n",
    "    elif(re.search(r'(..\\.FE)', filename)):\n",
    "        return 2\n",
    "    elif(re.search(r'(..\\.HA)', filename)):\n",
    "        return 3\n",
    "    elif(re.search(r'(..\\.SA)', filename)):\n",
    "        return 4\n",
    "    elif(re.search(r'(..\\.SU)', filename)):\n",
    "        return 5\n",
    "    elif(re.search(r'(..\\.NE)', filename)):\n",
    "        return 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output directory already exists\n",
      "start of pixel appending\n",
      "jaffe\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-21-9f62f022b38c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     19\u001b[0m                     \u001b[0;32mif\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mre\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msearch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mr'\\.tiff'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfilename\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mre\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msearch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mr'\\.jpg'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfilename\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mre\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msearch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mr'\\.png'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfilename\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mre\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msearch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mr'\\.jpeg'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfilename\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m                         \u001b[0mfilename\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdirpath\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m\"/\"\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m                         \u001b[0mdata_file\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnoi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_landmarks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcv2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcv2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIMREAD_UNCHANGED\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     22\u001b[0m                         \u001b[0;32mif\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnoi\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m                             \u001b[0;32mcontinue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-19-47ceafa2c64f>\u001b[0m in \u001b[0;36mget_landmarks\u001b[0;34m(frame)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_landmarks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mframe\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mpredictor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape_predictor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"shape_predictor_68_face_landmarks.dat\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0mdetector\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_frontal_face_detector\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mclahe\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcv2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreateCLAHE\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclipLimit\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2.0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtileGridSize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m8\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m8\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mclahe_image\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclahe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mframe\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "o_Dir = \"data_landmarks\"\n",
    "#dataset_jaffe = \"jaffe\"\n",
    "dataset_jaffe = \"train_dir\"\n",
    "data_file = []\n",
    "if not os.path.exists(o_Dir):\n",
    "    print(\"Making data_landmark directory\")\n",
    "    os.mkdir(o_Dir)\n",
    "else:\n",
    "    print(\"Output directory already exists\")\n",
    "import csv\n",
    "print(\"start of pixel appending\")\n",
    "with open(o_Dir+'/'+ dataset_jaffe+\" trn_label.csv\", 'w') as csvfile1:\n",
    "    lblw = csv.writer(csvfile1)\n",
    "    with open(o_Dir+'/'+ dataset_jaffe+\" trn_pixels.csv\", 'w') as csvfile2:\n",
    "        dw = csv.writer(csvfile2, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)\n",
    "        for (dirpath, dirnames, filenames) in os.walk(\"jaffe\"):\n",
    "                print(dirpath)\n",
    "                for filename in filenames:\n",
    "                    if(re.search(r'\\.tiff', filename) or re.search(r'\\.jpg', filename) or re.search(r'\\.png', filename) or re.search(r'\\.jpeg', filename)):\n",
    "                        filename = dirpath+\"/\"+filename\n",
    "                        data_file, noi = get_landmarks(cv2.imread(filename,cv2.IMREAD_UNCHANGED))\n",
    "                        if(noi == -1):\n",
    "                            continue\n",
    "                        else:\n",
    "                            label=(classify_jaffe(filename))\n",
    "                            for i in data_file:\n",
    "                                dw.writerow(i)\n",
    "                                lblw.writerow([label])\n",
    "print(\"appended to csv\")\n",
    "print(\"labels appended to csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data reshaped\n"
     ]
    }
   ],
   "source": [
    "### fer database creation\n",
    "data_comp = tarfile.open(\"fer2013.tar\") \n",
    "ds = pd.read_csv(data_comp.extractfile(\"fer2013/fer2013.csv\"))\n",
    "train = ds[[\"emotion\", \"pixels\"]][ds[\"Usage\"] == \"Training\"]\n",
    "train['pixels'] = train['pixels'].apply(lambda x: np.fromstring(x, sep=' '))\n",
    "train_pix = np.vstack(train['pixels'].values)\n",
    "test = ds[[\"emotion\", \"pixels\"]][ds[\"Usage\"] == \"PublicTest\"]\n",
    "test['pixels'] = test['pixels'].apply(lambda x: np.fromstring(x, sep=' '))\n",
    "test_pix = np.vstack(test['pixels'].values)\n",
    "test_pix = test_pix.reshape(-1,48,48)\n",
    "train_pix = train_pix.reshape(-1,48,48)\n",
    "print(\"all data reshaped\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((28709,), (3589,))"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_ind = np.array(train[\"emotion\"])\n",
    "test_ind = np.array(test[\"emotion\"])\n",
    "train_ind.shape, test_ind.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output directory already exists\n",
      "start of pixel appending\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-24-320ca4675a3c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     24\u001b[0m             \u001b[0mfilename\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtemp_image\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m             \u001b[0mimg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcv2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtemp_image\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m             \u001b[0mo_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnoi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_landmarks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     27\u001b[0m             \u001b[0;32mif\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnoi\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m                 \u001b[0;32mcontinue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-19-47ceafa2c64f>\u001b[0m in \u001b[0;36mget_landmarks\u001b[0;34m(frame)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_landmarks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mframe\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mpredictor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape_predictor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"shape_predictor_68_face_landmarks.dat\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0mdetector\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_frontal_face_detector\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mclahe\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcv2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreateCLAHE\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclipLimit\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2.0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtileGridSize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m8\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m8\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mclahe_image\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclahe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mframe\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#training fer\n",
    "o_Dir = \"data_landmarks\"\n",
    "#dataset_fer = \"fer2013\"\n",
    "dataset_fer = \"train_dir\"\n",
    "temp_image = \"saveddd.jpg\"\n",
    "data_file = []\n",
    "label= train_ind\n",
    "train_pix = np.array(train_pix)\n",
    "if not os.path.exists(o_Dir):\n",
    "    print(\"Making data_landmark directory\")\n",
    "    os.mkdir(o_Dir)\n",
    "else:\n",
    "    print(\"Output directory already exists\")\n",
    "import csv\n",
    "print(\"start of pixel appending\")\n",
    "with open(o_Dir+'/'+ dataset_fer+\" trn_label.csv\", 'a') as csvfile1:\n",
    "    lblw = csv.writer(csvfile1, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)\n",
    "    with open(o_Dir+'/'+ dataset_fer+\" trn_pixels.csv\", 'a') as csvfile2:\n",
    "        dw = csv.writer(csvfile2, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)\n",
    "        sh=train_pix.shape[0]\n",
    "        train_pix = list(train_pix)\n",
    "        for filen, lb in zip(range(sh),label):\n",
    "            filename = Image.fromarray(train_pix[filen]).convert('RGB')\n",
    "            filename.save(temp_image)\n",
    "            img = cv2.imread(temp_image,0)\n",
    "            o_data, noi = get_landmarks(img)\n",
    "            if(noi == -1):\n",
    "                continue\n",
    "            else:\n",
    "                for i in o_data:\n",
    "                    dw.writerow(i)\n",
    "                    lblw.writerow([lb])\n",
    "print(\"appended to csv\")\n",
    "print(\"labels appended to csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "##########"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output directory already exists\n",
      "start of pixel appending\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-26-4cccb9340e3b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     24\u001b[0m             \u001b[0mfilename\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtemp_image\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m             \u001b[0mimg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcv2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtemp_image\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m             \u001b[0mo_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnoi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_landmarks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     27\u001b[0m             \u001b[0;32mif\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnoi\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m                 \u001b[0;32mcontinue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-19-47ceafa2c64f>\u001b[0m in \u001b[0;36mget_landmarks\u001b[0;34m(frame)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_landmarks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mframe\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mpredictor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape_predictor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"shape_predictor_68_face_landmarks.dat\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0mdetector\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_frontal_face_detector\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mclahe\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcv2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreateCLAHE\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclipLimit\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2.0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtileGridSize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m8\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m8\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mclahe_image\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclahe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mframe\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#test\n",
    "o_Dir = \"data_landmarks\"\n",
    "#dataset_fer = \"fer2013\"\n",
    "dataset_fer = \"train_dir\"\n",
    "temp_image = \"saveddd.jpg\"\n",
    "data_file = []\n",
    "label= test_ind\n",
    "test_pix = np.array(test_pix)\n",
    "if not os.path.exists(o_Dir):\n",
    "    print(\"Making data_landmark directory\")\n",
    "    os.mkdir(o_Dir)\n",
    "else:\n",
    "    print(\"Output directory already exists\")\n",
    "import csv\n",
    "print(\"start of pixel appending\")\n",
    "with open(o_Dir+'/'+ dataset_fer+\" test_label.csv\", 'w') as csvfile1:\n",
    "    lblw = csv.writer(csvfile1, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)\n",
    "    with open(o_Dir+'/'+ dataset_fer+\" test_pixels.csv\", 'w') as csvfile2:\n",
    "        dw = csv.writer(csvfile2, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)\n",
    "        sh=test_pix.shape[0]\n",
    "        test_pix = list(test_pix)\n",
    "        for filen, lb in zip(range(sh),label):\n",
    "            filename = Image.fromarray(test_pix[filen]).convert('RGB')\n",
    "            filename.save(temp_image)\n",
    "            img = cv2.imread(temp_image,0)\n",
    "            o_data, noi = get_landmarks(img)\n",
    "            if(noi == -1):\n",
    "                continue\n",
    "            else:\n",
    "                for i in o_data:\n",
    "                    dw.writerow(i)\n",
    "                    lblw.writerow([lb])\n",
    "print(\"appended to csv\")\n",
    "print(\"labels appended to csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reading\n",
      "start of pixel reading\n"
     ]
    }
   ],
   "source": [
    "# reading csv file as whole\n",
    "o_Dir = \"data_landmarks\"\n",
    "dataset_fer = \"train_dir\"\n",
    "train_data=[]\n",
    "train_label=[]\n",
    "if not os.path.exists(o_Dir):\n",
    "    print(\"no traing file found\")\n",
    "else:\n",
    "    print(\"reading\")\n",
    "    import csv\n",
    "    print(\"start of pixel reading\")\n",
    "    with open(o_Dir+'/'+ dataset_fer+\" trn_pixels.csv\", 'r') as csvfile:\n",
    "        rd = csv.reader(csvfile)\n",
    "        for row in rd:\n",
    "            row = list(map(float, row))\n",
    "            train_data.append(row)\n",
    "    with open(o_Dir+'/'+ dataset_fer+\" trn_label.csv\", 'r') as csvfile:\n",
    "        rd = csv.reader(csvfile)\n",
    "        for row in rd:\n",
    "            train_label+=row\n",
    "train_label = list(map(int, train_label))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reading\n",
      "start of pixel reading\n"
     ]
    }
   ],
   "source": [
    "# reading csv file as whole\n",
    "o_Dir = \"data_landmarks\"\n",
    "dataset_fer = \"train_dir\"\n",
    "test_data=[]\n",
    "test_label=[]\n",
    "if not os.path.exists(o_Dir):\n",
    "    print(\"no test file found\")\n",
    "else:\n",
    "    print(\"reading\")\n",
    "    import csv\n",
    "    print(\"start of pixel reading\")\n",
    "    with open(o_Dir+'/'+ dataset_fer+\" test_pixels.csv\", 'r') as csvfile:\n",
    "        rd = csv.reader(csvfile)\n",
    "        for row in rd:\n",
    "            row = list(map(float, row))\n",
    "            test_data.append(row)\n",
    "    with open(o_Dir+'/'+ dataset_fer+\" test_label.csv\", 'r') as csvfile:\n",
    "        rd = csv.reader(csvfile)\n",
    "        for row in rd:\n",
    "            test_label+=row\n",
    "test_label = list(map(int, test_label))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((374, 256), (6, 256))"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data = np.vstack(train_data)\n",
    "train_data.shape\n",
    "test_data = np.vstack(test_data)\n",
    "train_data.shape, test_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((374,), (6,))"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_label = np.array(train_label)\n",
    "train_label.shape\n",
    "test_label = np.array(test_label)\n",
    "train_label.shape, test_label.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((374, 16, 16, 393), (6, 16, 16, 336))"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data = train_data.reshape(-1,16,16,1)\n",
    "train_label = np_utils.to_categorical(train_data)\n",
    "test_data = test_data.reshape(-1,16,16,1)\n",
    "test_label = np_utils.to_categorical(test_data)\n",
    "train_label.shape, test_label.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_12 (Conv2D)           (None, 13, 13, 64)        1088      \n",
      "_________________________________________________________________\n",
      "batch_normalization_13 (Batc (None, 13, 13, 64)        256       \n",
      "_________________________________________________________________\n",
      "activation_15 (Activation)   (None, 13, 13, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_13 (Conv2D)           (None, 10, 10, 64)        65600     \n",
      "_________________________________________________________________\n",
      "batch_normalization_14 (Batc (None, 10, 10, 64)        256       \n",
      "_________________________________________________________________\n",
      "activation_16 (Activation)   (None, 10, 10, 64)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_4 (MaxPooling2 (None, 5, 5, 64)          0         \n",
      "_________________________________________________________________\n",
      "conv2d_14 (Conv2D)           (None, 3, 3, 32)          18464     \n",
      "_________________________________________________________________\n",
      "batch_normalization_15 (Batc (None, 3, 3, 32)          128       \n",
      "_________________________________________________________________\n",
      "activation_17 (Activation)   (None, 3, 3, 32)          0         \n",
      "_________________________________________________________________\n",
      "conv2d_15 (Conv2D)           (None, 1, 1, 32)          9248      \n",
      "_________________________________________________________________\n",
      "batch_normalization_16 (Batc (None, 1, 1, 32)          128       \n",
      "_________________________________________________________________\n",
      "activation_18 (Activation)   (None, 1, 1, 32)          0         \n",
      "_________________________________________________________________\n",
      "flatten_3 (Flatten)          (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 128)               4224      \n",
      "_________________________________________________________________\n",
      "batch_normalization_17 (Batc (None, 128)               512       \n",
      "_________________________________________________________________\n",
      "activation_19 (Activation)   (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 7)                 903       \n",
      "_________________________________________________________________\n",
      "activation_20 (Activation)   (None, 7)                 0         \n",
      "=================================================================\n",
      "Total params: 100,807\n",
      "Trainable params: 100,167\n",
      "Non-trainable params: 640\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "###will be changed to random forest algo\n",
    "model = Sequential()\n",
    "model.add(Conv2D(64, (4,4), data_format=\"channels_last\", kernel_initializer=\"he_normal\", \n",
    "                 input_shape=(16, 16, 1)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Activation(\"relu\"))\n",
    "\n",
    "model.add(Conv2D(64, (4,4)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Activation(\"relu\"))\n",
    "model.add(MaxPool2D(pool_size=(2, 2), strides=2))\n",
    "\n",
    "model.add(Conv2D(32, (3,3)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Activation(\"relu\"))\n",
    "\n",
    "model.add(Conv2D(32, (3,3)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Activation(\"relu\"))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(128))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Activation(\"relu\"))\n",
    "model.add(Dropout(0.2))\n",
    "\n",
    "model.add(Dense(7))\n",
    "model.add(Activation('softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Error when checking target: expected activation_8 to have 2 dimensions, but got array with shape (374, 16, 16, 393)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-43-facc544855a7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m                  \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_label\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m                  \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcheckPoint\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m                  verbose=2)\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;31m# save model to json\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/keras/models.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m    963\u001b[0m                               \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    964\u001b[0m                               \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 965\u001b[0;31m                               validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m    966\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    967\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1591\u001b[0m             \u001b[0mclass_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mclass_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1592\u001b[0m             \u001b[0mcheck_batch_axis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1593\u001b[0;31m             batch_size=batch_size)\n\u001b[0m\u001b[1;32m   1594\u001b[0m         \u001b[0;31m# Prepare validation data.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1595\u001b[0m         \u001b[0mdo_validation\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_standardize_user_data\u001b[0;34m(self, x, y, sample_weight, class_weight, check_batch_axis, batch_size)\u001b[0m\n\u001b[1;32m   1428\u001b[0m                                     \u001b[0moutput_shapes\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1429\u001b[0m                                     \u001b[0mcheck_batch_axis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1430\u001b[0;31m                                     exception_prefix='target')\n\u001b[0m\u001b[1;32m   1431\u001b[0m         sample_weights = _standardize_sample_weights(sample_weight,\n\u001b[1;32m   1432\u001b[0m                                                      self._feed_output_names)\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_standardize_input_data\u001b[0;34m(data, names, shapes, check_batch_axis, exception_prefix)\u001b[0m\n\u001b[1;32m    108\u001b[0m                         \u001b[0;34m': expected '\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mnames\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m' to have '\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    109\u001b[0m                         \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m' dimensions, but got array '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 110\u001b[0;31m                         'with shape ' + str(data_shape))\n\u001b[0m\u001b[1;32m    111\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcheck_batch_axis\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    112\u001b[0m                     \u001b[0mdata_shape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata_shape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Error when checking target: expected activation_8 to have 2 dimensions, but got array with shape (374, 16, 16, 393)"
     ]
    }
   ],
   "source": [
    "###will be changed to random forest algo\n",
    "checkPoint = ModelCheckpoint(filepath='chkPt1.h5', verbose=1, save_best_only=True)\n",
    "res = model.fit(train_data, train_label, epochs=16,\n",
    "                 shuffle=True,\n",
    "                 batch_size=100, \n",
    "                 validation_data=(test_data, test_label),\n",
    "                 callbacks=[checkPoint], \n",
    "                 verbose=2)\n",
    "\n",
    "# save model to json\n",
    "model_json = model.to_json()\n",
    "with open(\"model1.json\", \"w\") as json_file:\n",
    "    json_file.write(model_json)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "plt.figure(figsize=(14,3))\n",
    "plt.subplot(1, 2, 1)\n",
    "plt.suptitle('Optimizer : Adam', fontsize=10)\n",
    "plt.ylabel('Loss', fontsize=16)\n",
    "plt.plot(res.history['loss'], color='b', label='Training Loss')\n",
    "plt.plot(res.history['val_loss'], color='r', label='Validation Loss')\n",
    "plt.legend(loc='upper right')\n",
    "\n",
    "plt.subplot(1, 2, 2)\n",
    "plt.ylabel('Accuracy', fontsize=16)\n",
    "plt.plot(res.history['acc'], color='b', label='Training Accuracy')\n",
    "plt.plot(res.history['val_acc'], color='r', label='Validation Accuracy')\n",
    "plt.legend(loc='lower right')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "###will be changed to random forest algo\n",
    "score = model.evaluate(test_data, test_label, verbose=0)\n",
    "score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import model_from_json\n",
    "import numpy as np\n",
    "\n",
    "class FacialExpressionModel(object):\n",
    "    EMOTIONS_LIST = [\"Angry\", \"Disgust\", \"Fear\", \"Happy\", \"Sad\", \"Surprise\", \"Neutral\"]\n",
    "\n",
    "    def __init__(self, model_json_file, model_weights_file):\n",
    "        # load model from JSON file\n",
    "        with open(model_json_file, \"r\") as json_file:\n",
    "            loaded_model_json = json_file.read()\n",
    "            self.loaded_model = model_from_json(loaded_model_json)\n",
    "\n",
    "        self.loaded_model.load_weights(model_weights_file)\n",
    "        print(\"Model loaded from disk\")\n",
    "        self.loaded_model.summary()\n",
    "\n",
    "    def predict_emotion(self, img):\n",
    "        self.preds = self.loaded_model.predict(img)\n",
    "        return FacialExpressionModel.EMOTIONS_LIST[np.argmax(self.preds)]\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import cv2\n",
    "import numpy as np\n",
    "rgb = cv2.VideoCapture(0)\n",
    "font = cv2.FONT_HERSHEY_SIMPLEX\n",
    "\n",
    "def start_app(cnn):\n",
    "    while True:\n",
    "\n",
    "        ret, frame = rgb.read()\n",
    "        fr = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)\n",
    "        data, noi = get_landmarks(fr)\n",
    "        data= np.array(data)\n",
    "#         print(data)\n",
    "#         for roi in data:\n",
    "#             pred = cnn.predict_emotion(roi.reshape(10,25,1))\n",
    "#             cv2.putText(frame, pred, (0, 0), font, 1, (255, 255, 0), 2)\n",
    "            #cv2.rectangle(frame,(x,y),(x+w,y+h),(255,0,0),2)\n",
    "\n",
    "        if cv2.waitKey(1) == 27:\n",
    "            break\n",
    "        cv2.imshow('Filter', frame)\n",
    "    cv2.destroyAllWindows()\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "#     model = FacialExpressionModel(\"model1.json\", \"chkPt1.h5\")\n",
    "    start_app(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}