CNN Sentiment Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import re\n",
    "import itertools\n",
    "from collections import Counter\n",
    "from tensorflow.contrib import learn\n",
    "import unittests as tests\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook was created for this talk:\n",
    "- https://www.infoq.com/br/presentations/deep-learning-for-sentiment-analysis?utm_source=infoq&utm_campaign=user_page&utm_medium=link\n",
    "\n",
    "And it was inspired by this blog post:\n",
    "- http://www.wildml.com/2015/12/implementing-a-cnn-for-text-classification-in-tensorflow/\n",
    "\n",
    "And this paper:\n",
    "- https://arxiv.org/abs/1408.5882"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Helpful functions (Taken from paper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def clean_str(string):\n",
    "    \"\"\"\n",
    "    Tokenization/string cleaning for all datasets except for SST.\n",
    "    Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py\n",
    "    \"\"\"\n",
    "    string = re.sub(r\"[^A-Za-z0-9(),!?\\'\\`]\", \" \", string)\n",
    "    string = re.sub(r\"\\'s\", \" \\'s\", string)\n",
    "    string = re.sub(r\"\\'ve\", \" \\'ve\", string)\n",
    "    string = re.sub(r\"n\\'t\", \" n\\'t\", string)\n",
    "    string = re.sub(r\"\\'re\", \" \\'re\", string)\n",
    "    string = re.sub(r\"\\'d\", \" \\'d\", string)\n",
    "    string = re.sub(r\"\\'ll\", \" \\'ll\", string)\n",
    "    string = re.sub(r\",\", \" , \", string)\n",
    "    string = re.sub(r\"!\", \" ! \", string)\n",
    "    string = re.sub(r\"\\(\", \" \\( \", string)\n",
    "    string = re.sub(r\"\\)\", \" \\) \", string)\n",
    "    string = re.sub(r\"\\?\", \" \\? \", string)\n",
    "    string = re.sub(r\"\\s{2,}\", \" \", string)\n",
    "    return string.strip().lower()\n",
    "\n",
    "\n",
    "def load_data_and_labels(positive_data_file, negative_data_file):\n",
    "    \"\"\"\n",
    "    Loads MR polarity data from files, splits the data into words and generates labels.\n",
    "    Returns split sentences and labels.\n",
    "    \"\"\"\n",
    "    # Load data from files\n",
    "    positive_examples = list(open(positive_data_file, \"r\").readlines())\n",
    "    positive_examples = [s.strip() for s in positive_examples]\n",
    "    negative_examples = list(open(negative_data_file, \"r\").readlines())\n",
    "    negative_examples = [s.strip() for s in negative_examples]\n",
    "    # Split by words\n",
    "    x_text = positive_examples + negative_examples\n",
    "    x_text = [clean_str(sent) for sent in x_text]\n",
    "    # Generate labels\n",
    "    positive_labels = [[0, 1] for _ in positive_examples]\n",
    "    negative_labels = [[1, 0] for _ in negative_examples]\n",
    "    y = np.concatenate([positive_labels, negative_labels], 0)\n",
    "    return [x_text, y]\n",
    "\n",
    "def batch_iter(data, batch_size, num_batches_per_epoch, num_epochs, shuffle=True):\n",
    "    \"\"\"\n",
    "    Generates a batch iterator for a dataset.\n",
    "    I changed it up a little bit for working on a single batch\n",
    "    \"\"\"\n",
    "    data = np.array(data)\n",
    "    #all data\n",
    "    data_size = len(data)\n",
    "    #each block has 64 data input, resulting in 150 blocks data\n",
    "    #num_batches_per_epoch = int((len(data)-1)/batch_size) + 1\n",
    "    for epoch in range(num_epochs):\n",
    "        # Shuffle the data at each epoch\n",
    "        if shuffle:\n",
    "            shuffle_indices = np.random.permutation(np.arange(data_size))\n",
    "            shuffled_data = data[shuffle_indices]\n",
    "        else:\n",
    "            shuffled_data = data\n",
    "        for batch_num in range(num_batches_per_epoch):\n",
    "            start_index = batch_num * batch_size\n",
    "            end_index = min((batch_num + 1) * batch_size, data_size)\n",
    "            print('Epoch {:>2}, Sentence Batch {}:  '.format(epoch + 1, batch_num), end='')\n",
    "            #150 blocks of data per epoch\n",
    "            yield shuffled_data[start_index:end_index]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data explore\n",
    "\n",
    "- Movie reviews from Rotten Tomatoes\n",
    "- 5331 positive reviews and 5331 negative reviews"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10662\n",
      "10662\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "data,labels = load_data_and_labels('data/rt-polaritydata/rt-polarity.pos','data/rt-polaritydata/rt-polarity.neg')\n",
    "\n",
    "print (len(data)) #all reviews each element is one review\n",
    "print (len(labels)) #all labels sentiments\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(\"the gorgeously elaborate continuation of the lord of the rings trilogy is so huge that a column of words cannot adequately describe co writer director peter jackson 's expanded vision of j r r tolkien 's middle earth\",\n",
       " array([0, 1]))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(data[1], labels[1]) #first review and its label"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pre processing\n",
    "- Build vocabulary\n",
    "- Maps documents to sequences of word ids.\n",
    "- Every sentence is padded as 0 until it gets max review size, which is 56"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 1,  2,  3,  4,  5,  6,  1,  7,  8,  9, 10, 11, 12, 13, 14,  9, 15,\n",
       "         5, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30,  0,\n",
       "         0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "         0,  0,  0,  0,  0]),\n",
       " \"the rock is destined to be the 21st century 's new conan and that he 's going to make a splash even greater than arnold schwarzenegger , jean claud van damme or steven segal\")"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_review_size = max([len(x.split(\" \")) for x in data])\n",
    "\n",
    "#Maps documents to sequences of word ids.\n",
    "vocab_processor = learn.preprocessing.VocabularyProcessor(max_review_size)\n",
    "x = np.array(list(vocab_processor.fit_transform(data)))\n",
    "(x[0], data[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "56"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_review_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "56"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(x[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Randomly shuffle data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 7359,  5573, 10180, ...,  1344,  7293,  1289])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Shuffle Data\n",
    "np.random.seed(10) #for debugging, garante que os numeros aleatorios gerados sempre sejam os mesmos\n",
    "shuffle_indices = np.random.permutation(np.arange(len(labels)))\n",
    "shuffle_indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#e.g.: x[7359] == x_shuffled[0]\n",
    "x_shuffled = x[shuffle_indices] \n",
    "y_shuffled = labels[shuffle_indices]\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " - Train/Validation split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "val_percentage = .1\n",
    "val_sample_index = -1 * int(val_percentage * float(len(labels)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train, x_val = x_shuffled[:val_sample_index], x_shuffled[val_sample_index:]\n",
    "y_train, y_val = y_shuffled[:val_sample_index], y_shuffled[val_sample_index:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocabulary Size: 18758\n",
      "Train/Val split: 9596/1066\n"
     ]
    }
   ],
   "source": [
    "print(\"Vocabulary Size: {:d}\".format(len(vocab_processor.vocabulary_)))\n",
    "print(\"Train/Val split: {:d}/{:d}\".format(len(y_train), len(y_val)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- The Architeture\n",
    "![](img/architeture.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build CNN\n",
    "\n",
    "- Inputs and Labels instances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sentence Input Tests Passed.\n",
      "Label Input Tests Passed.\n",
      "Keep Prob Tests Passed.\n"
     ]
    }
   ],
   "source": [
    "def neural_net_sentence_input(sentence_size):\n",
    "    \"\"\"\n",
    "    Return a Tensor for a batch of image input\n",
    "    : sentence_size: Size of the sentence with the biggest len\n",
    "    : return: Tensor for sentences input.\n",
    "    Remeber: all sentences were padded to get the max len\n",
    "    \"\"\"\n",
    "    return tf.placeholder(tf.int32, shape=[None,sentence_size],name='input_x')\n",
    "\n",
    "\n",
    "def neural_net_label_input(n_classes):\n",
    "    \"\"\"\n",
    "    Return a Tensor for a batch of label input\n",
    "    : n_classes: Number of classes\n",
    "    : return: Tensor for label input.\n",
    "    \"\"\"\n",
    "    return tf.placeholder(tf.float32, shape=[None,n_classes],name='input_y')\n",
    "\n",
    "\n",
    "def neural_net_keep_prob_input():\n",
    "    \"\"\"\n",
    "    Return a Tensor for keep probability\n",
    "    : return: Tensor for keep probability.\n",
    "    \"\"\"\n",
    "    return tf.placeholder(tf.float32, shape=None,name='keep_prob')\n",
    "\n",
    "\n",
    "\"\"\"\n",
    "UNIT TESTS\n",
    "\"\"\"\n",
    "tf.reset_default_graph()\n",
    "tests.test_nn_sentence_inputs(neural_net_sentence_input)\n",
    "tests.test_nn_label_inputs(neural_net_label_input)\n",
    "tests.test_nn_keep_prob_inputs(neural_net_keep_prob_input)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Load Pre Trained Word2Vec Model from GoogleNews Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-09-27 14:27:28,911 : INFO : loading projection weights from GoogleNews-vectors-negative300.bin\n",
      "2017-09-27 14:28:28,832 : INFO : loaded (3000000, 300) matrix from GoogleNews-vectors-negative300.bin\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 46.6 s, sys: 9.29 s, total: 55.8 s\n",
      "Wall time: 59.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import gensim\n",
    "import logging\n",
    "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)\n",
    "model = None\n",
    "model = gensim.models.KeyedVectors.load_word2vec_format('GoogleNews-vectors-negative300.bin', binary=True) \n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Store only the words that exists in our vocab \n",
    "- If I have a word that exists in my vocab but does not existis in the GoogleNews dataset I just randomize it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Remove previous weights, bias, inputs, etc..\n",
    "\n",
    "tf.reset_default_graph()\n",
    "vocab_size = len(vocab_processor.vocabulary_)\n",
    "W = tf.Variable(initial_value=tf.random_uniform([vocab_size, 300], -1.0, 1.0),name=\"K\")\n",
    "if(model):\n",
    "    T = np.random.rand(vocab_size, 300)\n",
    "vocab_dict = vocab_processor.vocabulary_._mapping\n",
    "for word,idx in vocab_dict.items():\n",
    "    if word in model:\n",
    "        T[idx] = model[word]\n",
    "    else:\n",
    "        T[idx] = np.random.uniform(low=-0.25, high=0.25, size=(300,))\n",
    "#save memory\n",
    "del model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Embedding Layer\n",
    "![](img/embed.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tests Passed\n"
     ]
    }
   ],
   "source": [
    "def embedding_creation(x_tensor,vocab_size,embedding_size):\n",
    "    embedded_chars = tf.nn.embedding_lookup(W, x_tensor)\n",
    "    embedded_chars_expanded = tf.expand_dims(embedded_chars, -1)\n",
    "    \n",
    "    return embedded_chars_expanded\n",
    "\n",
    "\n",
    "tests.test_embed(embedding_creation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Convolution Layer\n",
    "\n",
    "![](img/conv.png)\n",
    "![](img/maxpool.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tests Passed\n"
     ]
    }
   ],
   "source": [
    "def conv2d_maxpool(x_tensor, num_filters, filter_size):\n",
    "    \"\"\"\n",
    "    return: A tensor that represents convolution and max pooling of x_tensor\n",
    "    \"\"\"\n",
    "    embbeding_size = int(x_tensor.shape[2])\n",
    "    filter_shape = [filter_size,embbeding_size, 1, num_filters]\n",
    "    \n",
    "    weights = tf.Variable(tf.truncated_normal(filter_shape,stddev=0.1), name=\"W\")\n",
    "    bias = tf.Variable(tf.constant(0.1, shape=[num_filters]), name=\"b\")\n",
    "    \"\"\"\n",
    "    Strides controls how the filter convolves around the input\n",
    "    As we want to go each word per time, everything will have size one\n",
    "    As we apply conv layers, we could pad the image to preserve the dimension \n",
    "    (and try to extract more level features)\n",
    "    Because we are only dealing with words, this would not be necessary. This is known as narrow convolution\n",
    "    \n",
    "    Conv gives us an output of shape [1, sequence_length - filter_size + 1, 1, 1] - There is a formula to discover that\n",
    "    \n",
    "    \"\"\"\n",
    "    conv = tf.nn.conv2d(x_tensor, weights, strides=[1, 1, 1, 1], padding='VALID')\n",
    "\n",
    "    conv = tf.nn.bias_add(conv, bias)\n",
    "    #add non linearity\n",
    "    h = tf.nn.relu(conv, name=\"relu\")\n",
    "    sequence_length = int(x_tensor.shape[1])\n",
    "    conv_output = [1, sequence_length - filter_size + 1, 1, 1]\n",
    "    \n",
    "    #Maxpooling over the outputs\n",
    "    #this will heaturn a tensor of shape [batch_size, 1, 1, num_filters] \n",
    "    #which is essencialy a feature vector where the last dimension correspond to features\n",
    "    #Stride have this size basically because of the same logic applied before\n",
    "    pooled = tf.nn.max_pool(h, ksize=conv_output,\n",
    "                            strides=[1, 1, 1, 1],\n",
    "                            padding='VALID',\n",
    "                            name='pool') \n",
    "    \n",
    "    return pooled\n",
    "\n",
    "\n",
    "tests.test_con_pool(conv2d_maxpool)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Apply different filters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tests Passed\n"
     ]
    }
   ],
   "source": [
    "def apply_conv_filters(x_tensor,filter_sizes,num_filters):\n",
    "# Create a convolution + maxpool layer for each filter size\n",
    "    pooled_outputs = []\n",
    "    for i, filter_size in enumerate(filter_sizes):\n",
    "        with tf.name_scope(\"conv-maxpool-{}\".format(filter_size)):\n",
    "            pooled = conv2d_maxpool(x_tensor, num_filters, filter_size)\n",
    "            pooled_outputs.append(pooled)     \n",
    "    num_filters_total = num_filters * len(filter_sizes)\n",
    "    #concat -> sum(Daxis(i)) where Daxis is Dimension axis (in our case is the third one)\n",
    "    h_pool = tf.concat(pooled_outputs, 3)\n",
    "    return h_pool\n",
    "\n",
    "tests.test_apply_filters(apply_conv_filters,conv2d_maxpool)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Flatten Layer\n",
    "\n",
    "![](img/flatten.png)\n",
    "\n",
    "The output should be the shape (Batch Size, Flattened Features Size)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tests Passed\n"
     ]
    }
   ],
   "source": [
    "def flatten(x_tensor):\n",
    "    \"\"\"\n",
    "    : x_tensor: A tensor of size (Batch Size, ...), where ... are the image dimensions.\n",
    "    : return: A tensor of size (Batch Size, Flattened Image Size).\n",
    "    \"\"\"\n",
    "    #This is a general flatten function\n",
    "    flat = x_tensor.shape[1]*x_tensor.shape[2]*x_tensor.shape[3]\n",
    "    return tf.reshape(x_tensor,[-1,int(flat)])\n",
    "\n",
    "\n",
    "\n",
    "tests.test_flatten(flatten)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Output Layer\n",
    "\n",
    "![](img/output.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def output(x_tensor,num_classes):\n",
    "    num_filters_total = int(x_tensor.shape[1])\n",
    "    W = tf.Variable(tf.truncated_normal([num_filters_total, num_classes], stddev=0.1), name=\"W\")\n",
    "    b = tf.Variable(tf.constant(0.1, shape=[num_classes]), name=\"b\")\n",
    "\n",
    "    scores = tf.nn.xw_plus_b(x_tensor, W, b, name=\"scores\")\n",
    "    return scores, tf.nn.l2_loss(W), tf.nn.l2_loss(b)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Convolutional Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def conv_net(x, keep_prob):\n",
    "    \"\"\"\n",
    "    Create a convolutional neural network model\n",
    "    : x: Placeholder tensor that holds image data.\n",
    "    : keep_prob: Placeholder tensor that hold dropout keep probability.\n",
    "    : return: Tensor that represents logits\n",
    "    \"\"\"\n",
    "    #    Play around with different number of outputs, kernel size and stride\n",
    "    # Function Definition from Above:\n",
    "    #    conv2d_maxpool(x_tensor, conv_num_outputs, conv_ksize, conv_strides, pool_ksize, pool_strides)\n",
    "    \n",
    "    vocab_size = len(vocab_processor.vocabulary_)\n",
    "    embed_dim = 300\n",
    "    with tf.name_scope(\"embedding\"):\n",
    "        embbed_layer = embedding_creation(x,vocab_size,embed_dim)\n",
    "    \n",
    "    num_filters = 128\n",
    "    filter_sizes = [3,4,5]\n",
    "    conv_layer = apply_conv_filters(embbed_layer,filter_sizes,num_filters)\n",
    "\n",
    "    \n",
    "\n",
    "    flat_layer = flatten(conv_layer)\n",
    "    \n",
    "    with tf.name_scope(\"dropout\"):\n",
    "        dropout =  tf.nn.dropout(flat_layer, keep_prob)\n",
    "        \n",
    "    with tf.name_scope(\"output\"):\n",
    "        num_classes = 2\n",
    "        output_layer, l2_w, l2_b = output(dropout, num_classes)\n",
    "\n",
    "    \n",
    "    return output_layer, l2_w, l2_b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Neural Network Built!\n"
     ]
    }
   ],
   "source": [
    "#Regularization parameters\n",
    "l2_loss = tf.constant(0.0)\n",
    "l2_reg_lambda = 1.0\n",
    "\n",
    "\n",
    "# Inputs\n",
    "x_input = neural_net_sentence_input(56) #sequence_length\n",
    "y_input = neural_net_label_input(2) #positive or negative\n",
    "keep_prob = neural_net_keep_prob_input()\n",
    "\n",
    "# Model\n",
    "logits, l2_w, l2_b = conv_net(x_input, keep_prob)\n",
    "\n",
    "# Name logits Tensor, so that is can be loaded from disk after training\n",
    "logits = tf.identity(logits, name='logits')\n",
    "\n",
    "# Loss and Optimizer\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y_input))\n",
    "l2_loss += l2_w\n",
    "l2_loss += l2_b\n",
    "cost =  cost + l2_reg_lambda * l2_loss\n",
    "#optimizer = tf.train.AdamOptimizer().minimize(cost) - Other option for the optmizer, but got less validation acc\n",
    "global_step = tf.Variable(0, name=\"global_step\", trainable=False)\n",
    "optimizer = tf.train.AdamOptimizer(1e-3)\n",
    "grads_and_vars = optimizer.compute_gradients(cost)\n",
    "train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step)\n",
    "\n",
    "# Accuracy\n",
    "correct_pred = tf.equal(tf.argmax(logits, 1), tf.argmax(y_input, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32), name='accuracy')\n",
    "\n",
    "\n",
    "tests.test_conv_net(conv_net)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training Process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tests Passed\n"
     ]
    }
   ],
   "source": [
    "def train_neural_network(session, optimizer, keep_probability, feature_batch, label_batch):\n",
    "    \"\"\"\n",
    "    Optimize the session on a batch of images and labels\n",
    "    : session: Current TensorFlow session\n",
    "    : optimizer: TensorFlow optimizer function\n",
    "    : keep_probability: keep probability\n",
    "    : feature_batch: Batch of Numpy image data\n",
    "    : label_batch: Batch of Numpy label data\n",
    "    \"\"\"\n",
    "    \n",
    "    session.run(optimizer, feed_dict={\n",
    "            x_input: feature_batch,\n",
    "            y_input: label_batch,\n",
    "            keep_prob: keep_probability,\n",
    "            })\n",
    "\n",
    "\n",
    "tests.test_train_nn(train_neural_network)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Print statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def print_stats(session, feature_batch, label_batch, cost, accuracy):\n",
    "    \"\"\"\n",
    "    Print information about loss and validation accuracy\n",
    "    : session: Current TensorFlow session\n",
    "    : feature_batch: Batch of Numpy image data\n",
    "    : label_batch: Batch of Numpy label data\n",
    "    : cost: TensorFlow cost function\n",
    "    : accuracy: TensorFlow accuracy function\n",
    "    \"\"\"\n",
    "    loss,acc = session.run([cost,accuracy],feed_dict={\n",
    "            x_input: feature_batch,\n",
    "            y_input: label_batch,\n",
    "            keep_prob: 1.})\n",
    "    \n",
    "    \n",
    "    print('Loss: {:>10.4f} Training Accuracy: {:.6f}'.format(loss,acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def print_validation_stats(session):\n",
    "    \"\"\"\n",
    "    Print information about loss and validation accuracy\n",
    "    : session: Current TensorFlow session\n",
    "    : feature_batch: Batch of Numpy image data\n",
    "    : label_batch: Batch of Numpy label data\n",
    "    : cost: TensorFlow cost function\n",
    "    : accuracy: TensorFlow accuracy function\n",
    "    \"\"\"\n",
    "    \n",
    "    valid_acc = session.run(accuracy, feed_dict={\n",
    "        x_input: x_val,\n",
    "        y_input: y_val,\n",
    "        keep_prob: 1.})\n",
    "    \n",
    "    print('Validation Accuracy: {:.6f}'.format(valid_acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyperparameters\n",
    "\n",
    "- Just for a Single Batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "epochs = 12\n",
    "batch_size = 64\n",
    "keep_probability =  0.5\n",
    "num_batches_per_epoch = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training on a Single Batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking the Training on a Single Batch...\n",
      "Epoch  1, Sentence Batch 0:  Loss:     3.5260 Training Accuracy: 0.812500\n",
      "Epoch  2, Sentence Batch 0:  Loss:     3.3360 Training Accuracy: 0.953125\n",
      "Epoch  3, Sentence Batch 0:  Loss:     3.1959 Training Accuracy: 0.984375\n",
      "Epoch  4, Sentence Batch 0:  Loss:     3.0962 Training Accuracy: 1.000000\n",
      "Epoch  5, Sentence Batch 0:  Loss:     3.0177 Training Accuracy: 1.000000\n",
      "Epoch  6, Sentence Batch 0:  Loss:     2.9481 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 0:  Loss:     2.8869 Training Accuracy: 1.000000\n",
      "Epoch  8, Sentence Batch 0:  Loss:     2.8321 Training Accuracy: 1.000000\n",
      "Epoch  9, Sentence Batch 0:  Loss:     2.7813 Training Accuracy: 1.000000\n",
      "Epoch 10, Sentence Batch 0:  Loss:     2.7337 Training Accuracy: 1.000000\n",
      "Epoch 11, Sentence Batch 0:  Loss:     2.6881 Training Accuracy: 1.000000\n",
      "Epoch 12, Sentence Batch 0:  Loss:     2.6437 Training Accuracy: 1.000000\n",
      "#######VALIDATION STATS#######\n",
      "Validation Accuracy: 0.577861\n",
      "#######SAVING PARTIAL CHECKPOINT#######\n",
      "CPU times: user 27.4 s, sys: 3.75 s, total: 31.2 s\n",
      "Wall time: 6.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "print('Checking the Training on a Single Batch...')\n",
    "saver = tf.train.Saver()\n",
    "with tf.Session() as sess:\n",
    "    # Initializing the variables\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    if(T.any):\n",
    "        sess.run(W.assign(T))\n",
    "    # Generate single batches\n",
    "    batches = batch_iter(list(zip(x_train, y_train)), batch_size, 1, epochs, shuffle=False)\n",
    "    # Training cycle\n",
    "    for batch in batches:\n",
    "        batch_features, batch_labels = zip(*batch)\n",
    "        train_neural_network(sess, train_op, keep_probability, batch_features, batch_labels)\n",
    "        print_stats(sess, batch_features, batch_labels, cost, accuracy)\n",
    "    print(\"#######VALIDATION STATS#######\")\n",
    "    print_validation_stats(sess)\n",
    "    print(\"#######SAVING PARTIAL CHECKPOINT#######\")\n",
    "    save_path = saver.save(sess, \"./tmp/temp_ckpt.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Update Hyperparameters for full training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "epochs = 7\n",
    "batch_size = 64\n",
    "keep_probability =  0.3\n",
    "num_batches_per_epoch = int((len(list(zip(x_train, y_train)))-1)/batch_size) + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training...\n",
      "Epoch  1, Sentence Batch 0:  Validation Accuracy: 0.483114\n",
      "Loss:     3.7501 Training Accuracy: 0.703125\n",
      "Epoch  1, Sentence Batch 1:  Loss:     3.7115 Training Accuracy: 0.671875\n",
      "Epoch  1, Sentence Batch 2:  Loss:     4.0143 Training Accuracy: 0.484375\n",
      "Epoch  1, Sentence Batch 3:  Loss:     4.0085 Training Accuracy: 0.515625\n",
      "Epoch  1, Sentence Batch 4:  Loss:     3.9363 Training Accuracy: 0.484375\n",
      "Epoch  1, Sentence Batch 5:  Loss:     3.6925 Training Accuracy: 0.609375\n",
      "Epoch  1, Sentence Batch 6:  Loss:     3.6258 Training Accuracy: 0.625000\n",
      "Epoch  1, Sentence Batch 7:  Loss:     3.5709 Training Accuracy: 0.687500\n",
      "Epoch  1, Sentence Batch 8:  Loss:     3.6313 Training Accuracy: 0.578125\n",
      "Epoch  1, Sentence Batch 9:  Loss:     3.5387 Training Accuracy: 0.609375\n",
      "Epoch  1, Sentence Batch 10:  Loss:     3.6764 Training Accuracy: 0.578125\n",
      "Epoch  1, Sentence Batch 11:  Loss:     3.5353 Training Accuracy: 0.671875\n",
      "Epoch  1, Sentence Batch 12:  Loss:     3.8361 Training Accuracy: 0.515625\n",
      "Epoch  1, Sentence Batch 13:  Loss:     3.7059 Training Accuracy: 0.546875\n",
      "Epoch  1, Sentence Batch 14:  Loss:     3.6004 Training Accuracy: 0.500000\n",
      "Epoch  1, Sentence Batch 15:  Loss:     3.5452 Training Accuracy: 0.468750\n",
      "Epoch  1, Sentence Batch 16:  Loss:     3.4585 Training Accuracy: 0.421875\n",
      "Epoch  1, Sentence Batch 17:  Loss:     3.2073 Training Accuracy: 0.734375\n",
      "Epoch  1, Sentence Batch 18:  Loss:     3.2136 Training Accuracy: 0.671875\n",
      "Epoch  1, Sentence Batch 19:  Loss:     3.2927 Training Accuracy: 0.500000\n",
      "Epoch  1, Sentence Batch 20:  Loss:     3.2411 Training Accuracy: 0.625000\n",
      "Epoch  1, Sentence Batch 21:  Loss:     3.3001 Training Accuracy: 0.515625\n",
      "Epoch  1, Sentence Batch 22:  Loss:     3.2010 Training Accuracy: 0.546875\n",
      "Epoch  1, Sentence Batch 23:  Loss:     3.0539 Training Accuracy: 0.703125\n",
      "Epoch  1, Sentence Batch 24:  Loss:     3.0722 Training Accuracy: 0.562500\n",
      "Epoch  1, Sentence Batch 25:  Loss:     3.0126 Training Accuracy: 0.718750\n",
      "Epoch  1, Sentence Batch 26:  Loss:     2.9677 Training Accuracy: 0.671875\n",
      "Epoch  1, Sentence Batch 27:  Loss:     2.9448 Training Accuracy: 0.640625\n",
      "Epoch  1, Sentence Batch 28:  Loss:     2.9414 Training Accuracy: 0.671875\n",
      "Epoch  1, Sentence Batch 29:  Loss:     2.9166 Training Accuracy: 0.687500\n",
      "Epoch  1, Sentence Batch 30:  Loss:     2.9088 Training Accuracy: 0.609375\n",
      "Epoch  1, Sentence Batch 31:  Loss:     2.8605 Training Accuracy: 0.687500\n",
      "Epoch  1, Sentence Batch 32:  Loss:     2.9188 Training Accuracy: 0.578125\n",
      "Epoch  1, Sentence Batch 33:  Loss:     2.8656 Training Accuracy: 0.609375\n",
      "Epoch  1, Sentence Batch 34:  Loss:     2.7612 Training Accuracy: 0.671875\n",
      "Epoch  1, Sentence Batch 35:  Loss:     2.7797 Training Accuracy: 0.609375\n",
      "Epoch  1, Sentence Batch 36:  Loss:     2.7488 Training Accuracy: 0.609375\n",
      "Epoch  1, Sentence Batch 37:  Loss:     2.7093 Training Accuracy: 0.703125\n",
      "Epoch  1, Sentence Batch 38:  Loss:     2.6916 Training Accuracy: 0.687500\n",
      "Epoch  1, Sentence Batch 39:  Loss:     2.7514 Training Accuracy: 0.593750\n",
      "Epoch  1, Sentence Batch 40:  Loss:     2.6564 Training Accuracy: 0.656250\n",
      "Epoch  1, Sentence Batch 41:  Loss:     2.6541 Training Accuracy: 0.656250\n",
      "Epoch  1, Sentence Batch 42:  Loss:     2.7637 Training Accuracy: 0.500000\n",
      "Epoch  1, Sentence Batch 43:  Loss:     2.5265 Training Accuracy: 0.718750\n",
      "Epoch  1, Sentence Batch 44:  Loss:     2.5411 Training Accuracy: 0.687500\n",
      "Epoch  1, Sentence Batch 45:  Loss:     2.4774 Training Accuracy: 0.718750\n",
      "Epoch  1, Sentence Batch 46:  Loss:     2.4889 Training Accuracy: 0.656250\n",
      "Epoch  1, Sentence Batch 47:  Loss:     2.5014 Training Accuracy: 0.593750\n",
      "Epoch  1, Sentence Batch 48:  Loss:     2.4560 Training Accuracy: 0.625000\n",
      "Epoch  1, Sentence Batch 49:  Loss:     2.4063 Training Accuracy: 0.640625\n",
      "Epoch  1, Sentence Batch 50:  Loss:     2.3568 Training Accuracy: 0.671875\n",
      "Epoch  1, Sentence Batch 51:  Loss:     2.3492 Training Accuracy: 0.703125\n",
      "Epoch  1, Sentence Batch 52:  Loss:     2.3870 Training Accuracy: 0.656250\n",
      "Epoch  1, Sentence Batch 53:  Loss:     2.2617 Training Accuracy: 0.703125\n",
      "Epoch  1, Sentence Batch 54:  Loss:     2.2912 Training Accuracy: 0.718750\n",
      "Epoch  1, Sentence Batch 55:  Loss:     2.2732 Training Accuracy: 0.687500\n",
      "Epoch  1, Sentence Batch 56:  Loss:     2.1783 Training Accuracy: 0.812500\n",
      "Epoch  1, Sentence Batch 57:  Loss:     2.2088 Training Accuracy: 0.734375\n",
      "Epoch  1, Sentence Batch 58:  Loss:     2.0999 Training Accuracy: 0.781250\n",
      "Epoch  1, Sentence Batch 59:  Loss:     2.1678 Training Accuracy: 0.703125\n",
      "Epoch  1, Sentence Batch 60:  Loss:     2.2228 Training Accuracy: 0.609375\n",
      "Epoch  1, Sentence Batch 61:  Loss:     2.1559 Training Accuracy: 0.734375\n",
      "Epoch  1, Sentence Batch 62:  Loss:     2.0567 Training Accuracy: 0.734375\n",
      "Epoch  1, Sentence Batch 63:  Loss:     2.0553 Training Accuracy: 0.750000\n",
      "Epoch  1, Sentence Batch 64:  Loss:     2.0149 Training Accuracy: 0.765625\n",
      "Epoch  1, Sentence Batch 65:  Loss:     2.0675 Training Accuracy: 0.718750\n",
      "Epoch  1, Sentence Batch 66:  Loss:     2.0003 Training Accuracy: 0.750000\n",
      "Epoch  1, Sentence Batch 67:  Loss:     1.9597 Training Accuracy: 0.765625\n",
      "Epoch  1, Sentence Batch 68:  Loss:     2.0068 Training Accuracy: 0.718750\n",
      "Epoch  1, Sentence Batch 69:  Loss:     1.9798 Training Accuracy: 0.687500\n",
      "Epoch  1, Sentence Batch 70:  Loss:     1.9437 Training Accuracy: 0.781250\n",
      "Epoch  1, Sentence Batch 71:  Loss:     1.8574 Training Accuracy: 0.843750\n",
      "Epoch  1, Sentence Batch 72:  Loss:     1.8834 Training Accuracy: 0.718750\n",
      "Epoch  1, Sentence Batch 73:  Loss:     1.8503 Training Accuracy: 0.812500\n",
      "Epoch  1, Sentence Batch 74:  Loss:     1.8430 Training Accuracy: 0.812500\n",
      "Epoch  1, Sentence Batch 75:  Loss:     1.8359 Training Accuracy: 0.765625\n",
      "Epoch  1, Sentence Batch 76:  Loss:     1.8290 Training Accuracy: 0.734375\n",
      "Epoch  1, Sentence Batch 77:  Loss:     1.7529 Training Accuracy: 0.843750\n",
      "Epoch  1, Sentence Batch 78:  Loss:     1.8149 Training Accuracy: 0.671875\n",
      "Epoch  1, Sentence Batch 79:  Loss:     1.7887 Training Accuracy: 0.703125\n",
      "Epoch  1, Sentence Batch 80:  Loss:     1.7365 Training Accuracy: 0.812500\n",
      "Epoch  1, Sentence Batch 81:  Loss:     1.7720 Training Accuracy: 0.671875\n",
      "Epoch  1, Sentence Batch 82:  Loss:     1.7129 Training Accuracy: 0.734375\n",
      "Epoch  1, Sentence Batch 83:  Loss:     1.6958 Training Accuracy: 0.765625\n",
      "Epoch  1, Sentence Batch 84:  Loss:     1.6894 Training Accuracy: 0.765625\n",
      "Epoch  1, Sentence Batch 85:  Loss:     1.6960 Training Accuracy: 0.765625\n",
      "Epoch  1, Sentence Batch 86:  Loss:     1.6037 Training Accuracy: 0.859375\n",
      "Epoch  1, Sentence Batch 87:  Loss:     1.7049 Training Accuracy: 0.718750\n",
      "Epoch  1, Sentence Batch 88:  Loss:     1.7009 Training Accuracy: 0.671875\n",
      "Epoch  1, Sentence Batch 89:  Loss:     1.6591 Training Accuracy: 0.765625\n",
      "Epoch  1, Sentence Batch 90:  Loss:     1.6042 Training Accuracy: 0.781250\n",
      "Epoch  1, Sentence Batch 91:  Loss:     1.6386 Training Accuracy: 0.734375\n",
      "Epoch  1, Sentence Batch 92:  Loss:     1.5832 Training Accuracy: 0.718750\n",
      "Epoch  1, Sentence Batch 93:  Loss:     1.4790 Training Accuracy: 0.812500\n",
      "Epoch  1, Sentence Batch 94:  Loss:     1.5803 Training Accuracy: 0.734375\n",
      "Epoch  1, Sentence Batch 95:  Loss:     1.5419 Training Accuracy: 0.750000\n",
      "Epoch  1, Sentence Batch 96:  Loss:     1.6104 Training Accuracy: 0.640625\n",
      "Epoch  1, Sentence Batch 97:  Loss:     1.4455 Training Accuracy: 0.812500\n",
      "Epoch  1, Sentence Batch 98:  Loss:     1.5293 Training Accuracy: 0.671875\n",
      "Epoch  1, Sentence Batch 99:  Loss:     1.4856 Training Accuracy: 0.718750\n",
      "Epoch  1, Sentence Batch 100:  Validation Accuracy: 0.690432\n",
      "Loss:     1.4622 Training Accuracy: 0.765625\n",
      "Epoch  1, Sentence Batch 101:  Loss:     1.5166 Training Accuracy: 0.718750\n",
      "Epoch  1, Sentence Batch 102:  Loss:     1.4271 Training Accuracy: 0.781250\n",
      "Epoch  1, Sentence Batch 103:  Loss:     1.5271 Training Accuracy: 0.625000\n",
      "Epoch  1, Sentence Batch 104:  Loss:     1.4300 Training Accuracy: 0.750000\n",
      "Epoch  1, Sentence Batch 105:  Loss:     1.4447 Training Accuracy: 0.765625\n",
      "Epoch  1, Sentence Batch 106:  Loss:     1.3387 Training Accuracy: 0.859375\n",
      "Epoch  1, Sentence Batch 107:  Loss:     1.3286 Training Accuracy: 0.812500\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  1, Sentence Batch 108:  Loss:     1.4032 Training Accuracy: 0.718750\n",
      "Epoch  1, Sentence Batch 109:  Loss:     1.4064 Training Accuracy: 0.734375\n",
      "Epoch  1, Sentence Batch 110:  Loss:     1.3224 Training Accuracy: 0.859375\n",
      "Epoch  1, Sentence Batch 111:  Loss:     1.2664 Training Accuracy: 0.859375\n",
      "Epoch  1, Sentence Batch 112:  Loss:     1.3216 Training Accuracy: 0.765625\n",
      "Epoch  1, Sentence Batch 113:  Loss:     1.3077 Training Accuracy: 0.781250\n",
      "Epoch  1, Sentence Batch 114:  Loss:     1.3579 Training Accuracy: 0.750000\n",
      "Epoch  1, Sentence Batch 115:  Loss:     1.3314 Training Accuracy: 0.703125\n",
      "Epoch  1, Sentence Batch 116:  Loss:     1.3768 Training Accuracy: 0.656250\n",
      "Epoch  1, Sentence Batch 117:  Loss:     1.2969 Training Accuracy: 0.734375\n",
      "Epoch  1, Sentence Batch 118:  Loss:     1.3322 Training Accuracy: 0.656250\n",
      "Epoch  1, Sentence Batch 119:  Loss:     1.2895 Training Accuracy: 0.718750\n",
      "Epoch  1, Sentence Batch 120:  Loss:     1.2296 Training Accuracy: 0.781250\n",
      "Epoch  1, Sentence Batch 121:  Loss:     1.2500 Training Accuracy: 0.781250\n",
      "Epoch  1, Sentence Batch 122:  Loss:     1.2543 Training Accuracy: 0.750000\n",
      "Epoch  1, Sentence Batch 123:  Loss:     1.2238 Training Accuracy: 0.750000\n",
      "Epoch  1, Sentence Batch 124:  Loss:     1.1626 Training Accuracy: 0.796875\n",
      "Epoch  1, Sentence Batch 125:  Loss:     1.1681 Training Accuracy: 0.843750\n",
      "Epoch  1, Sentence Batch 126:  Loss:     1.2114 Training Accuracy: 0.750000\n",
      "Epoch  1, Sentence Batch 127:  Loss:     1.2215 Training Accuracy: 0.671875\n",
      "Epoch  1, Sentence Batch 128:  Loss:     1.2058 Training Accuracy: 0.734375\n",
      "Epoch  1, Sentence Batch 129:  Loss:     1.1586 Training Accuracy: 0.781250\n",
      "Epoch  1, Sentence Batch 130:  Loss:     1.1682 Training Accuracy: 0.687500\n",
      "Epoch  1, Sentence Batch 131:  Loss:     1.1606 Training Accuracy: 0.765625\n",
      "Epoch  1, Sentence Batch 132:  Loss:     1.1426 Training Accuracy: 0.734375\n",
      "Epoch  1, Sentence Batch 133:  Loss:     1.1261 Training Accuracy: 0.828125\n",
      "Epoch  1, Sentence Batch 134:  Loss:     1.1161 Training Accuracy: 0.734375\n",
      "Epoch  1, Sentence Batch 135:  Loss:     1.1221 Training Accuracy: 0.765625\n",
      "Epoch  1, Sentence Batch 136:  Loss:     1.0738 Training Accuracy: 0.828125\n",
      "Epoch  1, Sentence Batch 137:  Loss:     1.0906 Training Accuracy: 0.750000\n",
      "Epoch  1, Sentence Batch 138:  Loss:     1.0737 Training Accuracy: 0.796875\n",
      "Epoch  1, Sentence Batch 139:  Loss:     1.1175 Training Accuracy: 0.796875\n",
      "Epoch  1, Sentence Batch 140:  Loss:     1.0780 Training Accuracy: 0.828125\n",
      "Epoch  1, Sentence Batch 141:  Loss:     1.0410 Training Accuracy: 0.812500\n",
      "Epoch  1, Sentence Batch 142:  Loss:     1.0651 Training Accuracy: 0.703125\n",
      "Epoch  1, Sentence Batch 143:  Loss:     1.0812 Training Accuracy: 0.750000\n",
      "Epoch  1, Sentence Batch 144:  Loss:     1.0864 Training Accuracy: 0.781250\n",
      "Epoch  1, Sentence Batch 145:  Loss:     1.0398 Training Accuracy: 0.718750\n",
      "Epoch  1, Sentence Batch 146:  Loss:     1.0328 Training Accuracy: 0.750000\n",
      "Epoch  1, Sentence Batch 147:  Loss:     1.0308 Training Accuracy: 0.781250\n",
      "Epoch  1, Sentence Batch 148:  Loss:     1.0984 Training Accuracy: 0.562500\n",
      "Epoch  1, Sentence Batch 149:  Loss:     1.0268 Training Accuracy: 0.783333\n",
      "Epoch  2, Sentence Batch 0:  Loss:     0.9266 Training Accuracy: 0.890625\n",
      "Epoch  2, Sentence Batch 1:  Loss:     0.9059 Training Accuracy: 0.937500\n",
      "Epoch  2, Sentence Batch 2:  Loss:     0.9152 Training Accuracy: 0.890625\n",
      "Epoch  2, Sentence Batch 3:  Loss:     0.9190 Training Accuracy: 0.843750\n",
      "Epoch  2, Sentence Batch 4:  Loss:     0.8952 Training Accuracy: 0.828125\n",
      "Epoch  2, Sentence Batch 5:  Loss:     0.9293 Training Accuracy: 0.781250\n",
      "Epoch  2, Sentence Batch 6:  Loss:     0.8690 Training Accuracy: 0.843750\n",
      "Epoch  2, Sentence Batch 7:  Loss:     0.8650 Training Accuracy: 0.859375\n",
      "Epoch  2, Sentence Batch 8:  Loss:     0.8879 Training Accuracy: 0.828125\n",
      "Epoch  2, Sentence Batch 9:  Loss:     0.9121 Training Accuracy: 0.796875\n",
      "Epoch  2, Sentence Batch 10:  Loss:     0.8959 Training Accuracy: 0.843750\n",
      "Epoch  2, Sentence Batch 11:  Loss:     0.8781 Training Accuracy: 0.828125\n",
      "Epoch  2, Sentence Batch 12:  Loss:     0.9492 Training Accuracy: 0.734375\n",
      "Epoch  2, Sentence Batch 13:  Loss:     0.8491 Training Accuracy: 0.906250\n",
      "Epoch  2, Sentence Batch 14:  Loss:     0.8291 Training Accuracy: 0.890625\n",
      "Epoch  2, Sentence Batch 15:  Loss:     0.8716 Training Accuracy: 0.765625\n",
      "Epoch  2, Sentence Batch 16:  Loss:     0.8430 Training Accuracy: 0.890625\n",
      "Epoch  2, Sentence Batch 17:  Loss:     0.8083 Training Accuracy: 0.921875\n",
      "Epoch  2, Sentence Batch 18:  Loss:     0.8699 Training Accuracy: 0.875000\n",
      "Epoch  2, Sentence Batch 19:  Loss:     0.8633 Training Accuracy: 0.828125\n",
      "Epoch  2, Sentence Batch 20:  Loss:     0.8187 Training Accuracy: 0.906250\n",
      "Epoch  2, Sentence Batch 21:  Loss:     0.8004 Training Accuracy: 0.875000\n",
      "Epoch  2, Sentence Batch 22:  Loss:     0.8159 Training Accuracy: 0.890625\n",
      "Epoch  2, Sentence Batch 23:  Loss:     0.7898 Training Accuracy: 0.890625\n",
      "Epoch  2, Sentence Batch 24:  Loss:     0.7757 Training Accuracy: 0.921875\n",
      "Epoch  2, Sentence Batch 25:  Loss:     0.7979 Training Accuracy: 0.953125\n",
      "Epoch  2, Sentence Batch 26:  Loss:     0.7843 Training Accuracy: 0.859375\n",
      "Epoch  2, Sentence Batch 27:  Loss:     0.7746 Training Accuracy: 0.890625\n",
      "Epoch  2, Sentence Batch 28:  Loss:     0.9097 Training Accuracy: 0.734375\n",
      "Epoch  2, Sentence Batch 29:  Loss:     0.7702 Training Accuracy: 0.875000\n",
      "Epoch  2, Sentence Batch 30:  Loss:     0.7655 Training Accuracy: 0.906250\n",
      "Epoch  2, Sentence Batch 31:  Loss:     0.7948 Training Accuracy: 0.843750\n",
      "Epoch  2, Sentence Batch 32:  Loss:     0.7458 Training Accuracy: 0.937500\n",
      "Epoch  2, Sentence Batch 33:  Loss:     0.7478 Training Accuracy: 0.921875\n",
      "Epoch  2, Sentence Batch 34:  Loss:     0.7420 Training Accuracy: 0.890625\n",
      "Epoch  2, Sentence Batch 35:  Loss:     0.7868 Training Accuracy: 0.875000\n",
      "Epoch  2, Sentence Batch 36:  Loss:     0.7814 Training Accuracy: 0.828125\n",
      "Epoch  2, Sentence Batch 37:  Loss:     0.7375 Training Accuracy: 0.875000\n",
      "Epoch  2, Sentence Batch 38:  Loss:     0.7089 Training Accuracy: 0.875000\n",
      "Epoch  2, Sentence Batch 39:  Loss:     0.7450 Training Accuracy: 0.843750\n",
      "Epoch  2, Sentence Batch 40:  Loss:     0.7057 Training Accuracy: 0.906250\n",
      "Epoch  2, Sentence Batch 41:  Loss:     0.7463 Training Accuracy: 0.875000\n",
      "Epoch  2, Sentence Batch 42:  Loss:     0.7128 Training Accuracy: 0.906250\n",
      "Epoch  2, Sentence Batch 43:  Loss:     0.7069 Training Accuracy: 0.890625\n",
      "Epoch  2, Sentence Batch 44:  Loss:     0.7311 Training Accuracy: 0.906250\n",
      "Epoch  2, Sentence Batch 45:  Loss:     0.7123 Training Accuracy: 0.859375\n",
      "Epoch  2, Sentence Batch 46:  Loss:     0.7437 Training Accuracy: 0.843750\n",
      "Epoch  2, Sentence Batch 47:  Loss:     0.7595 Training Accuracy: 0.859375\n",
      "Epoch  2, Sentence Batch 48:  Loss:     0.7116 Training Accuracy: 0.890625\n",
      "Epoch  2, Sentence Batch 49:  Loss:     0.7161 Training Accuracy: 0.875000\n",
      "Epoch  2, Sentence Batch 50:  Validation Accuracy: 0.742026\n",
      "Loss:     0.7319 Training Accuracy: 0.875000\n",
      "Epoch  2, Sentence Batch 51:  Loss:     0.6808 Training Accuracy: 0.875000\n",
      "Epoch  2, Sentence Batch 52:  Loss:     0.7642 Training Accuracy: 0.828125\n",
      "Epoch  2, Sentence Batch 53:  Loss:     0.7009 Training Accuracy: 0.890625\n",
      "Epoch  2, Sentence Batch 54:  Loss:     0.6980 Training Accuracy: 0.843750\n",
      "Epoch  2, Sentence Batch 55:  Loss:     0.7123 Training Accuracy: 0.828125\n",
      "Epoch  2, Sentence Batch 56:  Loss:     0.6813 Training Accuracy: 0.859375\n",
      "Epoch  2, Sentence Batch 57:  Loss:     0.7361 Training Accuracy: 0.781250\n",
      "Epoch  2, Sentence Batch 58:  Loss:     0.7410 Training Accuracy: 0.812500\n",
      "Epoch  2, Sentence Batch 59:  Loss:     0.7068 Training Accuracy: 0.859375\n",
      "Epoch  2, Sentence Batch 60:  Loss:     0.6669 Training Accuracy: 0.890625\n",
      "Epoch  2, Sentence Batch 61:  Loss:     0.6876 Training Accuracy: 0.859375\n",
      "Epoch  2, Sentence Batch 62:  Loss:     0.7187 Training Accuracy: 0.812500\n",
      "Epoch  2, Sentence Batch 63:  Loss:     0.6736 Training Accuracy: 0.828125\n",
      "Epoch  2, Sentence Batch 64:  Loss:     0.7407 Training Accuracy: 0.796875\n",
      "Epoch  2, Sentence Batch 65:  Loss:     0.6585 Training Accuracy: 0.890625\n",
      "Epoch  2, Sentence Batch 66:  Loss:     0.6890 Training Accuracy: 0.843750\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  2, Sentence Batch 67:  Loss:     0.6261 Training Accuracy: 0.921875\n",
      "Epoch  2, Sentence Batch 68:  Loss:     0.6706 Training Accuracy: 0.796875\n",
      "Epoch  2, Sentence Batch 69:  Loss:     0.6575 Training Accuracy: 0.859375\n",
      "Epoch  2, Sentence Batch 70:  Loss:     0.6323 Training Accuracy: 0.828125\n",
      "Epoch  2, Sentence Batch 71:  Loss:     0.7361 Training Accuracy: 0.703125\n",
      "Epoch  2, Sentence Batch 72:  Loss:     0.6912 Training Accuracy: 0.828125\n",
      "Epoch  2, Sentence Batch 73:  Loss:     0.6399 Training Accuracy: 0.859375\n",
      "Epoch  2, Sentence Batch 74:  Loss:     0.6481 Training Accuracy: 0.812500\n",
      "Epoch  2, Sentence Batch 75:  Loss:     0.6521 Training Accuracy: 0.859375\n",
      "Epoch  2, Sentence Batch 76:  Loss:     0.6570 Training Accuracy: 0.859375\n",
      "Epoch  2, Sentence Batch 77:  Loss:     0.6398 Training Accuracy: 0.843750\n",
      "Epoch  2, Sentence Batch 78:  Loss:     0.6568 Training Accuracy: 0.843750\n",
      "Epoch  2, Sentence Batch 79:  Loss:     0.6114 Training Accuracy: 0.921875\n",
      "Epoch  2, Sentence Batch 80:  Loss:     0.6215 Training Accuracy: 0.890625\n",
      "Epoch  2, Sentence Batch 81:  Loss:     0.6424 Training Accuracy: 0.875000\n",
      "Epoch  2, Sentence Batch 82:  Loss:     0.6618 Training Accuracy: 0.812500\n",
      "Epoch  2, Sentence Batch 83:  Loss:     0.6534 Training Accuracy: 0.812500\n",
      "Epoch  2, Sentence Batch 84:  Loss:     0.6685 Training Accuracy: 0.734375\n",
      "Epoch  2, Sentence Batch 85:  Loss:     0.6868 Training Accuracy: 0.765625\n",
      "Epoch  2, Sentence Batch 86:  Loss:     0.5666 Training Accuracy: 0.921875\n",
      "Epoch  2, Sentence Batch 87:  Loss:     0.6783 Training Accuracy: 0.796875\n",
      "Epoch  2, Sentence Batch 88:  Loss:     0.6810 Training Accuracy: 0.781250\n",
      "Epoch  2, Sentence Batch 89:  Loss:     0.6063 Training Accuracy: 0.890625\n",
      "Epoch  2, Sentence Batch 90:  Loss:     0.6408 Training Accuracy: 0.843750\n",
      "Epoch  2, Sentence Batch 91:  Loss:     0.6326 Training Accuracy: 0.859375\n",
      "Epoch  2, Sentence Batch 92:  Loss:     0.6579 Training Accuracy: 0.781250\n",
      "Epoch  2, Sentence Batch 93:  Loss:     0.5916 Training Accuracy: 0.906250\n",
      "Epoch  2, Sentence Batch 94:  Loss:     0.5962 Training Accuracy: 0.906250\n",
      "Epoch  2, Sentence Batch 95:  Loss:     0.5447 Training Accuracy: 0.890625\n",
      "Epoch  2, Sentence Batch 96:  Loss:     0.5865 Training Accuracy: 0.906250\n",
      "Epoch  2, Sentence Batch 97:  Loss:     0.6104 Training Accuracy: 0.875000\n",
      "Epoch  2, Sentence Batch 98:  Loss:     0.5977 Training Accuracy: 0.875000\n",
      "Epoch  2, Sentence Batch 99:  Loss:     0.6144 Training Accuracy: 0.890625\n",
      "Epoch  2, Sentence Batch 100:  Loss:     0.6228 Training Accuracy: 0.812500\n",
      "Epoch  2, Sentence Batch 101:  Loss:     0.5466 Training Accuracy: 0.921875\n",
      "Epoch  2, Sentence Batch 102:  Loss:     0.5822 Training Accuracy: 0.859375\n",
      "Epoch  2, Sentence Batch 103:  Loss:     0.6038 Training Accuracy: 0.875000\n",
      "Epoch  2, Sentence Batch 104:  Loss:     0.5885 Training Accuracy: 0.843750\n",
      "Epoch  2, Sentence Batch 105:  Loss:     0.5794 Training Accuracy: 0.843750\n",
      "Epoch  2, Sentence Batch 106:  Loss:     0.5921 Training Accuracy: 0.828125\n",
      "Epoch  2, Sentence Batch 107:  Loss:     0.5635 Training Accuracy: 0.890625\n",
      "Epoch  2, Sentence Batch 108:  Loss:     0.5498 Training Accuracy: 0.921875\n",
      "Epoch  2, Sentence Batch 109:  Loss:     0.5826 Training Accuracy: 0.859375\n",
      "Epoch  2, Sentence Batch 110:  Loss:     0.5572 Training Accuracy: 0.890625\n",
      "Epoch  2, Sentence Batch 111:  Loss:     0.5814 Training Accuracy: 0.890625\n",
      "Epoch  2, Sentence Batch 112:  Loss:     0.5466 Training Accuracy: 0.859375\n",
      "Epoch  2, Sentence Batch 113:  Loss:     0.6515 Training Accuracy: 0.718750\n",
      "Epoch  2, Sentence Batch 114:  Loss:     0.6220 Training Accuracy: 0.796875\n",
      "Epoch  2, Sentence Batch 115:  Loss:     0.5859 Training Accuracy: 0.875000\n",
      "Epoch  2, Sentence Batch 116:  Loss:     0.5485 Training Accuracy: 0.890625\n",
      "Epoch  2, Sentence Batch 117:  Loss:     0.5724 Training Accuracy: 0.906250\n",
      "Epoch  2, Sentence Batch 118:  Loss:     0.5588 Training Accuracy: 0.890625\n",
      "Epoch  2, Sentence Batch 119:  Loss:     0.5912 Training Accuracy: 0.796875\n",
      "Epoch  2, Sentence Batch 120:  Loss:     0.5417 Training Accuracy: 0.921875\n",
      "Epoch  2, Sentence Batch 121:  Loss:     0.6052 Training Accuracy: 0.875000\n",
      "Epoch  2, Sentence Batch 122:  Loss:     0.5434 Training Accuracy: 0.859375\n",
      "Epoch  2, Sentence Batch 123:  Loss:     0.5964 Training Accuracy: 0.781250\n",
      "Epoch  2, Sentence Batch 124:  Loss:     0.6775 Training Accuracy: 0.718750\n",
      "Epoch  2, Sentence Batch 125:  Loss:     0.5582 Training Accuracy: 0.828125\n",
      "Epoch  2, Sentence Batch 126:  Loss:     0.5677 Training Accuracy: 0.796875\n",
      "Epoch  2, Sentence Batch 127:  Loss:     0.6089 Training Accuracy: 0.828125\n",
      "Epoch  2, Sentence Batch 128:  Loss:     0.5411 Training Accuracy: 0.859375\n",
      "Epoch  2, Sentence Batch 129:  Loss:     0.4999 Training Accuracy: 0.906250\n",
      "Epoch  2, Sentence Batch 130:  Loss:     0.5710 Training Accuracy: 0.828125\n",
      "Epoch  2, Sentence Batch 131:  Loss:     0.5515 Training Accuracy: 0.859375\n",
      "Epoch  2, Sentence Batch 132:  Loss:     0.5693 Training Accuracy: 0.828125\n",
      "Epoch  2, Sentence Batch 133:  Loss:     0.5290 Training Accuracy: 0.843750\n",
      "Epoch  2, Sentence Batch 134:  Loss:     0.5292 Training Accuracy: 0.890625\n",
      "Epoch  2, Sentence Batch 135:  Loss:     0.5603 Training Accuracy: 0.859375\n",
      "Epoch  2, Sentence Batch 136:  Loss:     0.6175 Training Accuracy: 0.796875\n",
      "Epoch  2, Sentence Batch 137:  Loss:     0.5236 Training Accuracy: 0.921875\n",
      "Epoch  2, Sentence Batch 138:  Loss:     0.5671 Training Accuracy: 0.843750\n",
      "Epoch  2, Sentence Batch 139:  Loss:     0.5427 Training Accuracy: 0.859375\n",
      "Epoch  2, Sentence Batch 140:  Loss:     0.5609 Training Accuracy: 0.812500\n",
      "Epoch  2, Sentence Batch 141:  Loss:     0.5711 Training Accuracy: 0.828125\n",
      "Epoch  2, Sentence Batch 142:  Loss:     0.5362 Training Accuracy: 0.859375\n",
      "Epoch  2, Sentence Batch 143:  Loss:     0.5864 Training Accuracy: 0.828125\n",
      "Epoch  2, Sentence Batch 144:  Loss:     0.5323 Training Accuracy: 0.859375\n",
      "Epoch  2, Sentence Batch 145:  Loss:     0.5223 Training Accuracy: 0.828125\n",
      "Epoch  2, Sentence Batch 146:  Loss:     0.5360 Training Accuracy: 0.828125\n",
      "Epoch  2, Sentence Batch 147:  Loss:     0.5543 Training Accuracy: 0.812500\n",
      "Epoch  2, Sentence Batch 148:  Loss:     0.5506 Training Accuracy: 0.828125\n",
      "Epoch  2, Sentence Batch 149:  Loss:     0.5623 Training Accuracy: 0.833333\n",
      "Epoch  3, Sentence Batch 0:  Validation Accuracy: 0.762664\n",
      "Loss:     0.5101 Training Accuracy: 0.875000\n",
      "Epoch  3, Sentence Batch 1:  Loss:     0.4732 Training Accuracy: 0.890625\n",
      "Epoch  3, Sentence Batch 2:  Loss:     0.5033 Training Accuracy: 0.890625\n",
      "Epoch  3, Sentence Batch 3:  Loss:     0.4733 Training Accuracy: 0.875000\n",
      "Epoch  3, Sentence Batch 4:  Loss:     0.5386 Training Accuracy: 0.796875\n",
      "Epoch  3, Sentence Batch 5:  Loss:     0.5129 Training Accuracy: 0.875000\n",
      "Epoch  3, Sentence Batch 6:  Loss:     0.4790 Training Accuracy: 0.937500\n",
      "Epoch  3, Sentence Batch 7:  Loss:     0.4477 Training Accuracy: 0.875000\n",
      "Epoch  3, Sentence Batch 8:  Loss:     0.4881 Training Accuracy: 0.921875\n",
      "Epoch  3, Sentence Batch 9:  Loss:     0.4832 Training Accuracy: 0.890625\n",
      "Epoch  3, Sentence Batch 10:  Loss:     0.4697 Training Accuracy: 0.921875\n",
      "Epoch  3, Sentence Batch 11:  Loss:     0.5392 Training Accuracy: 0.812500\n",
      "Epoch  3, Sentence Batch 12:  Loss:     0.4800 Training Accuracy: 0.890625\n",
      "Epoch  3, Sentence Batch 13:  Loss:     0.5208 Training Accuracy: 0.843750\n",
      "Epoch  3, Sentence Batch 14:  Loss:     0.4677 Training Accuracy: 0.890625\n",
      "Epoch  3, Sentence Batch 15:  Loss:     0.4727 Training Accuracy: 0.875000\n",
      "Epoch  3, Sentence Batch 16:  Loss:     0.4795 Training Accuracy: 0.906250\n",
      "Epoch  3, Sentence Batch 17:  Loss:     0.4654 Training Accuracy: 0.843750\n",
      "Epoch  3, Sentence Batch 18:  Loss:     0.4786 Training Accuracy: 0.859375\n",
      "Epoch  3, Sentence Batch 19:  Loss:     0.4570 Training Accuracy: 0.906250\n",
      "Epoch  3, Sentence Batch 20:  Loss:     0.4832 Training Accuracy: 0.890625\n",
      "Epoch  3, Sentence Batch 21:  Loss:     0.5026 Training Accuracy: 0.906250\n",
      "Epoch  3, Sentence Batch 22:  Loss:     0.4894 Training Accuracy: 0.875000\n",
      "Epoch  3, Sentence Batch 23:  Loss:     0.4749 Training Accuracy: 0.875000\n",
      "Epoch  3, Sentence Batch 24:  Loss:     0.5170 Training Accuracy: 0.828125\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  3, Sentence Batch 25:  Loss:     0.4421 Training Accuracy: 0.921875\n",
      "Epoch  3, Sentence Batch 26:  Loss:     0.4525 Training Accuracy: 0.890625\n",
      "Epoch  3, Sentence Batch 27:  Loss:     0.4776 Training Accuracy: 0.875000\n",
      "Epoch  3, Sentence Batch 28:  Loss:     0.5182 Training Accuracy: 0.812500\n",
      "Epoch  3, Sentence Batch 29:  Loss:     0.4744 Training Accuracy: 0.906250\n",
      "Epoch  3, Sentence Batch 30:  Loss:     0.4040 Training Accuracy: 0.921875\n",
      "Epoch  3, Sentence Batch 31:  Loss:     0.4121 Training Accuracy: 0.921875\n",
      "Epoch  3, Sentence Batch 32:  Loss:     0.4544 Training Accuracy: 0.906250\n",
      "Epoch  3, Sentence Batch 33:  Loss:     0.4625 Training Accuracy: 0.875000\n",
      "Epoch  3, Sentence Batch 34:  Loss:     0.4276 Training Accuracy: 0.953125\n",
      "Epoch  3, Sentence Batch 35:  Loss:     0.4615 Training Accuracy: 0.890625\n",
      "Epoch  3, Sentence Batch 36:  Loss:     0.4598 Training Accuracy: 0.906250\n",
      "Epoch  3, Sentence Batch 37:  Loss:     0.4002 Training Accuracy: 0.921875\n",
      "Epoch  3, Sentence Batch 38:  Loss:     0.4191 Training Accuracy: 0.921875\n",
      "Epoch  3, Sentence Batch 39:  Loss:     0.4110 Training Accuracy: 0.937500\n",
      "Epoch  3, Sentence Batch 40:  Loss:     0.4293 Training Accuracy: 0.937500\n",
      "Epoch  3, Sentence Batch 41:  Loss:     0.4264 Training Accuracy: 0.843750\n",
      "Epoch  3, Sentence Batch 42:  Loss:     0.4637 Training Accuracy: 0.906250\n",
      "Epoch  3, Sentence Batch 43:  Loss:     0.4911 Training Accuracy: 0.859375\n",
      "Epoch  3, Sentence Batch 44:  Loss:     0.4616 Training Accuracy: 0.843750\n",
      "Epoch  3, Sentence Batch 45:  Loss:     0.4631 Training Accuracy: 0.890625\n",
      "Epoch  3, Sentence Batch 46:  Loss:     0.4969 Training Accuracy: 0.828125\n",
      "Epoch  3, Sentence Batch 47:  Loss:     0.5261 Training Accuracy: 0.796875\n",
      "Epoch  3, Sentence Batch 48:  Loss:     0.5166 Training Accuracy: 0.812500\n",
      "Epoch  3, Sentence Batch 49:  Loss:     0.4782 Training Accuracy: 0.875000\n",
      "Epoch  3, Sentence Batch 50:  Loss:     0.4290 Training Accuracy: 0.937500\n",
      "Epoch  3, Sentence Batch 51:  Loss:     0.4159 Training Accuracy: 0.906250\n",
      "Epoch  3, Sentence Batch 52:  Loss:     0.4622 Training Accuracy: 0.843750\n",
      "Epoch  3, Sentence Batch 53:  Loss:     0.4836 Training Accuracy: 0.843750\n",
      "Epoch  3, Sentence Batch 54:  Loss:     0.4792 Training Accuracy: 0.812500\n",
      "Epoch  3, Sentence Batch 55:  Loss:     0.4153 Training Accuracy: 0.906250\n",
      "Epoch  3, Sentence Batch 56:  Loss:     0.3791 Training Accuracy: 0.984375\n",
      "Epoch  3, Sentence Batch 57:  Loss:     0.4864 Training Accuracy: 0.812500\n",
      "Epoch  3, Sentence Batch 58:  Loss:     0.4565 Training Accuracy: 0.843750\n",
      "Epoch  3, Sentence Batch 59:  Loss:     0.4172 Training Accuracy: 0.859375\n",
      "Epoch  3, Sentence Batch 60:  Loss:     0.4440 Training Accuracy: 0.890625\n",
      "Epoch  3, Sentence Batch 61:  Loss:     0.4455 Training Accuracy: 0.875000\n",
      "Epoch  3, Sentence Batch 62:  Loss:     0.4583 Training Accuracy: 0.859375\n",
      "Epoch  3, Sentence Batch 63:  Loss:     0.3936 Training Accuracy: 0.890625\n",
      "Epoch  3, Sentence Batch 64:  Loss:     0.4454 Training Accuracy: 0.890625\n",
      "Epoch  3, Sentence Batch 65:  Loss:     0.4494 Training Accuracy: 0.859375\n",
      "Epoch  3, Sentence Batch 66:  Loss:     0.4013 Training Accuracy: 0.953125\n",
      "Epoch  3, Sentence Batch 67:  Loss:     0.4156 Training Accuracy: 0.937500\n",
      "Epoch  3, Sentence Batch 68:  Loss:     0.4349 Training Accuracy: 0.937500\n",
      "Epoch  3, Sentence Batch 69:  Loss:     0.5124 Training Accuracy: 0.812500\n",
      "Epoch  3, Sentence Batch 70:  Loss:     0.3980 Training Accuracy: 0.906250\n",
      "Epoch  3, Sentence Batch 71:  Loss:     0.5077 Training Accuracy: 0.781250\n",
      "Epoch  3, Sentence Batch 72:  Loss:     0.5144 Training Accuracy: 0.859375\n",
      "Epoch  3, Sentence Batch 73:  Loss:     0.4871 Training Accuracy: 0.859375\n",
      "Epoch  3, Sentence Batch 74:  Loss:     0.4371 Training Accuracy: 0.890625\n",
      "Epoch  3, Sentence Batch 75:  Loss:     0.4217 Training Accuracy: 0.890625\n",
      "Epoch  3, Sentence Batch 76:  Loss:     0.4528 Training Accuracy: 0.843750\n",
      "Epoch  3, Sentence Batch 77:  Loss:     0.4619 Training Accuracy: 0.890625\n",
      "Epoch  3, Sentence Batch 78:  Loss:     0.4376 Training Accuracy: 0.890625\n",
      "Epoch  3, Sentence Batch 79:  Loss:     0.4350 Training Accuracy: 0.875000\n",
      "Epoch  3, Sentence Batch 80:  Loss:     0.4744 Training Accuracy: 0.843750\n",
      "Epoch  3, Sentence Batch 81:  Loss:     0.4129 Training Accuracy: 0.890625\n",
      "Epoch  3, Sentence Batch 82:  Loss:     0.4529 Training Accuracy: 0.875000\n",
      "Epoch  3, Sentence Batch 83:  Loss:     0.4205 Training Accuracy: 0.890625\n",
      "Epoch  3, Sentence Batch 84:  Loss:     0.4476 Training Accuracy: 0.875000\n",
      "Epoch  3, Sentence Batch 85:  Loss:     0.4869 Training Accuracy: 0.812500\n",
      "Epoch  3, Sentence Batch 86:  Loss:     0.4244 Training Accuracy: 0.906250\n",
      "Epoch  3, Sentence Batch 87:  Loss:     0.4445 Training Accuracy: 0.890625\n",
      "Epoch  3, Sentence Batch 88:  Loss:     0.3572 Training Accuracy: 0.937500\n",
      "Epoch  3, Sentence Batch 89:  Loss:     0.4187 Training Accuracy: 0.906250\n",
      "Epoch  3, Sentence Batch 90:  Loss:     0.3914 Training Accuracy: 0.921875\n",
      "Epoch  3, Sentence Batch 91:  Loss:     0.3905 Training Accuracy: 0.937500\n",
      "Epoch  3, Sentence Batch 92:  Loss:     0.4171 Training Accuracy: 0.906250\n",
      "Epoch  3, Sentence Batch 93:  Loss:     0.3998 Training Accuracy: 0.937500\n",
      "Epoch  3, Sentence Batch 94:  Loss:     0.4377 Training Accuracy: 0.859375\n",
      "Epoch  3, Sentence Batch 95:  Loss:     0.4105 Training Accuracy: 0.875000\n",
      "Epoch  3, Sentence Batch 96:  Loss:     0.4233 Training Accuracy: 0.890625\n",
      "Epoch  3, Sentence Batch 97:  Loss:     0.4362 Training Accuracy: 0.875000\n",
      "Epoch  3, Sentence Batch 98:  Loss:     0.4558 Training Accuracy: 0.812500\n",
      "Epoch  3, Sentence Batch 99:  Loss:     0.4317 Training Accuracy: 0.921875\n",
      "Epoch  3, Sentence Batch 100:  Validation Accuracy: 0.773921\n",
      "Loss:     0.4085 Training Accuracy: 0.906250\n",
      "Epoch  3, Sentence Batch 101:  Loss:     0.4389 Training Accuracy: 0.859375\n",
      "Epoch  3, Sentence Batch 102:  Loss:     0.3936 Training Accuracy: 0.921875\n",
      "Epoch  3, Sentence Batch 103:  Loss:     0.4439 Training Accuracy: 0.843750\n",
      "Epoch  3, Sentence Batch 104:  Loss:     0.4399 Training Accuracy: 0.875000\n",
      "Epoch  3, Sentence Batch 105:  Loss:     0.3944 Training Accuracy: 0.921875\n",
      "Epoch  3, Sentence Batch 106:  Loss:     0.4835 Training Accuracy: 0.828125\n",
      "Epoch  3, Sentence Batch 107:  Loss:     0.4188 Training Accuracy: 0.875000\n",
      "Epoch  3, Sentence Batch 108:  Loss:     0.4565 Training Accuracy: 0.843750\n",
      "Epoch  3, Sentence Batch 109:  Loss:     0.5035 Training Accuracy: 0.812500\n",
      "Epoch  3, Sentence Batch 110:  Loss:     0.4085 Training Accuracy: 0.890625\n",
      "Epoch  3, Sentence Batch 111:  Loss:     0.4392 Training Accuracy: 0.890625\n",
      "Epoch  3, Sentence Batch 112:  Loss:     0.4488 Training Accuracy: 0.890625\n",
      "Epoch  3, Sentence Batch 113:  Loss:     0.4412 Training Accuracy: 0.859375\n",
      "Epoch  3, Sentence Batch 114:  Loss:     0.4497 Training Accuracy: 0.843750\n",
      "Epoch  3, Sentence Batch 115:  Loss:     0.4204 Training Accuracy: 0.875000\n",
      "Epoch  3, Sentence Batch 116:  Loss:     0.4927 Training Accuracy: 0.812500\n",
      "Epoch  3, Sentence Batch 117:  Loss:     0.4253 Training Accuracy: 0.906250\n",
      "Epoch  3, Sentence Batch 118:  Loss:     0.4471 Training Accuracy: 0.906250\n",
      "Epoch  3, Sentence Batch 119:  Loss:     0.4369 Training Accuracy: 0.859375\n",
      "Epoch  3, Sentence Batch 120:  Loss:     0.3669 Training Accuracy: 0.921875\n",
      "Epoch  3, Sentence Batch 121:  Loss:     0.4195 Training Accuracy: 0.906250\n",
      "Epoch  3, Sentence Batch 122:  Loss:     0.4335 Training Accuracy: 0.875000\n",
      "Epoch  3, Sentence Batch 123:  Loss:     0.3844 Training Accuracy: 0.906250\n",
      "Epoch  3, Sentence Batch 124:  Loss:     0.4105 Training Accuracy: 0.890625\n",
      "Epoch  3, Sentence Batch 125:  Loss:     0.4024 Training Accuracy: 0.906250\n",
      "Epoch  3, Sentence Batch 126:  Loss:     0.5458 Training Accuracy: 0.796875\n",
      "Epoch  3, Sentence Batch 127:  Loss:     0.4162 Training Accuracy: 0.921875\n",
      "Epoch  3, Sentence Batch 128:  Loss:     0.4358 Training Accuracy: 0.859375\n",
      "Epoch  3, Sentence Batch 129:  Loss:     0.4494 Training Accuracy: 0.890625\n",
      "Epoch  3, Sentence Batch 130:  Loss:     0.3899 Training Accuracy: 0.953125\n",
      "Epoch  3, Sentence Batch 131:  Loss:     0.4119 Training Accuracy: 0.921875\n",
      "Epoch  3, Sentence Batch 132:  Loss:     0.4689 Training Accuracy: 0.796875\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  3, Sentence Batch 133:  Loss:     0.3797 Training Accuracy: 0.906250\n",
      "Epoch  3, Sentence Batch 134:  Loss:     0.3498 Training Accuracy: 0.937500\n",
      "Epoch  3, Sentence Batch 135:  Loss:     0.5191 Training Accuracy: 0.812500\n",
      "Epoch  3, Sentence Batch 136:  Loss:     0.4535 Training Accuracy: 0.875000\n",
      "Epoch  3, Sentence Batch 137:  Loss:     0.4481 Training Accuracy: 0.859375\n",
      "Epoch  3, Sentence Batch 138:  Loss:     0.4152 Training Accuracy: 0.937500\n",
      "Epoch  3, Sentence Batch 139:  Loss:     0.4339 Training Accuracy: 0.828125\n",
      "Epoch  3, Sentence Batch 140:  Loss:     0.3362 Training Accuracy: 0.953125\n",
      "Epoch  3, Sentence Batch 141:  Loss:     0.3682 Training Accuracy: 0.937500\n",
      "Epoch  3, Sentence Batch 142:  Loss:     0.3363 Training Accuracy: 0.921875\n",
      "Epoch  3, Sentence Batch 143:  Loss:     0.4850 Training Accuracy: 0.828125\n",
      "Epoch  3, Sentence Batch 144:  Loss:     0.4391 Training Accuracy: 0.875000\n",
      "Epoch  3, Sentence Batch 145:  Loss:     0.3821 Training Accuracy: 0.875000\n",
      "Epoch  3, Sentence Batch 146:  Loss:     0.4154 Training Accuracy: 0.875000\n",
      "Epoch  3, Sentence Batch 147:  Loss:     0.5089 Training Accuracy: 0.765625\n",
      "Epoch  3, Sentence Batch 148:  Loss:     0.4051 Training Accuracy: 0.890625\n",
      "Epoch  3, Sentence Batch 149:  Loss:     0.3582 Training Accuracy: 0.900000\n",
      "Epoch  4, Sentence Batch 0:  Loss:     0.3602 Training Accuracy: 0.921875\n",
      "Epoch  4, Sentence Batch 1:  Loss:     0.3479 Training Accuracy: 0.921875\n",
      "Epoch  4, Sentence Batch 2:  Loss:     0.3521 Training Accuracy: 0.953125\n",
      "Epoch  4, Sentence Batch 3:  Loss:     0.3972 Training Accuracy: 0.890625\n",
      "Epoch  4, Sentence Batch 4:  Loss:     0.3437 Training Accuracy: 0.937500\n",
      "Epoch  4, Sentence Batch 5:  Loss:     0.3459 Training Accuracy: 0.968750\n",
      "Epoch  4, Sentence Batch 6:  Loss:     0.3495 Training Accuracy: 0.921875\n",
      "Epoch  4, Sentence Batch 7:  Loss:     0.3851 Training Accuracy: 0.890625\n",
      "Epoch  4, Sentence Batch 8:  Loss:     0.3820 Training Accuracy: 0.921875\n",
      "Epoch  4, Sentence Batch 9:  Loss:     0.3130 Training Accuracy: 0.984375\n",
      "Epoch  4, Sentence Batch 10:  Loss:     0.4050 Training Accuracy: 0.859375\n",
      "Epoch  4, Sentence Batch 11:  Loss:     0.3514 Training Accuracy: 0.875000\n",
      "Epoch  4, Sentence Batch 12:  Loss:     0.3435 Training Accuracy: 0.953125\n",
      "Epoch  4, Sentence Batch 13:  Loss:     0.3328 Training Accuracy: 0.906250\n",
      "Epoch  4, Sentence Batch 14:  Loss:     0.3022 Training Accuracy: 0.984375\n",
      "Epoch  4, Sentence Batch 15:  Loss:     0.3797 Training Accuracy: 0.921875\n",
      "Epoch  4, Sentence Batch 16:  Loss:     0.3709 Training Accuracy: 0.890625\n",
      "Epoch  4, Sentence Batch 17:  Loss:     0.3675 Training Accuracy: 0.875000\n",
      "Epoch  4, Sentence Batch 18:  Loss:     0.4343 Training Accuracy: 0.875000\n",
      "Epoch  4, Sentence Batch 19:  Loss:     0.3640 Training Accuracy: 0.906250\n",
      "Epoch  4, Sentence Batch 20:  Loss:     0.3627 Training Accuracy: 0.906250\n",
      "Epoch  4, Sentence Batch 21:  Loss:     0.4208 Training Accuracy: 0.890625\n",
      "Epoch  4, Sentence Batch 22:  Loss:     0.3580 Training Accuracy: 0.953125\n",
      "Epoch  4, Sentence Batch 23:  Loss:     0.4147 Training Accuracy: 0.875000\n",
      "Epoch  4, Sentence Batch 24:  Loss:     0.4111 Training Accuracy: 0.875000\n",
      "Epoch  4, Sentence Batch 25:  Loss:     0.3535 Training Accuracy: 0.937500\n",
      "Epoch  4, Sentence Batch 26:  Loss:     0.3907 Training Accuracy: 0.890625\n",
      "Epoch  4, Sentence Batch 27:  Loss:     0.3048 Training Accuracy: 0.984375\n",
      "Epoch  4, Sentence Batch 28:  Loss:     0.3194 Training Accuracy: 0.937500\n",
      "Epoch  4, Sentence Batch 29:  Loss:     0.3314 Training Accuracy: 0.953125\n",
      "Epoch  4, Sentence Batch 30:  Loss:     0.3858 Training Accuracy: 0.906250\n",
      "Epoch  4, Sentence Batch 31:  Loss:     0.3336 Training Accuracy: 0.968750\n",
      "Epoch  4, Sentence Batch 32:  Loss:     0.3916 Training Accuracy: 0.906250\n",
      "Epoch  4, Sentence Batch 33:  Loss:     0.3486 Training Accuracy: 0.906250\n",
      "Epoch  4, Sentence Batch 34:  Loss:     0.3064 Training Accuracy: 0.953125\n",
      "Epoch  4, Sentence Batch 35:  Loss:     0.3332 Training Accuracy: 0.921875\n",
      "Epoch  4, Sentence Batch 36:  Loss:     0.3378 Training Accuracy: 0.937500\n",
      "Epoch  4, Sentence Batch 37:  Loss:     0.3648 Training Accuracy: 0.968750\n",
      "Epoch  4, Sentence Batch 38:  Loss:     0.3174 Training Accuracy: 0.921875\n",
      "Epoch  4, Sentence Batch 39:  Loss:     0.3317 Training Accuracy: 0.937500\n",
      "Epoch  4, Sentence Batch 40:  Loss:     0.3500 Training Accuracy: 0.921875\n",
      "Epoch  4, Sentence Batch 41:  Loss:     0.3615 Training Accuracy: 0.906250\n",
      "Epoch  4, Sentence Batch 42:  Loss:     0.4467 Training Accuracy: 0.828125\n",
      "Epoch  4, Sentence Batch 43:  Loss:     0.3626 Training Accuracy: 0.937500\n",
      "Epoch  4, Sentence Batch 44:  Loss:     0.4175 Training Accuracy: 0.859375\n",
      "Epoch  4, Sentence Batch 45:  Loss:     0.3081 Training Accuracy: 0.968750\n",
      "Epoch  4, Sentence Batch 46:  Loss:     0.3034 Training Accuracy: 0.968750\n",
      "Epoch  4, Sentence Batch 47:  Loss:     0.3338 Training Accuracy: 0.890625\n",
      "Epoch  4, Sentence Batch 48:  Loss:     0.4040 Training Accuracy: 0.875000\n",
      "Epoch  4, Sentence Batch 49:  Loss:     0.3136 Training Accuracy: 0.921875\n",
      "Epoch  4, Sentence Batch 50:  Validation Accuracy: 0.773921\n",
      "Loss:     0.3233 Training Accuracy: 0.953125\n",
      "Epoch  4, Sentence Batch 51:  Loss:     0.3059 Training Accuracy: 0.953125\n",
      "Epoch  4, Sentence Batch 52:  Loss:     0.4113 Training Accuracy: 0.875000\n",
      "Epoch  4, Sentence Batch 53:  Loss:     0.3434 Training Accuracy: 0.953125\n",
      "Epoch  4, Sentence Batch 54:  Loss:     0.2915 Training Accuracy: 0.953125\n",
      "Epoch  4, Sentence Batch 55:  Loss:     0.2924 Training Accuracy: 0.968750\n",
      "Epoch  4, Sentence Batch 56:  Loss:     0.3076 Training Accuracy: 0.953125\n",
      "Epoch  4, Sentence Batch 57:  Loss:     0.3294 Training Accuracy: 0.937500\n",
      "Epoch  4, Sentence Batch 58:  Loss:     0.3581 Training Accuracy: 0.953125\n",
      "Epoch  4, Sentence Batch 59:  Loss:     0.2985 Training Accuracy: 0.921875\n",
      "Epoch  4, Sentence Batch 60:  Loss:     0.2989 Training Accuracy: 0.937500\n",
      "Epoch  4, Sentence Batch 61:  Loss:     0.3081 Training Accuracy: 0.953125\n",
      "Epoch  4, Sentence Batch 62:  Loss:     0.3525 Training Accuracy: 0.921875\n",
      "Epoch  4, Sentence Batch 63:  Loss:     0.3697 Training Accuracy: 0.953125\n",
      "Epoch  4, Sentence Batch 64:  Loss:     0.3423 Training Accuracy: 0.921875\n",
      "Epoch  4, Sentence Batch 65:  Loss:     0.3813 Training Accuracy: 0.921875\n",
      "Epoch  4, Sentence Batch 66:  Loss:     0.2910 Training Accuracy: 0.968750\n",
      "Epoch  4, Sentence Batch 67:  Loss:     0.3064 Training Accuracy: 0.921875\n",
      "Epoch  4, Sentence Batch 68:  Loss:     0.2846 Training Accuracy: 0.921875\n",
      "Epoch  4, Sentence Batch 69:  Loss:     0.3275 Training Accuracy: 0.953125\n",
      "Epoch  4, Sentence Batch 70:  Loss:     0.3264 Training Accuracy: 0.968750\n",
      "Epoch  4, Sentence Batch 71:  Loss:     0.2967 Training Accuracy: 0.937500\n",
      "Epoch  4, Sentence Batch 72:  Loss:     0.3160 Training Accuracy: 0.937500\n",
      "Epoch  4, Sentence Batch 73:  Loss:     0.3598 Training Accuracy: 0.875000\n",
      "Epoch  4, Sentence Batch 74:  Loss:     0.3502 Training Accuracy: 0.890625\n",
      "Epoch  4, Sentence Batch 75:  Loss:     0.3720 Training Accuracy: 0.859375\n",
      "Epoch  4, Sentence Batch 76:  Loss:     0.3255 Training Accuracy: 0.953125\n",
      "Epoch  4, Sentence Batch 77:  Loss:     0.3407 Training Accuracy: 0.875000\n",
      "Epoch  4, Sentence Batch 78:  Loss:     0.3139 Training Accuracy: 0.921875\n",
      "Epoch  4, Sentence Batch 79:  Loss:     0.3139 Training Accuracy: 0.937500\n",
      "Epoch  4, Sentence Batch 80:  Loss:     0.3946 Training Accuracy: 0.875000\n",
      "Epoch  4, Sentence Batch 81:  Loss:     0.3361 Training Accuracy: 0.937500\n",
      "Epoch  4, Sentence Batch 82:  Loss:     0.4265 Training Accuracy: 0.812500\n",
      "Epoch  4, Sentence Batch 83:  Loss:     0.3056 Training Accuracy: 0.937500\n",
      "Epoch  4, Sentence Batch 84:  Loss:     0.4050 Training Accuracy: 0.875000\n",
      "Epoch  4, Sentence Batch 85:  Loss:     0.3401 Training Accuracy: 0.937500\n",
      "Epoch  4, Sentence Batch 86:  Loss:     0.3539 Training Accuracy: 0.906250\n",
      "Epoch  4, Sentence Batch 87:  Loss:     0.3495 Training Accuracy: 0.890625\n",
      "Epoch  4, Sentence Batch 88:  Loss:     0.3804 Training Accuracy: 0.906250\n",
      "Epoch  4, Sentence Batch 89:  Loss:     0.3004 Training Accuracy: 0.890625\n",
      "Epoch  4, Sentence Batch 90:  Loss:     0.3013 Training Accuracy: 0.953125\n",
      "Epoch  4, Sentence Batch 91:  Loss:     0.3674 Training Accuracy: 0.890625\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  4, Sentence Batch 92:  Loss:     0.3290 Training Accuracy: 0.937500\n",
      "Epoch  4, Sentence Batch 93:  Loss:     0.3963 Training Accuracy: 0.859375\n",
      "Epoch  4, Sentence Batch 94:  Loss:     0.3155 Training Accuracy: 0.953125\n",
      "Epoch  4, Sentence Batch 95:  Loss:     0.3390 Training Accuracy: 0.921875\n",
      "Epoch  4, Sentence Batch 96:  Loss:     0.3351 Training Accuracy: 0.953125\n",
      "Epoch  4, Sentence Batch 97:  Loss:     0.3046 Training Accuracy: 0.953125\n",
      "Epoch  4, Sentence Batch 98:  Loss:     0.3357 Training Accuracy: 0.953125\n",
      "Epoch  4, Sentence Batch 99:  Loss:     0.3069 Training Accuracy: 0.921875\n",
      "Epoch  4, Sentence Batch 100:  Loss:     0.3475 Training Accuracy: 0.906250\n",
      "Epoch  4, Sentence Batch 101:  Loss:     0.3221 Training Accuracy: 0.906250\n",
      "Epoch  4, Sentence Batch 102:  Loss:     0.3310 Training Accuracy: 0.968750\n",
      "Epoch  4, Sentence Batch 103:  Loss:     0.3567 Training Accuracy: 0.921875\n",
      "Epoch  4, Sentence Batch 104:  Loss:     0.3611 Training Accuracy: 0.906250\n",
      "Epoch  4, Sentence Batch 105:  Loss:     0.3328 Training Accuracy: 0.921875\n",
      "Epoch  4, Sentence Batch 106:  Loss:     0.3661 Training Accuracy: 0.890625\n",
      "Epoch  4, Sentence Batch 107:  Loss:     0.3341 Training Accuracy: 0.890625\n",
      "Epoch  4, Sentence Batch 108:  Loss:     0.3984 Training Accuracy: 0.828125\n",
      "Epoch  4, Sentence Batch 109:  Loss:     0.3961 Training Accuracy: 0.875000\n",
      "Epoch  4, Sentence Batch 110:  Loss:     0.3331 Training Accuracy: 0.921875\n",
      "Epoch  4, Sentence Batch 111:  Loss:     0.3703 Training Accuracy: 0.875000\n",
      "Epoch  4, Sentence Batch 112:  Loss:     0.4175 Training Accuracy: 0.828125\n",
      "Epoch  4, Sentence Batch 113:  Loss:     0.3595 Training Accuracy: 0.859375\n",
      "Epoch  4, Sentence Batch 114:  Loss:     0.3464 Training Accuracy: 0.921875\n",
      "Epoch  4, Sentence Batch 115:  Loss:     0.3813 Training Accuracy: 0.875000\n",
      "Epoch  4, Sentence Batch 116:  Loss:     0.3154 Training Accuracy: 0.890625\n",
      "Epoch  4, Sentence Batch 117:  Loss:     0.3562 Training Accuracy: 0.859375\n",
      "Epoch  4, Sentence Batch 118:  Loss:     0.3553 Training Accuracy: 0.921875\n",
      "Epoch  4, Sentence Batch 119:  Loss:     0.3240 Training Accuracy: 0.937500\n",
      "Epoch  4, Sentence Batch 120:  Loss:     0.3330 Training Accuracy: 0.890625\n",
      "Epoch  4, Sentence Batch 121:  Loss:     0.2867 Training Accuracy: 0.937500\n",
      "Epoch  4, Sentence Batch 122:  Loss:     0.2794 Training Accuracy: 0.937500\n",
      "Epoch  4, Sentence Batch 123:  Loss:     0.4519 Training Accuracy: 0.843750\n",
      "Epoch  4, Sentence Batch 124:  Loss:     0.3115 Training Accuracy: 0.921875\n",
      "Epoch  4, Sentence Batch 125:  Loss:     0.2924 Training Accuracy: 0.921875\n",
      "Epoch  4, Sentence Batch 126:  Loss:     0.3726 Training Accuracy: 0.937500\n",
      "Epoch  4, Sentence Batch 127:  Loss:     0.3329 Training Accuracy: 0.937500\n",
      "Epoch  4, Sentence Batch 128:  Loss:     0.3416 Training Accuracy: 0.921875\n",
      "Epoch  4, Sentence Batch 129:  Loss:     0.3803 Training Accuracy: 0.906250\n",
      "Epoch  4, Sentence Batch 130:  Loss:     0.3580 Training Accuracy: 0.906250\n",
      "Epoch  4, Sentence Batch 131:  Loss:     0.3169 Training Accuracy: 0.953125\n",
      "Epoch  4, Sentence Batch 132:  Loss:     0.3362 Training Accuracy: 0.937500\n",
      "Epoch  4, Sentence Batch 133:  Loss:     0.3173 Training Accuracy: 0.890625\n",
      "Epoch  4, Sentence Batch 134:  Loss:     0.3165 Training Accuracy: 0.906250\n",
      "Epoch  4, Sentence Batch 135:  Loss:     0.2773 Training Accuracy: 0.968750\n",
      "Epoch  4, Sentence Batch 136:  Loss:     0.3497 Training Accuracy: 0.906250\n",
      "Epoch  4, Sentence Batch 137:  Loss:     0.3583 Training Accuracy: 0.890625\n",
      "Epoch  4, Sentence Batch 138:  Loss:     0.2954 Training Accuracy: 0.968750\n",
      "Epoch  4, Sentence Batch 139:  Loss:     0.2912 Training Accuracy: 0.937500\n",
      "Epoch  4, Sentence Batch 140:  Loss:     0.3080 Training Accuracy: 0.953125\n",
      "Epoch  4, Sentence Batch 141:  Loss:     0.3864 Training Accuracy: 0.875000\n",
      "Epoch  4, Sentence Batch 142:  Loss:     0.3055 Training Accuracy: 0.937500\n",
      "Epoch  4, Sentence Batch 143:  Loss:     0.2827 Training Accuracy: 0.921875\n",
      "Epoch  4, Sentence Batch 144:  Loss:     0.4018 Training Accuracy: 0.843750\n",
      "Epoch  4, Sentence Batch 145:  Loss:     0.3340 Training Accuracy: 0.953125\n",
      "Epoch  4, Sentence Batch 146:  Loss:     0.3294 Training Accuracy: 0.906250\n",
      "Epoch  4, Sentence Batch 147:  Loss:     0.3124 Training Accuracy: 0.921875\n",
      "Epoch  4, Sentence Batch 148:  Loss:     0.2498 Training Accuracy: 0.984375\n",
      "Epoch  4, Sentence Batch 149:  Loss:     0.3440 Training Accuracy: 0.883333\n",
      "Epoch  5, Sentence Batch 0:  Validation Accuracy: 0.779550\n",
      "Loss:     0.2554 Training Accuracy: 0.968750\n",
      "Epoch  5, Sentence Batch 1:  Loss:     0.2849 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 2:  Loss:     0.2579 Training Accuracy: 0.984375\n",
      "Epoch  5, Sentence Batch 3:  Loss:     0.2838 Training Accuracy: 0.968750\n",
      "Epoch  5, Sentence Batch 4:  Loss:     0.2490 Training Accuracy: 0.968750\n",
      "Epoch  5, Sentence Batch 5:  Loss:     0.2195 Training Accuracy: 0.984375\n",
      "Epoch  5, Sentence Batch 6:  Loss:     0.2924 Training Accuracy: 0.968750\n",
      "Epoch  5, Sentence Batch 7:  Loss:     0.2764 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 8:  Loss:     0.2193 Training Accuracy: 0.984375\n",
      "Epoch  5, Sentence Batch 9:  Loss:     0.2954 Training Accuracy: 0.921875\n",
      "Epoch  5, Sentence Batch 10:  Loss:     0.2437 Training Accuracy: 0.968750\n",
      "Epoch  5, Sentence Batch 11:  Loss:     0.2443 Training Accuracy: 0.968750\n",
      "Epoch  5, Sentence Batch 12:  Loss:     0.3156 Training Accuracy: 0.937500\n",
      "Epoch  5, Sentence Batch 13:  Loss:     0.2922 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 14:  Loss:     0.3629 Training Accuracy: 0.859375\n",
      "Epoch  5, Sentence Batch 15:  Loss:     0.2595 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 16:  Loss:     0.2369 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 17:  Loss:     0.2652 Training Accuracy: 0.968750\n",
      "Epoch  5, Sentence Batch 18:  Loss:     0.2409 Training Accuracy: 0.984375\n",
      "Epoch  5, Sentence Batch 19:  Loss:     0.3399 Training Accuracy: 0.906250\n",
      "Epoch  5, Sentence Batch 20:  Loss:     0.3028 Training Accuracy: 0.968750\n",
      "Epoch  5, Sentence Batch 21:  Loss:     0.2383 Training Accuracy: 1.000000\n",
      "Epoch  5, Sentence Batch 22:  Loss:     0.3142 Training Accuracy: 0.921875\n",
      "Epoch  5, Sentence Batch 23:  Loss:     0.3120 Training Accuracy: 0.875000\n",
      "Epoch  5, Sentence Batch 24:  Loss:     0.2273 Training Accuracy: 0.984375\n",
      "Epoch  5, Sentence Batch 25:  Loss:     0.3023 Training Accuracy: 0.937500\n",
      "Epoch  5, Sentence Batch 26:  Loss:     0.2892 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 27:  Loss:     0.2534 Training Accuracy: 0.984375\n",
      "Epoch  5, Sentence Batch 28:  Loss:     0.2702 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 29:  Loss:     0.2258 Training Accuracy: 0.968750\n",
      "Epoch  5, Sentence Batch 30:  Loss:     0.2967 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 31:  Loss:     0.3490 Training Accuracy: 0.906250\n",
      "Epoch  5, Sentence Batch 32:  Loss:     0.2626 Training Accuracy: 0.937500\n",
      "Epoch  5, Sentence Batch 33:  Loss:     0.2578 Training Accuracy: 0.984375\n",
      "Epoch  5, Sentence Batch 34:  Loss:     0.3098 Training Accuracy: 0.921875\n",
      "Epoch  5, Sentence Batch 35:  Loss:     0.2623 Training Accuracy: 0.984375\n",
      "Epoch  5, Sentence Batch 36:  Loss:     0.2935 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 37:  Loss:     0.2582 Training Accuracy: 0.937500\n",
      "Epoch  5, Sentence Batch 38:  Loss:     0.3184 Training Accuracy: 0.906250\n",
      "Epoch  5, Sentence Batch 39:  Loss:     0.2452 Training Accuracy: 0.984375\n",
      "Epoch  5, Sentence Batch 40:  Loss:     0.2253 Training Accuracy: 1.000000\n",
      "Epoch  5, Sentence Batch 41:  Loss:     0.2797 Training Accuracy: 0.937500\n",
      "Epoch  5, Sentence Batch 42:  Loss:     0.2593 Training Accuracy: 0.968750\n",
      "Epoch  5, Sentence Batch 43:  Loss:     0.2504 Training Accuracy: 0.984375\n",
      "Epoch  5, Sentence Batch 44:  Loss:     0.2684 Training Accuracy: 0.984375\n",
      "Epoch  5, Sentence Batch 45:  Loss:     0.2126 Training Accuracy: 0.984375\n",
      "Epoch  5, Sentence Batch 46:  Loss:     0.2396 Training Accuracy: 0.968750\n",
      "Epoch  5, Sentence Batch 47:  Loss:     0.3015 Training Accuracy: 0.937500\n",
      "Epoch  5, Sentence Batch 48:  Loss:     0.2400 Training Accuracy: 0.984375\n",
      "Epoch  5, Sentence Batch 49:  Loss:     0.3253 Training Accuracy: 0.921875\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  5, Sentence Batch 50:  Loss:     0.2759 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 51:  Loss:     0.2502 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 52:  Loss:     0.3236 Training Accuracy: 0.875000\n",
      "Epoch  5, Sentence Batch 53:  Loss:     0.2198 Training Accuracy: 0.984375\n",
      "Epoch  5, Sentence Batch 54:  Loss:     0.2158 Training Accuracy: 1.000000\n",
      "Epoch  5, Sentence Batch 55:  Loss:     0.2638 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 56:  Loss:     0.3011 Training Accuracy: 0.921875\n",
      "Epoch  5, Sentence Batch 57:  Loss:     0.3229 Training Accuracy: 0.890625\n",
      "Epoch  5, Sentence Batch 58:  Loss:     0.2796 Training Accuracy: 0.937500\n",
      "Epoch  5, Sentence Batch 59:  Loss:     0.2681 Training Accuracy: 0.968750\n",
      "Epoch  5, Sentence Batch 60:  Loss:     0.2846 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 61:  Loss:     0.3186 Training Accuracy: 0.937500\n",
      "Epoch  5, Sentence Batch 62:  Loss:     0.3100 Training Accuracy: 0.937500\n",
      "Epoch  5, Sentence Batch 63:  Loss:     0.2268 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 64:  Loss:     0.3310 Training Accuracy: 0.937500\n",
      "Epoch  5, Sentence Batch 65:  Loss:     0.2551 Training Accuracy: 0.937500\n",
      "Epoch  5, Sentence Batch 66:  Loss:     0.3066 Training Accuracy: 0.890625\n",
      "Epoch  5, Sentence Batch 67:  Loss:     0.2706 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 68:  Loss:     0.2207 Training Accuracy: 1.000000\n",
      "Epoch  5, Sentence Batch 69:  Loss:     0.2556 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 70:  Loss:     0.2286 Training Accuracy: 0.984375\n",
      "Epoch  5, Sentence Batch 71:  Loss:     0.2771 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 72:  Loss:     0.2966 Training Accuracy: 0.921875\n",
      "Epoch  5, Sentence Batch 73:  Loss:     0.2674 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 74:  Loss:     0.2991 Training Accuracy: 0.921875\n",
      "Epoch  5, Sentence Batch 75:  Loss:     0.2657 Training Accuracy: 0.984375\n",
      "Epoch  5, Sentence Batch 76:  Loss:     0.2705 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 77:  Loss:     0.3624 Training Accuracy: 0.921875\n",
      "Epoch  5, Sentence Batch 78:  Loss:     0.2504 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 79:  Loss:     0.2613 Training Accuracy: 0.937500\n",
      "Epoch  5, Sentence Batch 80:  Loss:     0.2655 Training Accuracy: 0.937500\n",
      "Epoch  5, Sentence Batch 81:  Loss:     0.2204 Training Accuracy: 1.000000\n",
      "Epoch  5, Sentence Batch 82:  Loss:     0.3028 Training Accuracy: 0.890625\n",
      "Epoch  5, Sentence Batch 83:  Loss:     0.2588 Training Accuracy: 0.968750\n",
      "Epoch  5, Sentence Batch 84:  Loss:     0.2335 Training Accuracy: 0.984375\n",
      "Epoch  5, Sentence Batch 85:  Loss:     0.2679 Training Accuracy: 0.937500\n",
      "Epoch  5, Sentence Batch 86:  Loss:     0.3180 Training Accuracy: 0.937500\n",
      "Epoch  5, Sentence Batch 87:  Loss:     0.2641 Training Accuracy: 0.968750\n",
      "Epoch  5, Sentence Batch 88:  Loss:     0.2334 Training Accuracy: 1.000000\n",
      "Epoch  5, Sentence Batch 89:  Loss:     0.2697 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 90:  Loss:     0.2881 Training Accuracy: 0.906250\n",
      "Epoch  5, Sentence Batch 91:  Loss:     0.3190 Training Accuracy: 0.937500\n",
      "Epoch  5, Sentence Batch 92:  Loss:     0.2941 Training Accuracy: 0.921875\n",
      "Epoch  5, Sentence Batch 93:  Loss:     0.2603 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 94:  Loss:     0.2268 Training Accuracy: 0.968750\n",
      "Epoch  5, Sentence Batch 95:  Loss:     0.2949 Training Accuracy: 0.937500\n",
      "Epoch  5, Sentence Batch 96:  Loss:     0.2906 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 97:  Loss:     0.2715 Training Accuracy: 0.921875\n",
      "Epoch  5, Sentence Batch 98:  Loss:     0.2443 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 99:  Loss:     0.2229 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 100:  Validation Accuracy: 0.784240\n",
      "Loss:     0.3150 Training Accuracy: 0.906250\n",
      "Epoch  5, Sentence Batch 101:  Loss:     0.2420 Training Accuracy: 0.921875\n",
      "Epoch  5, Sentence Batch 102:  Loss:     0.2350 Training Accuracy: 0.984375\n",
      "Epoch  5, Sentence Batch 103:  Loss:     0.3418 Training Accuracy: 0.890625\n",
      "Epoch  5, Sentence Batch 104:  Loss:     0.2850 Training Accuracy: 0.968750\n",
      "Epoch  5, Sentence Batch 105:  Loss:     0.2750 Training Accuracy: 0.937500\n",
      "Epoch  5, Sentence Batch 106:  Loss:     0.2445 Training Accuracy: 0.921875\n",
      "Epoch  5, Sentence Batch 107:  Loss:     0.2570 Training Accuracy: 0.968750\n",
      "Epoch  5, Sentence Batch 108:  Loss:     0.2350 Training Accuracy: 0.984375\n",
      "Epoch  5, Sentence Batch 109:  Loss:     0.3683 Training Accuracy: 0.875000\n",
      "Epoch  5, Sentence Batch 110:  Loss:     0.2448 Training Accuracy: 0.968750\n",
      "Epoch  5, Sentence Batch 111:  Loss:     0.3083 Training Accuracy: 0.906250\n",
      "Epoch  5, Sentence Batch 112:  Loss:     0.2074 Training Accuracy: 1.000000\n",
      "Epoch  5, Sentence Batch 113:  Loss:     0.2464 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 114:  Loss:     0.3194 Training Accuracy: 0.890625\n",
      "Epoch  5, Sentence Batch 115:  Loss:     0.2879 Training Accuracy: 0.875000\n",
      "Epoch  5, Sentence Batch 116:  Loss:     0.2852 Training Accuracy: 0.937500\n",
      "Epoch  5, Sentence Batch 117:  Loss:     0.2315 Training Accuracy: 0.968750\n",
      "Epoch  5, Sentence Batch 118:  Loss:     0.3044 Training Accuracy: 0.890625\n",
      "Epoch  5, Sentence Batch 119:  Loss:     0.2183 Training Accuracy: 0.984375\n",
      "Epoch  5, Sentence Batch 120:  Loss:     0.2747 Training Accuracy: 0.937500\n",
      "Epoch  5, Sentence Batch 121:  Loss:     0.3347 Training Accuracy: 0.921875\n",
      "Epoch  5, Sentence Batch 122:  Loss:     0.3165 Training Accuracy: 0.906250\n",
      "Epoch  5, Sentence Batch 123:  Loss:     0.2312 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 124:  Loss:     0.3133 Training Accuracy: 0.937500\n",
      "Epoch  5, Sentence Batch 125:  Loss:     0.2870 Training Accuracy: 0.921875\n",
      "Epoch  5, Sentence Batch 126:  Loss:     0.2748 Training Accuracy: 0.921875\n",
      "Epoch  5, Sentence Batch 127:  Loss:     0.2745 Training Accuracy: 0.937500\n",
      "Epoch  5, Sentence Batch 128:  Loss:     0.3003 Training Accuracy: 0.890625\n",
      "Epoch  5, Sentence Batch 129:  Loss:     0.2034 Training Accuracy: 0.984375\n",
      "Epoch  5, Sentence Batch 130:  Loss:     0.3458 Training Accuracy: 0.921875\n",
      "Epoch  5, Sentence Batch 131:  Loss:     0.3034 Training Accuracy: 0.890625\n",
      "Epoch  5, Sentence Batch 132:  Loss:     0.2522 Training Accuracy: 0.984375\n",
      "Epoch  5, Sentence Batch 133:  Loss:     0.2202 Training Accuracy: 1.000000\n",
      "Epoch  5, Sentence Batch 134:  Loss:     0.2940 Training Accuracy: 0.937500\n",
      "Epoch  5, Sentence Batch 135:  Loss:     0.2480 Training Accuracy: 0.968750\n",
      "Epoch  5, Sentence Batch 136:  Loss:     0.2661 Training Accuracy: 0.937500\n",
      "Epoch  5, Sentence Batch 137:  Loss:     0.2761 Training Accuracy: 0.937500\n",
      "Epoch  5, Sentence Batch 138:  Loss:     0.2610 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 139:  Loss:     0.2671 Training Accuracy: 0.968750\n",
      "Epoch  5, Sentence Batch 140:  Loss:     0.2905 Training Accuracy: 0.921875\n",
      "Epoch  5, Sentence Batch 141:  Loss:     0.2444 Training Accuracy: 0.968750\n",
      "Epoch  5, Sentence Batch 142:  Loss:     0.2542 Training Accuracy: 0.984375\n",
      "Epoch  5, Sentence Batch 143:  Loss:     0.2657 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 144:  Loss:     0.2660 Training Accuracy: 0.937500\n",
      "Epoch  5, Sentence Batch 145:  Loss:     0.2739 Training Accuracy: 0.953125\n",
      "Epoch  5, Sentence Batch 146:  Loss:     0.3315 Training Accuracy: 0.906250\n",
      "Epoch  5, Sentence Batch 147:  Loss:     0.2880 Training Accuracy: 0.968750\n",
      "Epoch  5, Sentence Batch 148:  Loss:     0.2654 Training Accuracy: 0.968750\n",
      "Epoch  5, Sentence Batch 149:  Loss:     0.2261 Training Accuracy: 0.966667\n",
      "Epoch  6, Sentence Batch 0:  Loss:     0.2547 Training Accuracy: 0.953125\n",
      "Epoch  6, Sentence Batch 1:  Loss:     0.2072 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 2:  Loss:     0.1968 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 3:  Loss:     0.2074 Training Accuracy: 1.000000\n",
      "Epoch  6, Sentence Batch 4:  Loss:     0.2735 Training Accuracy: 0.921875\n",
      "Epoch  6, Sentence Batch 5:  Loss:     0.2161 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 6:  Loss:     0.2247 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 7:  Loss:     0.1862 Training Accuracy: 0.984375\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  6, Sentence Batch 8:  Loss:     0.2226 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 9:  Loss:     0.2509 Training Accuracy: 0.937500\n",
      "Epoch  6, Sentence Batch 10:  Loss:     0.2245 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 11:  Loss:     0.2315 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 12:  Loss:     0.2255 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 13:  Loss:     0.2623 Training Accuracy: 0.937500\n",
      "Epoch  6, Sentence Batch 14:  Loss:     0.2481 Training Accuracy: 0.937500\n",
      "Epoch  6, Sentence Batch 15:  Loss:     0.2296 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 16:  Loss:     0.2304 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 17:  Loss:     0.1986 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 18:  Loss:     0.1958 Training Accuracy: 1.000000\n",
      "Epoch  6, Sentence Batch 19:  Loss:     0.1880 Training Accuracy: 1.000000\n",
      "Epoch  6, Sentence Batch 20:  Loss:     0.2100 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 21:  Loss:     0.2135 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 22:  Loss:     0.2286 Training Accuracy: 0.953125\n",
      "Epoch  6, Sentence Batch 23:  Loss:     0.2059 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 24:  Loss:     0.2140 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 25:  Loss:     0.2364 Training Accuracy: 0.937500\n",
      "Epoch  6, Sentence Batch 26:  Loss:     0.2358 Training Accuracy: 0.953125\n",
      "Epoch  6, Sentence Batch 27:  Loss:     0.2638 Training Accuracy: 0.953125\n",
      "Epoch  6, Sentence Batch 28:  Loss:     0.1996 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 29:  Loss:     0.2445 Training Accuracy: 0.953125\n",
      "Epoch  6, Sentence Batch 30:  Loss:     0.2665 Training Accuracy: 0.953125\n",
      "Epoch  6, Sentence Batch 31:  Loss:     0.1890 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 32:  Loss:     0.1775 Training Accuracy: 1.000000\n",
      "Epoch  6, Sentence Batch 33:  Loss:     0.2236 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 34:  Loss:     0.2091 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 35:  Loss:     0.2134 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 36:  Loss:     0.2270 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 37:  Loss:     0.2006 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 38:  Loss:     0.1942 Training Accuracy: 1.000000\n",
      "Epoch  6, Sentence Batch 39:  Loss:     0.2189 Training Accuracy: 1.000000\n",
      "Epoch  6, Sentence Batch 40:  Loss:     0.2062 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 41:  Loss:     0.1846 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 42:  Loss:     0.2756 Training Accuracy: 0.921875\n",
      "Epoch  6, Sentence Batch 43:  Loss:     0.2194 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 44:  Loss:     0.2577 Training Accuracy: 0.953125\n",
      "Epoch  6, Sentence Batch 45:  Loss:     0.2510 Training Accuracy: 0.937500\n",
      "Epoch  6, Sentence Batch 46:  Loss:     0.2346 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 47:  Loss:     0.2319 Training Accuracy: 0.953125\n",
      "Epoch  6, Sentence Batch 48:  Loss:     0.2112 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 49:  Loss:     0.2027 Training Accuracy: 1.000000\n",
      "Epoch  6, Sentence Batch 50:  Validation Accuracy: 0.780488\n",
      "Loss:     0.1841 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 51:  Loss:     0.2069 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 52:  Loss:     0.2287 Training Accuracy: 0.953125\n",
      "Epoch  6, Sentence Batch 53:  Loss:     0.1928 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 54:  Loss:     0.1925 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 55:  Loss:     0.2005 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 56:  Loss:     0.1804 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 57:  Loss:     0.2062 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 58:  Loss:     0.2034 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 59:  Loss:     0.1880 Training Accuracy: 1.000000\n",
      "Epoch  6, Sentence Batch 60:  Loss:     0.2443 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 61:  Loss:     0.2046 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 62:  Loss:     0.1764 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 63:  Loss:     0.1632 Training Accuracy: 1.000000\n",
      "Epoch  6, Sentence Batch 64:  Loss:     0.2456 Training Accuracy: 0.953125\n",
      "Epoch  6, Sentence Batch 65:  Loss:     0.2462 Training Accuracy: 0.953125\n",
      "Epoch  6, Sentence Batch 66:  Loss:     0.2104 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 67:  Loss:     0.1813 Training Accuracy: 1.000000\n",
      "Epoch  6, Sentence Batch 68:  Loss:     0.1805 Training Accuracy: 1.000000\n",
      "Epoch  6, Sentence Batch 69:  Loss:     0.1971 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 70:  Loss:     0.1924 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 71:  Loss:     0.1961 Training Accuracy: 1.000000\n",
      "Epoch  6, Sentence Batch 72:  Loss:     0.2067 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 73:  Loss:     0.1833 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 74:  Loss:     0.2179 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 75:  Loss:     0.2029 Training Accuracy: 1.000000\n",
      "Epoch  6, Sentence Batch 76:  Loss:     0.1763 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 77:  Loss:     0.2322 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 78:  Loss:     0.2208 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 79:  Loss:     0.2196 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 80:  Loss:     0.1916 Training Accuracy: 1.000000\n",
      "Epoch  6, Sentence Batch 81:  Loss:     0.2412 Training Accuracy: 0.937500\n",
      "Epoch  6, Sentence Batch 82:  Loss:     0.2584 Training Accuracy: 0.953125\n",
      "Epoch  6, Sentence Batch 83:  Loss:     0.2160 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 84:  Loss:     0.2519 Training Accuracy: 0.953125\n",
      "Epoch  6, Sentence Batch 85:  Loss:     0.1975 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 86:  Loss:     0.2492 Training Accuracy: 0.937500\n",
      "Epoch  6, Sentence Batch 87:  Loss:     0.2477 Training Accuracy: 0.937500\n",
      "Epoch  6, Sentence Batch 88:  Loss:     0.1780 Training Accuracy: 0.953125\n",
      "Epoch  6, Sentence Batch 89:  Loss:     0.2685 Training Accuracy: 0.921875\n",
      "Epoch  6, Sentence Batch 90:  Loss:     0.2860 Training Accuracy: 0.937500\n",
      "Epoch  6, Sentence Batch 91:  Loss:     0.2099 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 92:  Loss:     0.2069 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 93:  Loss:     0.1993 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 94:  Loss:     0.2078 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 95:  Loss:     0.2662 Training Accuracy: 0.953125\n",
      "Epoch  6, Sentence Batch 96:  Loss:     0.2141 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 97:  Loss:     0.2167 Training Accuracy: 0.937500\n",
      "Epoch  6, Sentence Batch 98:  Loss:     0.1479 Training Accuracy: 1.000000\n",
      "Epoch  6, Sentence Batch 99:  Loss:     0.2209 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 100:  Loss:     0.1714 Training Accuracy: 1.000000\n",
      "Epoch  6, Sentence Batch 101:  Loss:     0.1911 Training Accuracy: 0.953125\n",
      "Epoch  6, Sentence Batch 102:  Loss:     0.1732 Training Accuracy: 1.000000\n",
      "Epoch  6, Sentence Batch 103:  Loss:     0.1757 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 104:  Loss:     0.2501 Training Accuracy: 0.921875\n",
      "Epoch  6, Sentence Batch 105:  Loss:     0.1927 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 106:  Loss:     0.2473 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 107:  Loss:     0.1979 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 108:  Loss:     0.2029 Training Accuracy: 1.000000\n",
      "Epoch  6, Sentence Batch 109:  Loss:     0.2179 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 110:  Loss:     0.2470 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 111:  Loss:     0.1857 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 112:  Loss:     0.2264 Training Accuracy: 0.937500\n",
      "Epoch  6, Sentence Batch 113:  Loss:     0.2505 Training Accuracy: 0.953125\n",
      "Epoch  6, Sentence Batch 114:  Loss:     0.2000 Training Accuracy: 0.953125\n",
      "Epoch  6, Sentence Batch 115:  Loss:     0.2308 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 116:  Loss:     0.2054 Training Accuracy: 0.984375\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  6, Sentence Batch 117:  Loss:     0.2207 Training Accuracy: 0.937500\n",
      "Epoch  6, Sentence Batch 118:  Loss:     0.2171 Training Accuracy: 0.953125\n",
      "Epoch  6, Sentence Batch 119:  Loss:     0.1683 Training Accuracy: 1.000000\n",
      "Epoch  6, Sentence Batch 120:  Loss:     0.2008 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 121:  Loss:     0.3283 Training Accuracy: 0.921875\n",
      "Epoch  6, Sentence Batch 122:  Loss:     0.1822 Training Accuracy: 1.000000\n",
      "Epoch  6, Sentence Batch 123:  Loss:     0.1740 Training Accuracy: 1.000000\n",
      "Epoch  6, Sentence Batch 124:  Loss:     0.1891 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 125:  Loss:     0.2525 Training Accuracy: 0.953125\n",
      "Epoch  6, Sentence Batch 126:  Loss:     0.1959 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 127:  Loss:     0.2573 Training Accuracy: 0.937500\n",
      "Epoch  6, Sentence Batch 128:  Loss:     0.2742 Training Accuracy: 0.937500\n",
      "Epoch  6, Sentence Batch 129:  Loss:     0.2343 Training Accuracy: 0.953125\n",
      "Epoch  6, Sentence Batch 130:  Loss:     0.2115 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 131:  Loss:     0.2029 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 132:  Loss:     0.2515 Training Accuracy: 0.937500\n",
      "Epoch  6, Sentence Batch 133:  Loss:     0.2603 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 134:  Loss:     0.1982 Training Accuracy: 0.937500\n",
      "Epoch  6, Sentence Batch 135:  Loss:     0.1978 Training Accuracy: 1.000000\n",
      "Epoch  6, Sentence Batch 136:  Loss:     0.1887 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 137:  Loss:     0.2191 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 138:  Loss:     0.2001 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 139:  Loss:     0.2093 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 140:  Loss:     0.2010 Training Accuracy: 0.984375\n",
      "Epoch  6, Sentence Batch 141:  Loss:     0.2448 Training Accuracy: 0.906250\n",
      "Epoch  6, Sentence Batch 142:  Loss:     0.2154 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 143:  Loss:     0.2395 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 144:  Loss:     0.2686 Training Accuracy: 0.953125\n",
      "Epoch  6, Sentence Batch 145:  Loss:     0.2154 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 146:  Loss:     0.1826 Training Accuracy: 0.953125\n",
      "Epoch  6, Sentence Batch 147:  Loss:     0.2062 Training Accuracy: 0.953125\n",
      "Epoch  6, Sentence Batch 148:  Loss:     0.2004 Training Accuracy: 0.968750\n",
      "Epoch  6, Sentence Batch 149:  Loss:     0.2354 Training Accuracy: 0.950000\n",
      "Epoch  7, Sentence Batch 0:  Validation Accuracy: 0.786116\n",
      "Loss:     0.1753 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 1:  Loss:     0.1451 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 2:  Loss:     0.1718 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 3:  Loss:     0.2207 Training Accuracy: 0.953125\n",
      "Epoch  7, Sentence Batch 4:  Loss:     0.1863 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 5:  Loss:     0.1639 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 6:  Loss:     0.1884 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 7:  Loss:     0.1962 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 8:  Loss:     0.1888 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 9:  Loss:     0.1710 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 10:  Loss:     0.1394 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 11:  Loss:     0.1597 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 12:  Loss:     0.1787 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 13:  Loss:     0.1868 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 14:  Loss:     0.1481 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 15:  Loss:     0.1811 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 16:  Loss:     0.1926 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 17:  Loss:     0.2187 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 18:  Loss:     0.1644 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 19:  Loss:     0.2063 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 20:  Loss:     0.1740 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 21:  Loss:     0.1744 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 22:  Loss:     0.1671 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 23:  Loss:     0.1381 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 24:  Loss:     0.1620 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 25:  Loss:     0.1348 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 26:  Loss:     0.1578 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 27:  Loss:     0.1786 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 28:  Loss:     0.1417 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 29:  Loss:     0.1591 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 30:  Loss:     0.2067 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 31:  Loss:     0.2255 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 32:  Loss:     0.1802 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 33:  Loss:     0.1601 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 34:  Loss:     0.1779 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 35:  Loss:     0.1764 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 36:  Loss:     0.1707 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 37:  Loss:     0.1661 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 38:  Loss:     0.1479 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 39:  Loss:     0.2362 Training Accuracy: 0.953125\n",
      "Epoch  7, Sentence Batch 40:  Loss:     0.1821 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 41:  Loss:     0.1648 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 42:  Loss:     0.1893 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 43:  Loss:     0.1804 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 44:  Loss:     0.1990 Training Accuracy: 0.937500\n",
      "Epoch  7, Sentence Batch 45:  Loss:     0.1948 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 46:  Loss:     0.1699 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 47:  Loss:     0.1628 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 48:  Loss:     0.1337 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 49:  Loss:     0.1691 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 50:  Loss:     0.1790 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 51:  Loss:     0.1787 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 52:  Loss:     0.1683 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 53:  Loss:     0.1774 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 54:  Loss:     0.1600 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 55:  Loss:     0.1669 Training Accuracy: 0.953125\n",
      "Epoch  7, Sentence Batch 56:  Loss:     0.1765 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 57:  Loss:     0.1799 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 58:  Loss:     0.1448 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 59:  Loss:     0.1817 Training Accuracy: 0.953125\n",
      "Epoch  7, Sentence Batch 60:  Loss:     0.1665 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 61:  Loss:     0.1569 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 62:  Loss:     0.1756 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 63:  Loss:     0.1578 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 64:  Loss:     0.1635 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 65:  Loss:     0.1679 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 66:  Loss:     0.1447 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 67:  Loss:     0.1898 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 68:  Loss:     0.1673 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 69:  Loss:     0.1969 Training Accuracy: 0.953125\n",
      "Epoch  7, Sentence Batch 70:  Loss:     0.1881 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 71:  Loss:     0.1641 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 72:  Loss:     0.1858 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 73:  Loss:     0.1725 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 74:  Loss:     0.1534 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 75:  Loss:     0.1704 Training Accuracy: 0.968750\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  7, Sentence Batch 76:  Loss:     0.1960 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 77:  Loss:     0.1873 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 78:  Loss:     0.1723 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 79:  Loss:     0.1593 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 80:  Loss:     0.1349 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 81:  Loss:     0.1585 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 82:  Loss:     0.1444 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 83:  Loss:     0.1582 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 84:  Loss:     0.2071 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 85:  Loss:     0.1868 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 86:  Loss:     0.1784 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 87:  Loss:     0.1606 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 88:  Loss:     0.1996 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 89:  Loss:     0.1642 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 90:  Loss:     0.1654 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 91:  Loss:     0.1587 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 92:  Loss:     0.1887 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 93:  Loss:     0.1630 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 94:  Loss:     0.1579 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 95:  Loss:     0.1787 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 96:  Loss:     0.1959 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 97:  Loss:     0.1773 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 98:  Loss:     0.1788 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 99:  Loss:     0.1899 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 100:  Validation Accuracy: 0.792683\n",
      "Loss:     0.2027 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 101:  Loss:     0.1580 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 102:  Loss:     0.1864 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 103:  Loss:     0.1523 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 104:  Loss:     0.2102 Training Accuracy: 0.953125\n",
      "Epoch  7, Sentence Batch 105:  Loss:     0.1864 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 106:  Loss:     0.1601 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 107:  Loss:     0.1933 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 108:  Loss:     0.1309 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 109:  Loss:     0.2073 Training Accuracy: 0.953125\n",
      "Epoch  7, Sentence Batch 110:  Loss:     0.1604 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 111:  Loss:     0.1332 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 112:  Loss:     0.1568 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 113:  Loss:     0.1880 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 114:  Loss:     0.1590 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 115:  Loss:     0.2059 Training Accuracy: 0.953125\n",
      "Epoch  7, Sentence Batch 116:  Loss:     0.1573 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 117:  Loss:     0.1693 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 118:  Loss:     0.1581 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 119:  Loss:     0.1490 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 120:  Loss:     0.1669 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 121:  Loss:     0.1943 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 122:  Loss:     0.2053 Training Accuracy: 0.953125\n",
      "Epoch  7, Sentence Batch 123:  Loss:     0.1428 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 124:  Loss:     0.2017 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 125:  Loss:     0.1792 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 126:  Loss:     0.1698 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 127:  Loss:     0.1740 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 128:  Loss:     0.1563 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 129:  Loss:     0.1905 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 130:  Loss:     0.1639 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 131:  Loss:     0.1465 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 132:  Loss:     0.1421 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 133:  Loss:     0.1575 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 134:  Loss:     0.1418 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 135:  Loss:     0.1564 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 136:  Loss:     0.1805 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 137:  Loss:     0.1775 Training Accuracy: 0.953125\n",
      "Epoch  7, Sentence Batch 138:  Loss:     0.2886 Training Accuracy: 0.937500\n",
      "Epoch  7, Sentence Batch 139:  Loss:     0.1912 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 140:  Loss:     0.1755 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 141:  Loss:     0.1460 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 142:  Loss:     0.1777 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 143:  Loss:     0.1697 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 144:  Loss:     0.1566 Training Accuracy: 1.000000\n",
      "Epoch  7, Sentence Batch 145:  Loss:     0.1767 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 146:  Loss:     0.1642 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 147:  Loss:     0.1441 Training Accuracy: 0.984375\n",
      "Epoch  7, Sentence Batch 148:  Loss:     0.1714 Training Accuracy: 0.968750\n",
      "Epoch  7, Sentence Batch 149:  Loss:     0.1607 Training Accuracy: 0.966667\n",
      "#######VALIDATION STATS#######\n",
      "Validation Accuracy: 0.783302\n",
      "#######SAVING FINAL RESULTS#######\n"
     ]
    }
   ],
   "source": [
    "print('Training...')\n",
    "with tf.Session() as sess:\n",
    "    # Initializing the variables\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    if(T.any):\n",
    "        sess.run(W.assign(T))\n",
    "    # Generate single batches\n",
    "    batches = batch_iter(list(zip(x_train, y_train)), batch_size, num_batches_per_epoch, epochs, shuffle=True)\n",
    "    # Training cycle\n",
    "    i = 0 \n",
    "    for batch in batches:\n",
    "        if(i%100 == 0):\n",
    "            print_validation_stats(sess)\n",
    "            i = 0\n",
    "        batch_features, batch_labels = zip(*batch)\n",
    "        train_neural_network(sess, train_op, keep_probability, batch_features, batch_labels)\n",
    "        print_stats(sess, batch_features, batch_labels, cost, accuracy)\n",
    "        i+=1\n",
    "        \n",
    "    print(\"#######VALIDATION STATS#######\")\n",
    "    print_validation_stats(sess)\n",
    "    print(\"#######SAVING FINAL RESULTS#######\")\n",
    "    save_path = saver.save(sess, \"./tmp/full_model.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}