04_batch_normalisation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "(train_images, train_labels),  (test_images, test_labels) = keras.datasets.fashion_mnist.load_data()\n",
    "train_images = train_images / 255.0\n",
    "test_images = test_images / 255.0\n",
    "validation_images = train_images[:5000]\n",
    "validation_labels = train_labels[:5000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Placing batch normalization layer before the activation layers\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28,28]),\n",
    "    keras.layers.Dense(300, use_bias=False),\n",
    "    keras.layers.BatchNormalization(),\n",
    "    keras.layers.Dense(200, use_bias=False),\n",
    "    keras.layers.BatchNormalization(),\n",
    "    keras.layers.Activation(keras.activations.relu),\n",
    "    keras.layers.Dense(100, use_bias=False),\n",
    "    keras.layers.BatchNormalization(),\n",
    "    keras.layers.Activation(keras.activations.relu),\n",
    "    keras.layers.Dense(10, activation=keras.activations.softmax)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten (Flatten)            (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 300)               235200    \n",
      "_________________________________________________________________\n",
      "batch_normalization (BatchNo (None, 300)               1200      \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 200)               60000     \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 200)               800       \n",
      "_________________________________________________________________\n",
      "activation (Activation)      (None, 200)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 100)               20000     \n",
      "_________________________________________________________________\n",
      "batch_normalization_2 (Batch (None, 100)               400       \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 10)                1010      \n",
      "=================================================================\n",
      "Total params: 318,610\n",
      "Trainable params: 317,410\n",
      "Non-trainable params: 1,200\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'batch_normalization/gamma:0' shape=(300,) dtype=float32, numpy=\n",
       " array([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.], dtype=float32)>,\n",
       " <tf.Variable 'batch_normalization/beta:0' shape=(300,) dtype=float32, numpy=\n",
       " array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], dtype=float32)>,\n",
       " <tf.Variable 'batch_normalization/moving_mean:0' shape=(300,) dtype=float32, numpy=\n",
       " array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], dtype=float32)>,\n",
       " <tf.Variable 'batch_normalization/moving_variance:0' shape=(300,) dtype=float32, numpy=\n",
       " array([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.], dtype=float32)>]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.layers[2].variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "batch_normalization/gamma:0\n",
      "batch_normalization/beta:0\n",
      "batch_normalization/moving_mean:0\n",
      "batch_normalization/moving_variance:0\n"
     ]
    }
   ],
   "source": [
    "for variable in model.layers[2].variables:\n",
    "    print(variable.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "sgd = keras.optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)\n",
    "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=sgd, metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 5000 samples\n",
      "Epoch 1/60\n",
      "60000/60000 [==============================] - 8s 131us/sample - loss: 0.4694 - accuracy: 0.8304 - val_loss: 0.3797 - val_accuracy: 0.8614\n",
      "Epoch 2/60\n",
      "60000/60000 [==============================] - 7s 111us/sample - loss: 0.3584 - accuracy: 0.8694 - val_loss: 0.2981 - val_accuracy: 0.8932\n",
      "Epoch 3/60\n",
      "60000/60000 [==============================] - 7s 118us/sample - loss: 0.3206 - accuracy: 0.8809 - val_loss: 0.2643 - val_accuracy: 0.8986\n",
      "Epoch 4/60\n",
      "60000/60000 [==============================] - 9s 142us/sample - loss: 0.2988 - accuracy: 0.8900 - val_loss: 0.2520 - val_accuracy: 0.9032\n",
      "Epoch 5/60\n",
      "60000/60000 [==============================] - 7s 124us/sample - loss: 0.2825 - accuracy: 0.8953 - val_loss: 0.2345 - val_accuracy: 0.9136\n",
      "Epoch 6/60\n",
      "60000/60000 [==============================] - 7s 122us/sample - loss: 0.2674 - accuracy: 0.9008 - val_loss: 0.2283 - val_accuracy: 0.9136\n",
      "Epoch 7/60\n",
      "60000/60000 [==============================] - 7s 120us/sample - loss: 0.2525 - accuracy: 0.9051 - val_loss: 0.2201 - val_accuracy: 0.9174\n",
      "Epoch 8/60\n",
      "60000/60000 [==============================] - 7s 120us/sample - loss: 0.2430 - accuracy: 0.9092 - val_loss: 0.2180 - val_accuracy: 0.9190\n",
      "Epoch 9/60\n",
      "60000/60000 [==============================] - 7s 124us/sample - loss: 0.2332 - accuracy: 0.9134 - val_loss: 0.1930 - val_accuracy: 0.9238\n",
      "Epoch 10/60\n",
      "60000/60000 [==============================] - 7s 122us/sample - loss: 0.2255 - accuracy: 0.9143 - val_loss: 0.1926 - val_accuracy: 0.9266\n",
      "Epoch 11/60\n",
      "60000/60000 [==============================] - 7s 120us/sample - loss: 0.2141 - accuracy: 0.9195 - val_loss: 0.1814 - val_accuracy: 0.9314\n",
      "Epoch 12/60\n",
      "60000/60000 [==============================] - 7s 124us/sample - loss: 0.2077 - accuracy: 0.9213 - val_loss: 0.1930 - val_accuracy: 0.9256\n",
      "Epoch 13/60\n",
      "60000/60000 [==============================] - 7s 121us/sample - loss: 0.1996 - accuracy: 0.9249 - val_loss: 0.1732 - val_accuracy: 0.9336\n",
      "Epoch 14/60\n",
      "60000/60000 [==============================] - 8s 125us/sample - loss: 0.1963 - accuracy: 0.9266 - val_loss: 0.1638 - val_accuracy: 0.9382\n",
      "Epoch 15/60\n",
      "60000/60000 [==============================] - 7s 122us/sample - loss: 0.1880 - accuracy: 0.9288 - val_loss: 0.1634 - val_accuracy: 0.9376\n",
      "Epoch 16/60\n",
      "60000/60000 [==============================] - 7s 123us/sample - loss: 0.1857 - accuracy: 0.9300 - val_loss: 0.1509 - val_accuracy: 0.9450\n",
      "Epoch 17/60\n",
      "60000/60000 [==============================] - 8s 127us/sample - loss: 0.1786 - accuracy: 0.9325 - val_loss: 0.1713 - val_accuracy: 0.9366\n",
      "Epoch 18/60\n",
      "60000/60000 [==============================] - 7s 119us/sample - loss: 0.1711 - accuracy: 0.9353 - val_loss: 0.1581 - val_accuracy: 0.9394\n",
      "Epoch 19/60\n",
      "60000/60000 [==============================] - 7s 123us/sample - loss: 0.1704 - accuracy: 0.9349 - val_loss: 0.1474 - val_accuracy: 0.9420\n",
      "Epoch 20/60\n",
      "60000/60000 [==============================] - 8s 129us/sample - loss: 0.1635 - accuracy: 0.9377 - val_loss: 0.1313 - val_accuracy: 0.9518\n",
      "Epoch 21/60\n",
      "60000/60000 [==============================] - 8s 130us/sample - loss: 0.1576 - accuracy: 0.9399 - val_loss: 0.1403 - val_accuracy: 0.9482\n",
      "Epoch 22/60\n",
      "60000/60000 [==============================] - 8s 132us/sample - loss: 0.1547 - accuracy: 0.9416 - val_loss: 0.1158 - val_accuracy: 0.9560\n",
      "Epoch 23/60\n",
      "60000/60000 [==============================] - 7s 123us/sample - loss: 0.1500 - accuracy: 0.9428 - val_loss: 0.1864 - val_accuracy: 0.9298\n",
      "Epoch 24/60\n",
      "60000/60000 [==============================] - 8s 140us/sample - loss: 0.1477 - accuracy: 0.9439 - val_loss: 0.1164 - val_accuracy: 0.9556\n",
      "Epoch 25/60\n",
      "60000/60000 [==============================] - 9s 150us/sample - loss: 0.1460 - accuracy: 0.9446 - val_loss: 0.1338 - val_accuracy: 0.9502\n",
      "Epoch 26/60\n",
      "60000/60000 [==============================] - 8s 138us/sample - loss: 0.1393 - accuracy: 0.9478 - val_loss: 0.1211 - val_accuracy: 0.9498\n",
      "Epoch 27/60\n",
      "60000/60000 [==============================] - 8s 141us/sample - loss: 0.1345 - accuracy: 0.9484 - val_loss: 0.1140 - val_accuracy: 0.9558\n",
      "Epoch 28/60\n",
      "60000/60000 [==============================] - 8s 134us/sample - loss: 0.1336 - accuracy: 0.9503 - val_loss: 0.1576 - val_accuracy: 0.9406\n",
      "Epoch 29/60\n",
      "60000/60000 [==============================] - 8s 129us/sample - loss: 0.1282 - accuracy: 0.9503 - val_loss: 0.1006 - val_accuracy: 0.9636\n",
      "Epoch 30/60\n",
      "60000/60000 [==============================] - 9s 147us/sample - loss: 0.1274 - accuracy: 0.9516 - val_loss: 0.1030 - val_accuracy: 0.9604\n",
      "Epoch 31/60\n",
      "60000/60000 [==============================] - 8s 137us/sample - loss: 0.1243 - accuracy: 0.9540 - val_loss: 0.0961 - val_accuracy: 0.9622\n",
      "Epoch 32/60\n",
      "60000/60000 [==============================] - 7s 125us/sample - loss: 0.1236 - accuracy: 0.9532 - val_loss: 0.0803 - val_accuracy: 0.9694\n",
      "Epoch 33/60\n",
      "60000/60000 [==============================] - 8s 133us/sample - loss: 0.1205 - accuracy: 0.9542 - val_loss: 0.1007 - val_accuracy: 0.9618\n",
      "Epoch 34/60\n",
      "60000/60000 [==============================] - 8s 137us/sample - loss: 0.1137 - accuracy: 0.9561 - val_loss: 0.1251 - val_accuracy: 0.9544\n",
      "Epoch 35/60\n",
      "60000/60000 [==============================] - 7s 118us/sample - loss: 0.1132 - accuracy: 0.9571 - val_loss: 0.1573 - val_accuracy: 0.9430\n",
      "Epoch 36/60\n",
      "60000/60000 [==============================] - 7s 120us/sample - loss: 0.1108 - accuracy: 0.9589 - val_loss: 0.0984 - val_accuracy: 0.9608\n",
      "Epoch 37/60\n",
      "60000/60000 [==============================] - 8s 129us/sample - loss: 0.1076 - accuracy: 0.9591 - val_loss: 0.0846 - val_accuracy: 0.9684\n",
      "Epoch 38/60\n",
      "60000/60000 [==============================] - 7s 116us/sample - loss: 0.1085 - accuracy: 0.9589 - val_loss: 0.0661 - val_accuracy: 0.9766\n",
      "Epoch 39/60\n",
      "60000/60000 [==============================] - 9s 150us/sample - loss: 0.1038 - accuracy: 0.9617 - val_loss: 0.0916 - val_accuracy: 0.9634\n",
      "Epoch 40/60\n",
      "60000/60000 [==============================] - 8s 133us/sample - loss: 0.1003 - accuracy: 0.9623 - val_loss: 0.1022 - val_accuracy: 0.9590\n",
      "Epoch 41/60\n",
      "60000/60000 [==============================] - 8s 141us/sample - loss: 0.1033 - accuracy: 0.9605 - val_loss: 0.0880 - val_accuracy: 0.9680\n",
      "Epoch 42/60\n",
      "60000/60000 [==============================] - 8s 132us/sample - loss: 0.0956 - accuracy: 0.9632 - val_loss: 0.0704 - val_accuracy: 0.9736\n",
      "Epoch 43/60\n",
      "60000/60000 [==============================] - 7s 120us/sample - loss: 0.0955 - accuracy: 0.9632 - val_loss: 0.1018 - val_accuracy: 0.9612\n",
      "Epoch 44/60\n",
      "60000/60000 [==============================] - 7s 115us/sample - loss: 0.0952 - accuracy: 0.9632 - val_loss: 0.0774 - val_accuracy: 0.9700\n",
      "Epoch 45/60\n",
      "60000/60000 [==============================] - 7s 121us/sample - loss: 0.0944 - accuracy: 0.9636 - val_loss: 0.0542 - val_accuracy: 0.9798\n",
      "Epoch 46/60\n",
      "60000/60000 [==============================] - 7s 120us/sample - loss: 0.0905 - accuracy: 0.9659 - val_loss: 0.0729 - val_accuracy: 0.9734\n",
      "Epoch 47/60\n",
      "60000/60000 [==============================] - 7s 121us/sample - loss: 0.0893 - accuracy: 0.9666 - val_loss: 0.0734 - val_accuracy: 0.9742\n",
      "Epoch 48/60\n",
      "60000/60000 [==============================] - 7s 115us/sample - loss: 0.0878 - accuracy: 0.9673 - val_loss: 0.0620 - val_accuracy: 0.9768\n",
      "Epoch 49/60\n",
      "60000/60000 [==============================] - 7s 114us/sample - loss: 0.0833 - accuracy: 0.9687 - val_loss: 0.0705 - val_accuracy: 0.9752\n",
      "Epoch 50/60\n",
      "60000/60000 [==============================] - 7s 111us/sample - loss: 0.0840 - accuracy: 0.9678 - val_loss: 0.0665 - val_accuracy: 0.9744\n",
      "Epoch 51/60\n",
      "60000/60000 [==============================] - 7s 112us/sample - loss: 0.0805 - accuracy: 0.9696 - val_loss: 0.0739 - val_accuracy: 0.9726\n",
      "Epoch 52/60\n",
      "60000/60000 [==============================] - 7s 112us/sample - loss: 0.0810 - accuracy: 0.9688 - val_loss: 0.0677 - val_accuracy: 0.9734\n",
      "Epoch 53/60\n",
      "60000/60000 [==============================] - 7s 121us/sample - loss: 0.0789 - accuracy: 0.9706 - val_loss: 0.0534 - val_accuracy: 0.9792\n",
      "Epoch 54/60\n",
      "60000/60000 [==============================] - 7s 119us/sample - loss: 0.0760 - accuracy: 0.9721 - val_loss: 0.0844 - val_accuracy: 0.9692\n",
      "Epoch 55/60\n",
      "60000/60000 [==============================] - 7s 113us/sample - loss: 0.0777 - accuracy: 0.9715 - val_loss: 0.0551 - val_accuracy: 0.9784\n",
      "Epoch 56/60\n",
      "60000/60000 [==============================] - 7s 117us/sample - loss: 0.0771 - accuracy: 0.9708 - val_loss: 0.0476 - val_accuracy: 0.9826\n",
      "Epoch 57/60\n",
      "60000/60000 [==============================] - 8s 133us/sample - loss: 0.0734 - accuracy: 0.9728 - val_loss: 0.0630 - val_accuracy: 0.9766\n",
      "Epoch 58/60\n",
      "60000/60000 [==============================] - 8s 138us/sample - loss: 0.0729 - accuracy: 0.9719 - val_loss: 0.0481 - val_accuracy: 0.9812\n",
      "Epoch 59/60\n",
      "60000/60000 [==============================] - 9s 145us/sample - loss: 0.0730 - accuracy: 0.9722 - val_loss: 0.0522 - val_accuracy: 0.9774\n",
      "Epoch 60/60\n",
      "60000/60000 [==============================] - 9s 149us/sample - loss: 0.0709 - accuracy: 0.9734 - val_loss: 0.0627 - val_accuracy: 0.9776\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x21d03562388>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(train_images, train_labels, epochs=60, validation_data=(validation_images, validation_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 0s 45us/sample - loss: 0.5108 - accuracy: 0.8879\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.5108350421398878, 0.8879]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(test_images, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}