18-autoencoder-02.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ[\"KERAS_BACKEND\"] = \"tensorflow\"\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from keras.models import Sequential, Model, load_model\n",
    "from keras.layers import Dense, Activation, BatchNormalization, Dropout, Input\n",
    "from IPython.display import SVG\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "import keras\n",
    "from datetime import datetime\n",
    "from sklearn import preprocessing\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.utils import np_utils\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from keras.utils import plot_model\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.utils import shuffle\n",
    "import pickle\n",
    "import glob"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load processed dataset\n",
    "The data we load here are already shuffled and normalized."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train: (966, 26)\n",
      "y_train: (966, 2)\n",
      "x_test: (242, 26)\n",
      "y_test: (242, 2)\n"
     ]
    }
   ],
   "source": [
    "data_directory = 'dataset/3. normalized/'\n",
    "x_train = pickle.load(open(data_directory + \"x_train.p\", \"rb\"))\n",
    "y_train = pickle.load(open(data_directory + \"y_train.p\", \"rb\"))\n",
    "x_test = pickle.load(open(data_directory + \"x_test.p\", \"rb\"))\n",
    "y_test = pickle.load(open(data_directory + \"y_test.p\", \"rb\"))\n",
    "print('x_train: ' + str(x_train.shape))\n",
    "print('y_train: ' + str(y_train.shape))\n",
    "print('x_test: ' + str(x_test.shape))\n",
    "print('y_test: ' + str(y_test.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tunable parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 1300\n",
    "batch_size = 20\n",
    "learning_rate = 0.0001\n",
    "decay = 0.0001\n",
    "dropout = 0.4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Autoencoder #2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "date = datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\")\n",
    "root_directory = 'results/results-autoencoder-02/'\n",
    "autoencoder_1_directory = 'results/results-autoencoder-01/model/'\n",
    "model_directory = root_directory + \"model/\"\n",
    "tensorboard_directory = root_directory + \"tensorboard/\"\n",
    "autoencoder_1_path = ''\n",
    "def create_model(layers):\n",
    "    global date\n",
    "    global autoencoder_1_path\n",
    "\n",
    "    x = Input(shape=(x_train.shape[1],))\n",
    "\n",
    "    y = Dense(units = layers['layer_1'], activation='relu')(x)\n",
    "    y = Dropout(dropout)(y)\n",
    "    \n",
    "    y = Dense(units = x_train.shape[1], activation='sigmoid')(y)\n",
    "    model = None\n",
    "    model = Model(x, y)\n",
    "    \n",
    "    # Create directory\n",
    "    directory = model_directory + date + '/'\n",
    "    if not os.path.exists(directory):\n",
    "        os.makedirs(directory)\n",
    "    \n",
    "    # Write model hyper-parameters\n",
    "    df = pd.DataFrame({\n",
    "        'Layer 1': [layers['layer_1']],\n",
    "        'Dropout': [dropout],\n",
    "        'Learning Rate': [learning_rate],\n",
    "        'Decay': [decay],\n",
    "        'Autoencoder 1': [str(autencoder_1_path)]\n",
    "    })\n",
    "    df.to_csv(model_directory + date + '/params.csv')\n",
    "    \n",
    "    # Write model summary\n",
    "    file2 = open(directory + \"summary.txt\", \"a\")\n",
    "    model.summary(print_fn=lambda line: file2.write(line + '\\n'))\n",
    "    file2.close()\n",
    "\n",
    "    # Write model diagram\n",
    "    plot_model(model, to_file=directory + 'model.png', show_shapes=True, show_layer_names=False)\n",
    "    \n",
    "    # Compile the model\n",
    "    model.compile(optimizer=keras.optimizers.Adam(lr=learning_rate, decay=decay), loss=keras.losses.categorical_crossentropy, metrics=['accuracy'])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare callbacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class KerasClassifierTensorBoard(KerasClassifier):\n",
    "    def fit(self, x, y, **kwargs):\n",
    "        global date\n",
    "        date = datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\")\n",
    "        tensorboard_callback = keras.callbacks.TensorBoard(log_dir=tensorboard_directory + date + '/')\n",
    "        csv_logger = keras.callbacks.CSVLogger(model_directory + date + '/epochs.csv')\n",
    "        model_checkpoint = keras.callbacks.ModelCheckpoint(model_directory + date + '/weights.hdf5', save_best_only=True)\n",
    "        callbacks = [tensorboard_callback, csv_logger, model_checkpoint]\n",
    "        return super(KerasClassifierTensorBoard, self).fit(x, y, callbacks=callbacks, verbose=0, **kwargs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get autoencoder #1 paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "weight_paths = glob.glob(autoencoder_1_directory + '*/weights.hdf5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Grid search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "from search: results/results-autoencoder-01/model\\2018-03-04-20-57-52\\weights.hdf5\n",
      "Fitting 1 folds for each of 4 candidates, totalling 4 fits\n",
      "[CV] layers={'layer_1': 10} ..........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-20-57-52\\weights.hdf5\n",
      " 20/966 [..............................] - ETA: 0s[CV] ........................... layers={'layer_1': 10}, total= 3.8min\n",
      "[CV] layers={'layer_1': 8} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-20-57-52\\weights.hdf5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  3.8min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 20/966 [..............................] - ETA: 0s[CV] ............................ layers={'layer_1': 8}, total= 3.9min\n",
      "[CV] layers={'layer_1': 6} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-20-57-52\\weights.hdf5\n",
      " 20/966 [..............................] - ETA: 0s[CV] ............................ layers={'layer_1': 6}, total= 3.9min\n",
      "[CV] layers={'layer_1': 4} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-20-57-52\\weights.hdf5\n",
      " 20/966 [..............................] - ETA: 0s[CV] ............................ layers={'layer_1': 4}, total= 4.0min\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-20-57-52\\weights.hdf5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed: 15.6min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "from search: results/results-autoencoder-01/model\\2018-03-04-21-04-24\\weights.hdf5\n",
      "Fitting 1 folds for each of 4 candidates, totalling 4 fits\n",
      "[CV] layers={'layer_1': 10} ..........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-04-24\\weights.hdf5\n",
      "800/966 [=======================>......] - ETA: 0s[CV] ........................... layers={'layer_1': 10}, total= 4.2min\n",
      "[CV] layers={'layer_1': 8} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-04-24\\weights.hdf5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  4.2min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 20/966 [..............................] - ETA: 0s[CV] ............................ layers={'layer_1': 8}, total= 4.1min\n",
      "[CV] layers={'layer_1': 6} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-04-24\\weights.hdf5\n",
      " 20/966 [..............................] - ETA: 0s[CV] ............................ layers={'layer_1': 6}, total= 4.3min\n",
      "[CV] layers={'layer_1': 4} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-04-24\\weights.hdf5\n",
      " 20/966 [..............................] - ETA: 0s[CV] ............................ layers={'layer_1': 4}, total= 4.3min\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-04-24\\weights.hdf5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed: 16.9min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "from search: results/results-autoencoder-01/model\\2018-03-04-21-12-11\\weights.hdf5\n",
      "Fitting 1 folds for each of 4 candidates, totalling 4 fits\n",
      "[CV] layers={'layer_1': 10} ..........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-12-11\\weights.hdf5\n",
      "840/966 [=========================>....] - ETA: 0s[CV] ........................... layers={'layer_1': 10}, total= 4.8min\n",
      "[CV] layers={'layer_1': 8} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-12-11\\weights.hdf5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  4.8min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "720/966 [=====================>........] - ETA: 0s[CV] ............................ layers={'layer_1': 8}, total= 4.6min\n",
      "[CV] layers={'layer_1': 6} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-12-11\\weights.hdf5\n",
      "900/966 [==========================>...] - ETA: 0s[CV] ............................ layers={'layer_1': 6}, total= 4.5min\n",
      "[CV] layers={'layer_1': 4} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-12-11\\weights.hdf5\n",
      "900/966 [==========================>...] - ETA: 0s[CV] ............................ layers={'layer_1': 4}, total= 4.5min\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-12-11\\weights.hdf5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed: 18.4min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "from search: results/results-autoencoder-01/model\\2018-03-04-21-19-15\\weights.hdf5\n",
      "Fitting 1 folds for each of 4 candidates, totalling 4 fits\n",
      "[CV] layers={'layer_1': 10} ..........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-19-15\\weights.hdf5\n",
      " 20/966 [..............................] - ETA: 0s[CV] ........................... layers={'layer_1': 10}, total= 4.6min\n",
      "[CV] layers={'layer_1': 8} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-19-15\\weights.hdf5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  4.6min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "966/966 [==============================] - 0s     \n",
      "960/966 [============================>.] - ETA: 0s[CV] ............................ layers={'layer_1': 8}, total= 4.7min\n",
      "[CV] layers={'layer_1': 6} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-19-15\\weights.hdf5\n",
      "680/966 [====================>.........] - ETA: 0s[CV] ............................ layers={'layer_1': 6}, total= 4.8min\n",
      "[CV] layers={'layer_1': 4} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-19-15\\weights.hdf5\n",
      "940/966 [============================>.] - ETA: 0s[CV] ............................ layers={'layer_1': 4}, total= 5.1min\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-19-15\\weights.hdf5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed: 19.2min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "from search: results/results-autoencoder-01/model\\2018-03-04-21-26-59\\weights.hdf5\n",
      "Fitting 1 folds for each of 4 candidates, totalling 4 fits\n",
      "[CV] layers={'layer_1': 10} ..........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-26-59\\weights.hdf5\n",
      "966/966 [==============================] - 0s     \n",
      "[CV] ........................... layers={'layer_1': 10}, total= 4.9min\n",
      "[CV] layers={'layer_1': 8} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-26-59\\weights.hdf5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  4.9min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "860/966 [=========================>....] - ETA: 0s[CV] ............................ layers={'layer_1': 8}, total= 5.0min\n",
      "[CV] layers={'layer_1': 6} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-26-59\\weights.hdf5\n",
      "966/966 [==============================] - 0s     \n",
      "[CV] ............................ layers={'layer_1': 6}, total= 5.1min\n",
      "[CV] layers={'layer_1': 4} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-26-59\\weights.hdf5\n",
      "960/966 [============================>.] - ETA: 0s[CV] ............................ layers={'layer_1': 4}, total= 5.2min\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-26-59\\weights.hdf5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed: 20.2min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "from search: results/results-autoencoder-01/model\\2018-03-04-21-33-55\\weights.hdf5\n",
      "Fitting 1 folds for each of 4 candidates, totalling 4 fits\n",
      "[CV] layers={'layer_1': 10} ..........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-33-55\\weights.hdf5\n",
      "940/966 [============================>.] - ETA: 0s[CV] ........................... layers={'layer_1': 10}, total= 5.0min\n",
      "[CV] layers={'layer_1': 8} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-33-55\\weights.hdf5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  5.0min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "960/966 [============================>.] - ETA: 0s[CV] ............................ layers={'layer_1': 8}, total= 5.2min\n",
      "[CV] layers={'layer_1': 6} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-33-55\\weights.hdf5\n",
      " 20/966 [..............................] - ETA: 0s[CV] ............................ layers={'layer_1': 6}, total= 5.1min\n",
      "[CV] layers={'layer_1': 4} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-33-55\\weights.hdf5\n",
      "860/966 [=========================>....] - ETA: 0s[CV] ............................ layers={'layer_1': 4}, total= 5.3min\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-33-55\\weights.hdf5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed: 20.7min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "from search: results/results-autoencoder-01/model\\2018-03-04-21-41-36\\weights.hdf5\n",
      "Fitting 1 folds for each of 4 candidates, totalling 4 fits\n",
      "[CV] layers={'layer_1': 10} ..........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-41-36\\weights.hdf5\n",
      "820/966 [========================>.....] - ETA: 0s[CV] ........................... layers={'layer_1': 10}, total= 5.3min\n",
      "[CV] layers={'layer_1': 8} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-41-36\\weights.hdf5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  5.3min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 20/966 [..............................] - ETA: 0s[CV] ............................ layers={'layer_1': 8}, total= 5.3min\n",
      "[CV] layers={'layer_1': 6} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-41-36\\weights.hdf5\n",
      "660/966 [===================>..........] - ETA: 0s[CV] ............................ layers={'layer_1': 6}, total= 5.3min\n",
      "[CV] layers={'layer_1': 4} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-41-36\\weights.hdf5\n",
      "900/966 [==========================>...] - ETA: 0s[CV] ............................ layers={'layer_1': 4}, total= 4.9min\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-41-36\\weights.hdf5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed: 20.9min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "from search: results/results-autoencoder-01/model\\2018-03-04-21-48-57\\weights.hdf5\n",
      "Fitting 1 folds for each of 4 candidates, totalling 4 fits\n",
      "[CV] layers={'layer_1': 10} ..........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-48-57\\weights.hdf5\n",
      " 20/966 [..............................] - ETA: 0s[CV] ........................... layers={'layer_1': 10}, total= 4.8min\n",
      "[CV] layers={'layer_1': 8} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-48-57\\weights.hdf5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  4.8min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "840/966 [=========================>....] - ETA: 0s[CV] ............................ layers={'layer_1': 8}, total= 4.9min\n",
      "[CV] layers={'layer_1': 6} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-48-57\\weights.hdf5\n",
      "966/966 [==============================] - 0s     \n",
      "680/966 [====================>.........] - ETA: 0s[CV] ............................ layers={'layer_1': 6}, total= 4.9min\n",
      "[CV] layers={'layer_1': 4} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-48-57\\weights.hdf5\n",
      "660/966 [===================>..........] - ETA: 0s[CV] ............................ layers={'layer_1': 4}, total= 5.2min\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-48-57\\weights.hdf5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed: 19.8min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "from search: results/results-autoencoder-01/model\\2018-03-04-21-56-46\\weights.hdf5\n",
      "Fitting 1 folds for each of 4 candidates, totalling 4 fits\n",
      "[CV] layers={'layer_1': 10} ..........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-56-46\\weights.hdf5\n",
      "860/966 [=========================>....] - ETA: 0s[CV] ........................... layers={'layer_1': 10}, total= 4.9min\n",
      "[CV] layers={'layer_1': 8} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-56-46\\weights.hdf5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  4.9min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "960/966 [============================>.] - ETA: 0s[CV] ............................ layers={'layer_1': 8}, total= 4.8min\n",
      "[CV] layers={'layer_1': 6} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-56-46\\weights.hdf5\n",
      "720/966 [=====================>........] - ETA: 0s[CV] ............................ layers={'layer_1': 6}, total= 5.1min\n",
      "[CV] layers={'layer_1': 4} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-56-46\\weights.hdf5\n",
      "920/966 [===========================>..] - ETA: 0s[CV] ............................ layers={'layer_1': 4}, total= 4.9min\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-21-56-46\\weights.hdf5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed: 19.7min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "from search: results/results-autoencoder-01/model\\2018-03-04-22-03-55\\weights.hdf5\n",
      "Fitting 1 folds for each of 4 candidates, totalling 4 fits\n",
      "[CV] layers={'layer_1': 10} ..........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-22-03-55\\weights.hdf5\n",
      "740/966 [=====================>........] - ETA: 0s[CV] ........................... layers={'layer_1': 10}, total= 5.0min\n",
      "[CV] layers={'layer_1': 8} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-22-03-55\\weights.hdf5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  5.0min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "900/966 [==========================>...] - ETA: 0s[CV] ............................ layers={'layer_1': 8}, total= 4.8min\n",
      "[CV] layers={'layer_1': 6} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-22-03-55\\weights.hdf5\n",
      "820/966 [========================>.....] - ETA: 0s[CV] ............................ layers={'layer_1': 6}, total= 4.8min\n",
      "[CV] layers={'layer_1': 4} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-22-03-55\\weights.hdf5\n",
      "940/966 [============================>.] - ETA: 0s[CV] ............................ layers={'layer_1': 4}, total= 4.8min\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-22-03-55\\weights.hdf5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed: 19.5min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "from search: results/results-autoencoder-01/model\\2018-03-04-22-11-48\\weights.hdf5\n",
      "Fitting 1 folds for each of 4 candidates, totalling 4 fits\n",
      "[CV] layers={'layer_1': 10} ..........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-22-11-48\\weights.hdf5\n",
      "920/966 [===========================>..] - ETA: 0s[CV] ........................... layers={'layer_1': 10}, total= 4.8min\n",
      "[CV] layers={'layer_1': 8} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-22-11-48\\weights.hdf5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  4.8min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "800/966 [=======================>......] - ETA: 0s[CV] ............................ layers={'layer_1': 8}, total= 4.9min\n",
      "[CV] layers={'layer_1': 6} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-22-11-48\\weights.hdf5\n",
      "960/966 [============================>.] - ETA: 0s[CV] ............................ layers={'layer_1': 6}, total= 4.8min\n",
      "[CV] layers={'layer_1': 4} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-22-11-48\\weights.hdf5\n",
      "860/966 [=========================>....] - ETA: 0s[CV] ............................ layers={'layer_1': 4}, total= 4.7min\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-22-11-48\\weights.hdf5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed: 19.2min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "from search: results/results-autoencoder-01/model\\2018-03-04-22-18-55\\weights.hdf5\n",
      "Fitting 1 folds for each of 4 candidates, totalling 4 fits\n",
      "[CV] layers={'layer_1': 10} ..........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-22-18-55\\weights.hdf5\n",
      "740/966 [=====================>........] - ETA: 0s[CV] ........................... layers={'layer_1': 10}, total= 5.1min\n",
      "[CV] layers={'layer_1': 8} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-22-18-55\\weights.hdf5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  5.1min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "840/966 [=========================>....] - ETA: 0s[CV] ............................ layers={'layer_1': 8}, total= 5.4min\n",
      "[CV] layers={'layer_1': 6} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-22-18-55\\weights.hdf5\n",
      "580/966 [=================>............] - ETA: 0s[CV] ............................ layers={'layer_1': 6}, total= 5.4min\n",
      "[CV] layers={'layer_1': 4} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-22-18-55\\weights.hdf5\n",
      "780/966 [=======================>......] - ETA: 0s[CV] ............................ layers={'layer_1': 4}, total= 5.6min\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-22-18-55\\weights.hdf5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed: 21.6min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "from search: results/results-autoencoder-01/model\\2018-03-04-22-26-57\\weights.hdf5\n",
      "Fitting 1 folds for each of 4 candidates, totalling 4 fits\n",
      "[CV] layers={'layer_1': 10} ..........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-22-26-57\\weights.hdf5\n",
      "640/966 [==================>...........] - ETA: 0s[CV] ........................... layers={'layer_1': 10}, total= 5.4min\n",
      "[CV] layers={'layer_1': 8} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-22-26-57\\weights.hdf5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  5.4min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "780/966 [=======================>......] - ETA: 0s[CV] ............................ layers={'layer_1': 8}, total= 5.4min\n",
      "[CV] layers={'layer_1': 6} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-22-26-57\\weights.hdf5\n",
      "720/966 [=====================>........] - ETA: 0s[CV] ............................ layers={'layer_1': 6}, total= 5.7min\n",
      "[CV] layers={'layer_1': 4} ...........................................\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-22-26-57\\weights.hdf5\n",
      "560/966 [================>.............] - ETA: 0s[CV] ............................ layers={'layer_1': 4}, total= 5.7min\n",
      "from model: results/results-autoencoder-01/model\\2018-03-04-22-26-57\\weights.hdf5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed: 22.2min finished\n"
     ]
    }
   ],
   "source": [
    "for weight_path in weight_paths:\n",
    "    autoencoder_1 = load_model(weight_path)\n",
    "    autoencoder_1_output = autoencoder_1.predict(x_train)\n",
    "    autoencoder_1_output_test = autoencoder_1.predict(x_test)\n",
    "    autencoder_1_path = weight_path\n",
    "    \n",
    "    model = None\n",
    "    model = KerasClassifierTensorBoard(build_fn=create_model, epochs=epochs, batch_size=batch_size)\n",
    "\n",
    "    layers = [\n",
    "        {'layer_1': 10},\n",
    "        {'layer_1': 8},\n",
    "        {'layer_1': 6},\n",
    "        {'layer_1': 4}\n",
    "    ]\n",
    "    param_grid = dict(\n",
    "        layers=layers\n",
    "    )\n",
    "\n",
    "    grid = GridSearchCV(estimator = model, param_grid = param_grid, error_score=0, verbose=2, cv=[(slice(None), slice(None))], n_jobs=1, fit_params=dict(validation_data=(autoencoder_1_output_test, autoencoder_1_output_test)))\n",
    "    grid_result = grid.fit(autoencoder_1_output, autoencoder_1_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tensorflow]",
   "language": "python",
   "name": "conda-env-tensorflow-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}