experiments/evaluate-random.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "#%load_ext line_profiler\n",
    "\n",
    "import tensorflow as tf\n",
    "tf.logging.set_verbosity(tf.logging.INFO)\n",
    "sess_config = tf.ConfigProto()\n",
    "\n",
    "import sys\n",
    "import os\n",
    "\n",
    "COCO_DATA = '../data/coco'\n",
    "MASK_RCNN_MODEL_PATH = '../lib/Mask_RCNN/'\n",
    "SIAMESE_MASK_RCNN_PATH = '../lib/'\n",
    "\n",
    "if MASK_RCNN_MODEL_PATH not in sys.path:\n",
    "    sys.path.append(MASK_RCNN_MODEL_PATH)\n",
    "if SIAMESE_MASK_RCNN_PATH not in sys.path:\n",
    "    sys.path.append(SIAMESE_MASK_RCNN_PATH)\n",
    "    \n",
    "from samples.coco import coco\n",
    "from mrcnn import utils\n",
    "from mrcnn import model as modellib\n",
    "from mrcnn import visualize\n",
    "    \n",
    "import utils as siamese_utils\n",
    "import model as siamese_model\n",
    "import config as siamese_config\n",
    "   \n",
    "import time\n",
    "import datetime\n",
    "import random\n",
    "import numpy as np\n",
    "import skimage.io\n",
    "import imgaug\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import OrderedDict\n",
    "\n",
    "# Root directory of the project\n",
    "ROOT_DIR = os.getcwd()\n",
    "\n",
    "# Directory to save logs and trained model\n",
    "MODEL_DIR = os.path.join(ROOT_DIR, \"logs\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load COCO/val dataset\n",
    "test_dataset = siamese_utils.IndexedCocoDataset()\n",
    "# coco_val.set_active_classes(train_classes)\n",
    "coco_test_object = test_dataset.load_coco(COCO_DATA, subset=\"val\", year=\"2017\", return_coco=True)\n",
    "test_dataset.prepare()\n",
    "test_dataset.build_indices()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ParallelTrainConfig(siamese_config.Config):\n",
    "    # Set batch size to 1 since we'll be running inference on\n",
    "    # one image at a time. Batch size = GPU_COUNT * IMAGES_PER_GPU\n",
    "    GPU_COUNT = 1\n",
    "    IMAGES_PER_GPU = 1\n",
    "    NUM_CLASSES = 1 + 1\n",
    "    NAME = 'coco'\n",
    "    EXPERIMENT = 'evaluation'\n",
    "    \n",
    "    # Large image sizes\n",
    "    TARGET_MAX_DIM = 192\n",
    "    TARGET_MIN_DIM = 150\n",
    "    IMAGE_MIN_DIM = 800\n",
    "    IMAGE_MAX_DIM = 1024\n",
    "    # Large model size\n",
    "    FPN_CLASSIF_FC_LAYERS_SIZE = 1024\n",
    "    FPN_FEATUREMAPS = 256\n",
    "    # Large number of rois at all stages\n",
    "    RPN_ANCHOR_STRIDE = 1\n",
    "    RPN_TRAIN_ANCHORS_PER_IMAGE = 256\n",
    "    POST_NMS_ROIS_TRAINING = 2000\n",
    "    POST_NMS_ROIS_INFERENCE = 1000\n",
    "    TRAIN_ROIS_PER_IMAGE = 200\n",
    "    DETECTION_MAX_INSTANCES = 100\n",
    "    MAX_GT_INSTANCES = 100\n",
    "    \n",
    "class SmallTrainConfig(siamese_config.Config):\n",
    "    # Set batch size to 1 since we'll be running inference on\n",
    "    # one image at a time. Batch size = GPU_COUNT * IMAGES_PER_GPU\n",
    "    GPU_COUNT = 1\n",
    "    IMAGES_PER_GPU = 1\n",
    "    NUM_CLASSES = 1 + 1\n",
    "    NAME = 'coco'\n",
    "    EXPERIMENT = 'evaluation'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = ParallelTrainConfig()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Provide training schedule of the model\n",
    "# When evaluationg intermediate steps the tranining schedule must be provided\n",
    "short_train_schedule = OrderedDict()\n",
    "short_train_schedule[1] = {\"learning_rate\": config.LEARNING_RATE, \"layers\": \"heads\"}\n",
    "short_train_schedule[120] = {\"learning_rate\": config.LEARNING_RATE, \"layers\": \"all\"}\n",
    "short_train_schedule[160] = {\"learning_rate\": config.LEARNING_RATE/10, \"layers\": \"all\"}\n",
    "\n",
    "# Provide training schedule of the model\n",
    "# When evaluationg intermediate steps the tranining schedule must be provided\n",
    "long_train_schedule = OrderedDict()\n",
    "long_train_schedule[1] = {\"learning_rate\": config.LEARNING_RATE, \"layers\": \"heads\"}\n",
    "long_train_schedule[240] = {\"learning_rate\": config.LEARNING_RATE, \"layers\": \"all\"}\n",
    "long_train_schedule[320] = {\"learning_rate\": config.LEARNING_RATE/10, \"layers\": \"all\"}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Siamese Mask R-CNN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### i-split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "index = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "one_shot_classes = np.array([4*i + index for i in range(20)])\n",
    "train_classes = np.array(range(1,81))[np.array([i not in one_shot_classes for i in range(1,81)])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint = '../checkpoints/large_siamese_mrcnn_coco_i{}_0160.h5'.format(index)\n",
    "config = ParallelTrainConfig()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### one-shot classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluate on one-shot classes\n",
    "test_dataset.ACTIVE_CLASSES = one_shot_classes\n",
    "\n",
    "# Load and evaluate models\n",
    "\n",
    "# Create model object in inference mode.\n",
    "model = siamese_model.SiameseMaskRCNN(mode=\"inference\", model_dir=MODEL_DIR, config=config)\n",
    "model.load_checkpoint(checkpoint, training_schedule=short_train_schedule)\n",
    "# Evaluate only active classes\n",
    "active_class_idx = np.array(test_dataset.ACTIVE_CLASSES) - 1\n",
    "\n",
    "# Evaluate on the validation set\n",
    "print('evaluating five times')\n",
    "\n",
    "for run in range(5):\n",
    "    print('\\t*** Evaluation run {} ***'.format(run + 1))\n",
    "\n",
    "    # Evaluate on the validation set\n",
    "    siamese_utils.evaluate_dataset(model, test_dataset, coco_test_object, eval_type=[\"bbox\", \"segm\"], \n",
    "                     dataset_type='coco', limit=0, image_ids=None, \n",
    "                     class_index=active_class_idx, verbose=1,\n",
    "                     random_detections=True)\n",
    "    \n",
    "    print('\\n' * 5, end='')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}