low_clearance_detector_200426.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"accelerator":"GPU","colab":{"name":"low_clearance_detector_200426.ipynb","provenance":[{"file_id":"1SNJsRpyKiaxF2ni-D__uE7JZe8O_sdHU","timestamp":1587946624718},{"file_id":"1UQFT_I48SivoCuP-b3gMZ5nK7hQejfHK","timestamp":1587938990186},{"file_id":"https://github.com/tensorflow/models/blob/master/research/object_detection/object_detection_tutorial.ipynb","timestamp":1585790513367},{"file_id":"1LNYL6Zsn9Xlil2CVNOTsgDZQSBKeOjCh","timestamp":1566498233247},{"file_id":"/piper/depot/google3/third_party/tensorflow_models/object_detection/object_detection_tutorial.ipynb?workspaceId=markdaoust:copybara_AFABFE845DCD573AD3D43A6BAFBE77D4_0::citc","timestamp":1566488313397},{"file_id":"/piper/depot/google3/third_party/py/tensorflow_docs/g3doc/en/r2/tutorials/generative/object_detection_tutorial.ipynb?workspaceId=markdaoust:copybara_AFABFE845DCD573AD3D43A6BAFBE77D4_0::citc","timestamp":1566145894046},{"file_id":"1nBPoWynOV0auSIy40eQcBIk9C6YRSkI8","timestamp":1566145841085},{"file_id":"/piper/depot/google3/third_party/tensorflow_models/object_detection/object_detection_tutorial.ipynb?workspaceId=markdaoust:copybara_AFABFE845DCD573AD3D43A6BAFBE77D4_0::citc","timestamp":1556295408037},{"file_id":"1layerger-51XwWOwYMY_5zHaCavCeQkO","timestamp":1556214267924},{"file_id":"/piper/depot/google3/third_party/tensorflow_models/object_detection/object_detection_tutorial.ipynb?workspaceId=markdaoust:copybara_AFABFE845DCD573AD3D43A6BAFBE77D4_0::citc","timestamp":1556207836484},{"file_id":"1w6mqQiNV3liPIX70NOgitOlDF1_4sRMw","timestamp":1556154824101},{"file_id":"https://github.com/tensorflow/models/blob/master/research/object_detection/object_detection_tutorial.ipynb","timestamp":1556150293326}],"private_outputs":true,"collapsed_sections":[],"last_runtime":{"build_target":"//learning/brain/python/client:colab_notebook","kind":"private"},"toc_visible":true},"kernelspec":{"name":"python3","display_name":"Python 3"}},"cells":[{"cell_type":"markdown","metadata":{"colab_type":"text","id":"V8-yl-s-WKMG"},"source":["# Custom Object Detection - Low Clearance Sign Detector"]},{"cell_type":"code","metadata":{"id":"_pyc2PL32lEs","colab_type":"code","colab":{}},"source":["# clone low-clearance-detector repo\n","!git clone https://github.com/lbborkowski/low-clearance-detector.git"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"colab_type":"text","id":"kFSqkTCdWKMI"},"source":["## Setup"]},{"cell_type":"markdown","metadata":{"colab_type":"text","id":"p3UGXxUii5Ym"},"source":["### Install"]},{"cell_type":"markdown","metadata":{"id":"9ddaQ5hRX-fe","colab_type":"text"},"source":["Install tensorflow 1.* -- tensorflow object detection api is not compatible with tensorflow 2.*"]},{"cell_type":"code","metadata":{"id":"yBkS3K0iVCCM","colab_type":"code","colab":{}},"source":["pip install tensorflow-gpu==1.*"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"gMvI3vo-Z-GV","colab_type":"text"},"source":["Downgrade to numpy 1.17.4 to avoid an error during evaluation\n","\n","Requires a restart of the runtime before continuing through the notebook - click **RESTART RUNTIME** in output below"]},{"cell_type":"code","metadata":{"id":"Kc2Bx6bRaABb","colab_type":"code","colab":{}},"source":["!pip install numpy==1.17.4"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"colab_type":"text","id":"n_ap_s9ajTHH"},"source":["Make sure you have `pycocotools` installed"]},{"cell_type":"code","metadata":{"colab_type":"code","id":"Bg8ZyA47i3pY","colab":{}},"source":["!pip install pycocotools"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"colab_type":"text","id":"-vsOL3QR6kqs"},"source":["Get `tensorflow/models` or `cd` to parent directory of the repository"]},{"cell_type":"code","metadata":{"colab_type":"code","id":"ykA0c-om51s1","colab":{}},"source":["import os\n","import pathlib\n","\n","\n","if \"models\" in pathlib.Path.cwd().parts:\n","  while \"models\" in pathlib.Path.cwd().parts:\n","    os.chdir('..')\n","elif not pathlib.Path('models').exists():\n","  !git clone --depth 1 https://github.com/tensorflow/models"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"colab_type":"text","id":"O219m6yWAj9l"},"source":["Compile protobufs and install the object_detection package"]},{"cell_type":"code","metadata":{"colab_type":"code","id":"PY41vdYYNlXc","colab":{}},"source":["%%bash\n","cd models/research/\n","protoc object_detection/protos/*.proto --python_out=."],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"colab_type":"code","id":"s62yJyQUcYbp","colab":{}},"source":["%%bash \n","cd models/research\n","pip install ."],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"9A672DOWamIv","colab_type":"text"},"source":["Add necessary libraries to PYTHONPATH"]},{"cell_type":"code","metadata":{"id":"FIig7e3Cv4js","colab_type":"code","colab":{}},"source":["%set_env PYTHONPATH=$PYTHONPATH:/content/models/research:/content/models/research/slim"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"colab_type":"text","id":"LBdjK2G5ywuc"},"source":["### Imports"]},{"cell_type":"code","metadata":{"colab_type":"code","id":"hV4P5gyTWKMI","colab":{}},"source":["import numpy as np\n","import os\n","import six.moves.urllib as urllib\n","import sys\n","import tarfile\n","import tensorflow as tf\n","import zipfile\n","\n","from collections import defaultdict\n","from io import StringIO\n","from matplotlib import pyplot as plt\n","from PIL import Image\n","from IPython.display import display"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"colab_type":"text","id":"r5FNuiRPWKMN"},"source":["Import the object detection module"]},{"cell_type":"code","metadata":{"colab_type":"code","id":"4-IMl4b6BdGO","colab":{}},"source":["from object_detection.utils import ops as utils_ops\n","from object_detection.utils import label_map_util\n","from object_detection.utils import visualization_utils as vis_util"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"colab_type":"text","id":"RYPCiag2iz_q"},"source":["Patches:"]},{"cell_type":"code","metadata":{"colab_type":"code","id":"mF-YlMl8c_bM","colab":{}},"source":["# patch tf1 into `utils.ops`\n","utils_ops.tf = tf.compat.v1\n","\n","# Patch the location of gfile\n","tf.gfile = tf.io.gfile"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"lwXWMlXzgNuA","colab_type":"text"},"source":["## Training"]},{"cell_type":"markdown","metadata":{"id":"x_Bz36jG7_Cg","colab_type":"text"},"source":["### Create TFRecords for train and test images"]},{"cell_type":"code","metadata":{"id":"aAuUnMyb4Yy7","colab_type":"code","colab":{}},"source":["%cd /content/low-clearance-detector/"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"75MxFapr4OdG","colab_type":"code","colab":{}},"source":["# Create train TFRecord\n","!python generate_tfrecord.py --label='low-clearance-sign' --csv_input=/content/low-clearance-detector/annotations/train_labels.csv --img_path=/content/low-clearance-detector/images/train --output_path=/content/low-clearance-detector/annotations/train.record"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"0W6-CXaP5SaD","colab_type":"code","colab":{}},"source":["# Create test TFRecord\n","!python generate_tfrecord.py --label='low-clearance-sign' --csv_input=/content/low-clearance-detector/annotations/test_labels.csv --img_path=/content/low-clearance-detector/images/test --output_path=/content/low-clearance-detector/annotations/test.record"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"-ErExTtagVAy","colab_type":"text"},"source":["### Train custom object detection model\n","Train the model for 6500 steps"]},{"cell_type":"code","metadata":{"id":"oKTkJYRlNKSX","colab_type":"code","colab":{}},"source":["%cd /content/low-clearance-detector\n","!python model_main.py --pipeline_config_path=training/faster_rcnn_inception_v2_coco_lcsd.config --model_dir=training/ --num_train_steps=6500 --alsologtostderr"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"GvyXtzq9c0k9","colab_type":"text"},"source":["#### Setup and run tensorboard\n","Visualize training losses and evaluation results"]},{"cell_type":"code","metadata":{"id":"cbST3fa1cpW-","colab_type":"code","colab":{}},"source":["# Uninstall tensorboard plugin to avoid conflict\n","pip uninstall -y tensorboard-plugin-wit"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"CbmvQKl5csvE","colab_type":"code","colab":{}},"source":["from tensorflow import summary\n","%load_ext tensorboard"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"WaDTOqTHcvfT","colab_type":"code","colab":{}},"source":["%tensorboard --logdir training/"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"uVmBnBNkj5JU","colab_type":"text"},"source":["### Export trained low clearance sign detection model"]},{"cell_type":"code","metadata":{"id":"qAO_XU5ZzCzp","colab_type":"code","colab":{}},"source":["%cd /content/low-clearance-detector"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"CoDkhrGUj_j7","colab_type":"code","colab":{}},"source":["!python /content/models/research/object_detection/export_inference_graph.py \\\n","    --input_type=image_tensor \\\n","    --pipeline_config_path=training/faster_rcnn_inception_v2_coco_lcsd.config \\\n","    --output_directory=trained-model \\\n","    --trained_checkpoint_prefix=training/model.ckpt-6500"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"dXEimgrRnfGe","colab_type":"text"},"source":["## Inference"]},{"cell_type":"markdown","metadata":{"id":"fRaBKXAScrKe","colab_type":"text"},"source":["### Imports"]},{"cell_type":"markdown","metadata":{"id":"yheZE3voc0R8","colab_type":"text"},"source":["#### Object detection imports\n","Imports from the object detection module"]},{"cell_type":"code","metadata":{"id":"8doBUW1CyMT6","colab_type":"code","colab":{}},"source":["%cd /content/models/research/object_detection"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"Ok4APcd1nwae","colab_type":"code","colab":{}},"source":["from utils import label_map_util\n","\n","from utils import visualization_utils as vis_util"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"kD41NIfqc3kw","colab_type":"text"},"source":["### Model preparation "]},{"cell_type":"markdown","metadata":{"id":"-_2n5Elnc6nP","colab_type":"text"},"source":["#### Variables\n","\n","Any model exported using the \"export_inference_graph.py\" tool can be loaded here simply by changing \"PATH_TO_FROZEN_GRAPH\" to point to a new .pb file"]},{"cell_type":"code","metadata":{"id":"vbCWyuzCnzHP","colab_type":"code","colab":{}},"source":["# Path to frozen detection graph. This is the actual model that is used for the object detection.\n","PATH_TO_FROZEN_GRAPH = '/content/low-clearance-detector/trained-model/frozen_inference_graph.pb'\n","\n","# List of the strings that is used to add correct label for each box.\n","PATH_TO_LABELS = '/content/low-clearance-detector/annotations/label_map.pbtxt'"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"o9goacMAdHYI","colab_type":"text"},"source":["#### Load a (frozen) tensorflow model into memory"]},{"cell_type":"code","metadata":{"id":"5wbOQ0CNob4b","colab_type":"code","colab":{}},"source":["detection_graph = tf.Graph()\n","with detection_graph.as_default():\n","  od_graph_def = tf.GraphDef()\n","  with tf.gfile.GFile(PATH_TO_FROZEN_GRAPH, 'rb') as fid:\n","    serialized_graph = fid.read()\n","    od_graph_def.ParseFromString(serialized_graph)\n","    tf.import_graph_def(od_graph_def, name='')"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"H2q6btrzdK2w","colab_type":"text"},"source":["#### Loading label map"]},{"cell_type":"code","metadata":{"id":"SEU9uGKYodMb","colab_type":"code","colab":{}},"source":["category_index = label_map_util.create_category_index_from_labelmap(PATH_TO_LABELS, use_display_name=True)\n","print(category_index)"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"SHQm3vYsdV5j","colab_type":"text"},"source":["#### Helper code"]},{"cell_type":"code","metadata":{"id":"u4CS_XnAoecX","colab_type":"code","colab":{}},"source":["def load_image_into_numpy_array(image):\n","  (im_width, im_height) = image.size\n","  return np.array(image.getdata()).reshape(\n","      (im_height, im_width, 3)).astype(np.uint8)"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"kXrkaMMBdY40","colab_type":"text"},"source":["#### Inference function\n","Perform inference one image at a time"]},{"cell_type":"code","metadata":{"id":"54ZmWuJ3og71","colab_type":"code","colab":{}},"source":["def run_inference_for_single_image(image, graph):\n","  with graph.as_default():\n","    with tf.Session() as sess:\n","      # Get handles to input and output tensors\n","      ops = tf.get_default_graph().get_operations()\n","      all_tensor_names = {output.name for op in ops for output in op.outputs}\n","      tensor_dict = {}\n","      for key in [\n","          'num_detections', 'detection_boxes', 'detection_scores',\n","          'detection_classes', 'detection_masks'\n","      ]:\n","        tensor_name = key + ':0'\n","        if tensor_name in all_tensor_names:\n","          tensor_dict[key] = tf.get_default_graph().get_tensor_by_name(\n","              tensor_name)\n","      if 'detection_masks' in tensor_dict:\n","        # The following processing is only for single image\n","        detection_boxes = tf.squeeze(tensor_dict['detection_boxes'], [0])\n","        detection_masks = tf.squeeze(tensor_dict['detection_masks'], [0])\n","        # Reframe is required to translate mask from box coordinates to image coordinates and fit the image size.\n","        real_num_detection = tf.cast(tensor_dict['num_detections'][0], tf.int32)\n","        detection_boxes = tf.slice(detection_boxes, [0, 0], [real_num_detection, -1])\n","        detection_masks = tf.slice(detection_masks, [0, 0, 0], [real_num_detection, -1, -1])\n","        detection_masks_reframed = utils_ops.reframe_box_masks_to_image_masks(\n","            detection_masks, detection_boxes, image.shape[0], image.shape[1])\n","        detection_masks_reframed = tf.cast(\n","            tf.greater(detection_masks_reframed, 0.5), tf.uint8)\n","        # Follow the convention by adding back the batch dimension\n","        tensor_dict['detection_masks'] = tf.expand_dims(\n","            detection_masks_reframed, 0)\n","      image_tensor = tf.get_default_graph().get_tensor_by_name('image_tensor:0')\n","\n","      # Run inference\n","      output_dict = sess.run(tensor_dict,\n","                             feed_dict={image_tensor: np.expand_dims(image, 0)})\n","\n","      # all outputs are float32 numpy arrays, so convert types as appropriate\n","      output_dict['num_detections'] = int(output_dict['num_detections'][0])\n","      output_dict['detection_classes'] = output_dict[\n","          'detection_classes'][0].astype(np.uint8)\n","      output_dict['detection_boxes'] = output_dict['detection_boxes'][0]\n","      output_dict['detection_scores'] = output_dict['detection_scores'][0]\n","      if 'detection_masks' in output_dict:\n","        output_dict['detection_masks'] = output_dict['detection_masks'][0]\n","  return output_dict"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"jzvE4xStddRz","colab_type":"text"},"source":["### Detection - validation images\n","Detect low clearance signs in validation image set"]},{"cell_type":"code","metadata":{"id":"6FclsOd_oja_","colab_type":"code","colab":{}},"source":["import os\n","import glob\n","\n","PATH_TO_TEST_IMAGES_DIR = '/content/low-clearance-detector/images/valid'\n","TEST_IMAGE_PATHS = glob.glob(os.path.join(PATH_TO_TEST_IMAGES_DIR, \"*.*\"))\n","print(TEST_IMAGE_PATHS)\n","\n","# Size, in inches, of the output images\n","IMAGE_SIZE = (16, 16)"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"JRRU0iMRdrhH","colab_type":"text"},"source":["#### Perform inference on all validation images\n","Loop over validation set and output images containing detected low clearance signs with its location denoted using a bounding box"]},{"cell_type":"code","metadata":{"id":"Qt5BHVJuolpR","colab_type":"code","colab":{}},"source":["%matplotlib inline\n","for image_path in TEST_IMAGE_PATHS:\n","  image = Image.open(image_path)\n","  # the array based representation of the image will be used later in order to prepare the\n","  # result image with boxes and labels on it.\n","  image_np = load_image_into_numpy_array(image)\n","  # Expand dimensions since the model expects images to have shape: [1, None, None, 3]\n","  image_np_expanded = np.expand_dims(image_np, axis=0)\n","  # Actual detection.\n","  output_dict = run_inference_for_single_image(image_np, detection_graph)\n","  # Visualization of the results of a detection.\n","  vis_util.visualize_boxes_and_labels_on_image_array(\n","      image_np,\n","      output_dict['detection_boxes'],\n","      output_dict['detection_classes'],\n","      output_dict['detection_scores'],\n","      category_index,\n","      instance_masks=output_dict.get('detection_masks'),\n","      use_normalized_coordinates=True,\n","      line_thickness=8)\n","  plt.figure(figsize=IMAGE_SIZE)\n","  plt.axis('off')\n","  plt.imshow(image_np)\n","  print(output_dict['detection_scores'][0], output_dict['detection_scores'][1])"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"uPoo042dajgW","colab_type":"code","colab":{}},"source":[""],"execution_count":0,"outputs":[]}]}