deep_residual_network.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "deep-residual-network.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "9a14b9a2dab8451bbd0632c6c0b8fae7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_83a16fdffadc42b3bbac421bdd074638",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c551fe0437a644fb9b0393c1a25c1f73",
              "IPY_MODEL_c2e5cfd12a4e42f8be2fe4f01019c80a",
              "IPY_MODEL_79f5a3177b3c4bcbbda7d52c34173dd0"
            ]
          }
        },
        "83a16fdffadc42b3bbac421bdd074638": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c551fe0437a644fb9b0393c1a25c1f73": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_3e5df088779f42aca10feef6044d8f2b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a0be0e5aaa7a4564b03974000672f9c7"
          }
        },
        "c2e5cfd12a4e42f8be2fe4f01019c80a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2976c0297c884eaaa903baf203d62065",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 170498071,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 170498071,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0c47f199f49e492a839988b5ed875ec0"
          }
        },
        "79f5a3177b3c4bcbbda7d52c34173dd0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_df97a6e9eec846399399802bbf3c34c5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 170499072/? [00:11&lt;00:00, 17049209.21it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d5cbffe0afc243798c21650eed0b2ccf"
          }
        },
        "3e5df088779f42aca10feef6044d8f2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a0be0e5aaa7a4564b03974000672f9c7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2976c0297c884eaaa903baf203d62065": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0c47f199f49e492a839988b5ed875ec0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "df97a6e9eec846399399802bbf3c34c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d5cbffe0afc243798c21650eed0b2ccf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GBNyn6TAMA0O"
      },
      "source": [
        "# deep residual network\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hZMu7z3qcfUX"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FpvUOuC3j27n"
      },
      "source": [
        "%tensorflow_version 2.x\n",
        "import tensorflow as tf\n",
        "print(\"Tensorflow version \" + tf.__version__)\n",
        "\n",
        "try:\n",
        "  tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection\n",
        "  print('Running on TPU ', tpu.cluster_spec().as_dict()['worker'])\n",
        "except ValueError:\n",
        "  raise BaseException('ERROR: Not connected to a TPU runtime; please see the previous cell in this notebook for instructions!')\n",
        "\n",
        "tf.config.experimental_connect_to_cluster(tpu)\n",
        "tf.tpu.experimental.initialize_tpu_system(tpu)\n",
        "tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 534,
          "referenced_widgets": [
            "9a14b9a2dab8451bbd0632c6c0b8fae7",
            "83a16fdffadc42b3bbac421bdd074638",
            "c551fe0437a644fb9b0393c1a25c1f73",
            "c2e5cfd12a4e42f8be2fe4f01019c80a",
            "79f5a3177b3c4bcbbda7d52c34173dd0",
            "3e5df088779f42aca10feef6044d8f2b",
            "a0be0e5aaa7a4564b03974000672f9c7",
            "2976c0297c884eaaa903baf203d62065",
            "0c47f199f49e492a839988b5ed875ec0",
            "df97a6e9eec846399399802bbf3c34c5",
            "d5cbffe0afc243798c21650eed0b2ccf"
          ]
        },
        "id": "xDmVvn1wLjAk",
        "outputId": "586f07a9-9796-4a69-c69c-3e2394e544c9"
      },
      "source": [
        "# code by Jinshun Lin\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "\n",
        "# Device configuration\n",
        "#device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "\n",
        "# Hyper-parameters\n",
        "num_epochs = 80\n",
        "batch_size = 100\n",
        "learning_rate = 0.001\n",
        "\n",
        "# Image preprocessing modules\n",
        "transform = transforms.Compose(\n",
        "    [transforms.Pad(4),\n",
        "     transforms.RandomHorizontalFlip(),\n",
        "     transforms.RandomCrop(32),\n",
        "     transforms.ToTensor()]\n",
        ")\n",
        "# CIFAR-10 dataset\n",
        "train_dataset = torchvision.datasets.CIFAR10(root='../../data/',\n",
        "                                             train=True,\n",
        "                                             transform=transform,\n",
        "                                             download=True)\n",
        "test_dataset = torchvision.datasets.CIFAR10(root='../../data/',\n",
        "                                            train=False,\n",
        "                                            transform=transforms.ToTensor())\n",
        "# Data loader\n",
        "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n",
        "                                           batch_size=batch_size,\n",
        "                                           shuffle=True)\n",
        "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n",
        "                                          batch_size=batch_size,\n",
        "                                          shuffle=False)\n",
        "\n",
        "# 3x3 convolution\n",
        "def conv3x3(in_channels, out_channels, stride=1):\n",
        "    return nn.Conv2d(in_channels, out_channels, kernel_size=3,\n",
        "                     stride=stride, padding=1, bias=False)\n",
        "# Residual block\n",
        "class ResidualBlock(nn.Module):\n",
        "    def __init__(self, in_channels, out_channels, stride=1, downsample=None):\n",
        "        super(ResidualBlock, self).__init__()\n",
        "        self.conv1 = conv3x3(in_channels, out_channels, stride)\n",
        "        self.bn1 = nn.BatchNorm2d(out_channels)\n",
        "        self.relu = nn.ReLU(inplace=True)\n",
        "        self.conv2 = conv3x3(out_channels, out_channels)\n",
        "        self.bn2 = nn.BatchNorm2d(out_channels)\n",
        "        self.downsample = downsample\n",
        "\n",
        "    def forward(self, x):\n",
        "        residual = x\n",
        "        out = self.conv1(x)\n",
        "        out = self.bn1(out)\n",
        "        out = self.relu(out)\n",
        "        out = self.conv2(out)\n",
        "        out = self.bn2(out)\n",
        "        if self.downsample:\n",
        "            residual = self.downsample(x)\n",
        "        out += residual\n",
        "        out = self.relu(out)\n",
        "        return out\n",
        "\n",
        "# ResNet\n",
        "class ResNet(nn.Module):\n",
        "    def __init__(self, block, layers, num_classes=10):\n",
        "        super(ResNet, self).__init__()\n",
        "        self.in_channels = 16\n",
        "        self.conv = conv3x3(3,16)\n",
        "        self.bn = nn.BatchNorm2d(16)\n",
        "        self.relu = nn.ReLU(inplace=True)\n",
        "        self.layer1 = self.make_layer(block, 16, layers[0])\n",
        "        self.layer2 = self.make_layer(block, 32, layers[1], 2)\n",
        "        self.layer3 = self.make_layer(block, 64, layers[2], 2)\n",
        "        self.avg_pool = nn.AvgPool2d(8)\n",
        "        self.fc = nn.Linear(64, num_classes)\n",
        "\n",
        "    def make_layer(self, block, out_channels, blocks, stride=1):\n",
        "        downsample = None\n",
        "        if (stride != 1) or (self.in_channels != out_channels):\n",
        "            downsample = nn.Sequential(\n",
        "                conv3x3(self.in_channels, out_channels, stride=stride),\n",
        "                nn.BatchNorm2d(out_channels)\n",
        "            )\n",
        "        layers = []\n",
        "        layers.append(block(self.in_channels, out_channels, stride, downsample))\n",
        "        self.in_channels = out_channels\n",
        "        for i in range(1, blocks):\n",
        "            layers.append(block(out_channels, out_channels))\n",
        "        return nn.Sequential(*layers)\n",
        "\n",
        "    def forward(self, x):\n",
        "        out = self.conv(x)\n",
        "        out = self.bn(out)\n",
        "        out = self.relu(out)\n",
        "        out = self.layer1(out)\n",
        "        out = self.layer2(out)\n",
        "        out = self.layer3(out)\n",
        "        out = self.avg_pool(out)\n",
        "        out = out.view(out.size(0), -1)\n",
        "        out = self.fc(out)\n",
        "        return out\n",
        "\n",
        "model = ResNet(ResidualBlock, [2, 2, 2]) #.to(device)\n",
        "\n",
        "# Loss and optimizer\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
        "\n",
        "# For updating learning rate\n",
        "def update_lr(optimizer, lr):\n",
        "    for param_group in optimizer.param_groups:\n",
        "        param_group['lr'] = lr\n",
        "\n",
        "# Train the model\n",
        "total_step = len(train_loader)\n",
        "curr_lr = learning_rate\n",
        "for epoch in range(num_epochs):\n",
        "    for i, (images, labels) in enumerate(train_loader):\n",
        "        #images = images.to(device)\n",
        "        #labels = labels.to(device)\n",
        "\n",
        "        # Forward pass\n",
        "        outputs = model(images)\n",
        "        loss = criterion(outputs, labels)\n",
        "\n",
        "        # Backward and optimize\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "\n",
        "        if (i+1) % 100 == 0:\n",
        "            print('Epoch [{}/{}], Step [{}/{}] Loss: {:.4f}'.format(epoch+1, num_epochs, i+1, total_step,loss.item()))\n",
        "        \n",
        "    # Decay learning rate\n",
        "    if (epoch+1) % 20 == 0:\n",
        "        curr_lr /= 3\n",
        "        update_lr(optimizer, curr_lr)\n",
        "\n",
        "# Test the model\n",
        "model.eval()\n",
        "with torch.no_grad():\n",
        "    correct = 0\n",
        "    total = 0\n",
        "    for images, labels in test_loader:\n",
        "        #images = images.to(device)\n",
        "        #labels = labels.to(device)\n",
        "        outputs = model(images)\n",
        "        _, predicted = torch.max(outputs.data, 1)\n",
        "        total += labels.size(0)\n",
        "        correct += (predicted == labels).sum().item()\n",
        "\n",
        "    print('Accuracy of the model on the test images: {} %'.format(100 * correct / total))\n",
        "\n",
        "# Save the model checkpoint\n",
        "torch.save(model.state_dict(), 'model.ckpt')\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ../../data/cifar-10-python.tar.gz\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9a14b9a2dab8451bbd0632c6c0b8fae7",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/170498071 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting ../../data/cifar-10-python.tar.gz to ../../data/\n",
            "Epoch [1/80], Step [100/500] Loss: 2.4476\n",
            "Epoch [1/80], Step [200/500] Loss: 2.5646\n",
            "Epoch [1/80], Step [300/500] Loss: 2.5019\n",
            "Epoch [1/80], Step [400/500] Loss: 2.5112\n",
            "Epoch [1/80], Step [500/500] Loss: 2.5561\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-1-3ecb658308b3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    129\u001b[0m         \u001b[0;31m# Backward and optimize\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    130\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 131\u001b[0;31m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    132\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    133\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m100\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/_tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m    305\u001b[0m                 \u001b[0mcreate_graph\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    306\u001b[0m                 inputs=inputs)\n\u001b[0;32m--> 307\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    308\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    309\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m    154\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m    155\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 156\u001b[0;31m         allow_unreachable=True, accumulate_grad=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m    157\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    158\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    }
  ]
}