export_lucid.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "export_lucid.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/arunraja-hub/Preference_Extraction/blob/master/export_lucid.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9tAQ3OiTyBY0",
        "colab_type": "text"
      },
      "source": [
        "#Install and imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2qFyxgPWtOMF",
        "colab_type": "code",
        "outputId": "a7fee4e1-38cb-4b60-8f29-b77a8c232989",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "%tensorflow_version 1.x\n",
        "\n",
        "!pip uninstall lucid -y\n",
        "!pip install git+https://github.com/tensorflow/lucid.git#egg=lucid\n",
        "\n",
        "!git clone https://github.com/arunraja-hub/Preference_Extraction.git\n",
        "\n",
        "!pip install tf-agents==0.3.0\n",
        "\n",
        "!pip uninstall tensorflow-probability -y\n",
        "!pip install tensorflow-probability==0.7.0\n",
        "\n",
        "!npm install -g svelte-cli@2.2.0\n",
        "\n",
        "import tensorflow as tf\n",
        "\n",
        "tf.compat.v1.disable_eager_execution()"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TensorFlow 1.x selected.\n",
            "Uninstalling lucid-0.3.8:\n",
            "  Successfully uninstalled lucid-0.3.8\n",
            "Collecting lucid\n",
            "  Cloning https://github.com/tensorflow/lucid.git to /tmp/pip-install-4czqr4ke/lucid\n",
            "  Running command git clone -q https://github.com/tensorflow/lucid.git /tmp/pip-install-4czqr4ke/lucid\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from lucid) (1.18.2)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from lucid) (1.4.1)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from lucid) (0.22.2.post1)\n",
            "Requirement already satisfied: umap-learn in /usr/local/lib/python3.6/dist-packages (from lucid) (0.4.1)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.6/dist-packages (from lucid) (3.2.5)\n",
            "Requirement already satisfied: ipython in /usr/local/lib/python3.6/dist-packages (from lucid) (5.5.0)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.6/dist-packages (from lucid) (7.0.0)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from lucid) (0.16.0)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.6/dist-packages (from lucid) (4.4.2)\n",
            "Requirement already satisfied: pyopengl in /usr/local/lib/python3.6/dist-packages (from lucid) (3.1.5)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from lucid) (3.0.12)\n",
            "Requirement already satisfied: cachetools in /usr/local/lib/python3.6/dist-packages (from lucid) (3.1.1)\n",
            "Requirement already satisfied: more-itertools in /usr/local/lib/python3.6/dist-packages (from lucid) (8.2.0)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->lucid) (0.14.1)\n",
            "Requirement already satisfied: tbb>=2019.0 in /usr/local/lib/python3.6/dist-packages (from umap-learn->lucid) (2020.0.133)\n",
            "Requirement already satisfied: numba!=0.47,>=0.46 in /usr/local/lib/python3.6/dist-packages (from umap-learn->lucid) (0.48.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from nltk->lucid) (1.12.0)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.6/dist-packages (from ipython->lucid) (2.1.3)\n",
            "Requirement already satisfied: setuptools>=18.5 in /usr/local/lib/python3.6/dist-packages (from ipython->lucid) (46.1.3)\n",
            "Requirement already satisfied: simplegeneric>0.8 in /usr/local/lib/python3.6/dist-packages (from ipython->lucid) (0.8.1)\n",
            "Requirement already satisfied: pexpect; sys_platform != \"win32\" in /usr/local/lib/python3.6/dist-packages (from ipython->lucid) (4.8.0)\n",
            "Requirement already satisfied: prompt-toolkit<2.0.0,>=1.0.4 in /usr/local/lib/python3.6/dist-packages (from ipython->lucid) (1.0.18)\n",
            "Requirement already satisfied: traitlets>=4.2 in /usr/local/lib/python3.6/dist-packages (from ipython->lucid) (4.3.3)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.6/dist-packages (from ipython->lucid) (0.7.5)\n",
            "Requirement already satisfied: llvmlite<0.32.0,>=0.31.0dev0 in /usr/local/lib/python3.6/dist-packages (from numba!=0.47,>=0.46->umap-learn->lucid) (0.31.0)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.6/dist-packages (from pexpect; sys_platform != \"win32\"->ipython->lucid) (0.6.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.6/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->ipython->lucid) (0.1.9)\n",
            "Requirement already satisfied: ipython-genutils in /usr/local/lib/python3.6/dist-packages (from traitlets>=4.2->ipython->lucid) (0.2.0)\n",
            "Building wheels for collected packages: lucid\n",
            "  Building wheel for lucid (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for lucid: filename=lucid-0.3.9-cp36-none-any.whl size=155742 sha256=22948157518f6eb5cca797b231bac996cbf935188cc113aa416cbee0ca4ecc4a\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-u3y8ii2j/wheels/64/c2/51/7238e75f457453774587708b21cd3cf32f9a86456648688634\n",
            "Successfully built lucid\n",
            "Installing collected packages: lucid\n",
            "Successfully installed lucid-0.3.9\n",
            "Cloning into 'Preference_Extraction'...\n",
            "remote: Enumerating objects: 571, done.\u001b[K\n",
            "remote: Counting objects: 100% (571/571), done.\u001b[K\n",
            "remote: Compressing objects: 100% (557/557), done.\u001b[K\n",
            "remote: Total 571 (delta 38), reused 523 (delta 13), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (571/571), 20.89 MiB | 15.19 MiB/s, done.\n",
            "Resolving deltas: 100% (38/38), done.\n",
            "Collecting tf-agents==0.3.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0e/a5/07aa82a3cd586d193b2f086b50a2fd0f48bd888ae204389f666eb178cfb3/tf_agents-0.3.0-py2.py3-none-any.whl (839kB)\n",
            "\u001b[K     |████████████████████████████████| 839kB 9.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: absl-py>=0.6.1 in /usr/local/lib/python3.6/dist-packages (from tf-agents==0.3.0) (0.9.0)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tf-agents==0.3.0) (1.12.0)\n",
            "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from tf-agents==0.3.0) (1.18.2)\n",
            "Collecting gin-config==0.1.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/8c/be/c984b1c8a7ba1c385b32bf39c7a225cd9f713d49705898309d01b60fd0e7/gin_config-0.1.3-py3-none-any.whl (43kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 7.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: tensorflow-probability>=0.6.0 in /tensorflow-1.15.2/python3.6 (from tf-agents==0.3.0) (0.7.0)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.6/dist-packages (from tensorflow-probability>=0.6.0->tf-agents==0.3.0) (4.4.2)\n",
            "Requirement already satisfied: cloudpickle>=0.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-probability>=0.6.0->tf-agents==0.3.0) (1.3.0)\n",
            "Installing collected packages: gin-config, tf-agents\n",
            "  Found existing installation: gin-config 0.3.0\n",
            "    Uninstalling gin-config-0.3.0:\n",
            "      Successfully uninstalled gin-config-0.3.0\n",
            "Successfully installed gin-config-0.1.3 tf-agents-0.3.0\n",
            "Uninstalling tensorflow-probability-0.7.0:\n",
            "  Successfully uninstalled tensorflow-probability-0.7.0\n",
            "Collecting tensorflow-probability==0.7.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/3e/3a/c10b6c22320531c774402ac7186d1b673374e2a9d12502cbc8d811e4601c/tensorflow_probability-0.7.0-py2.py3-none-any.whl (981kB)\n",
            "\u001b[K     |████████████████████████████████| 983kB 8.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-probability==0.7.0) (1.12.0)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.6/dist-packages (from tensorflow-probability==0.7.0) (4.4.2)\n",
            "Requirement already satisfied: cloudpickle>=0.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-probability==0.7.0) (1.3.0)\n",
            "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from tensorflow-probability==0.7.0) (1.18.2)\n",
            "Installing collected packages: tensorflow-probability\n",
            "  Found existing installation: tensorflow-probability 0.9.0\n",
            "    Uninstalling tensorflow-probability-0.9.0:\n",
            "      Successfully uninstalled tensorflow-probability-0.9.0\n",
            "Successfully installed tensorflow-probability-0.7.0\n",
            "\u001b[K\u001b[?25h/tools/node/bin/svelte -> /tools/node/lib/node_modules/svelte-cli/bin.js\n",
            "+ svelte-cli@2.2.0\n",
            "added 2 packages from 1 contributor in 0.937s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ap5q8W9Mx8Yi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "from tf_agents.trajectories.time_step import TimeStep\n",
        "from tf_agents.specs.tensor_spec import TensorSpec\n",
        "from tf_agents.specs.tensor_spec import TensorSpec\n",
        "from tf_agents.specs.tensor_spec import BoundedTensorSpec\n",
        "from tf_agents.networks import q_network\n",
        "\n",
        "import lucid.modelzoo.vision_models as models\n",
        "\n",
        "import concurrent.futures\n",
        "import itertools\n",
        "import os\n",
        "import pickle\n",
        "import random\n",
        "import sys\n",
        "import time\n",
        "\n",
        "import numpy as np\n",
        "import io\n",
        "import collections\n",
        "\n",
        "import urllib.request\n",
        "from urllib.error import HTTPError"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QI_qeDUfyJEn",
        "colab_type": "text"
      },
      "source": [
        "# Examine the checkpoint vars"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NCPu6NZBEjGh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cpt_name = \"Preference_Extraction/model_ckpt\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2H4LfezV9Kfh",
        "colab_type": "code",
        "outputId": "2ebaabbd-c2a5-4e69-9f6c-a1733851c397",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 544
        }
      },
      "source": [
        "cpt_var_names = tf.compat.v1.train.list_variables(cpt_name)\n",
        "[name for name in cpt_var_names if ((\"bias\" in name[0]) or (\"kernel\" in name[0])) and not (\"OPTIMIZER\" in name[0])]"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('agent/_q_network/_encoder/_postprocessing_layers/0/bias/.ATTRIBUTES/VARIABLE_VALUE',\n",
              "  [16]),\n",
              " ('agent/_q_network/_encoder/_postprocessing_layers/0/kernel/.ATTRIBUTES/VARIABLE_VALUE',\n",
              "  [3, 3, 5, 16]),\n",
              " ('agent/_q_network/_encoder/_postprocessing_layers/1/bias/.ATTRIBUTES/VARIABLE_VALUE',\n",
              "  [32]),\n",
              " ('agent/_q_network/_encoder/_postprocessing_layers/1/kernel/.ATTRIBUTES/VARIABLE_VALUE',\n",
              "  [3, 3, 16, 32]),\n",
              " ('agent/_q_network/_encoder/_postprocessing_layers/3/bias/.ATTRIBUTES/VARIABLE_VALUE',\n",
              "  [64]),\n",
              " ('agent/_q_network/_encoder/_postprocessing_layers/3/kernel/.ATTRIBUTES/VARIABLE_VALUE',\n",
              "  [960, 64]),\n",
              " ('agent/_q_network/_q_value_layer/bias/.ATTRIBUTES/VARIABLE_VALUE', [3]),\n",
              " ('agent/_q_network/_q_value_layer/kernel/.ATTRIBUTES/VARIABLE_VALUE',\n",
              "  [64, 3]),\n",
              " ('agent/_target_q_network/_encoder/_postprocessing_layers/0/bias/.ATTRIBUTES/VARIABLE_VALUE',\n",
              "  [16]),\n",
              " ('agent/_target_q_network/_encoder/_postprocessing_layers/0/kernel/.ATTRIBUTES/VARIABLE_VALUE',\n",
              "  [3, 3, 5, 16]),\n",
              " ('agent/_target_q_network/_encoder/_postprocessing_layers/1/bias/.ATTRIBUTES/VARIABLE_VALUE',\n",
              "  [32]),\n",
              " ('agent/_target_q_network/_encoder/_postprocessing_layers/1/kernel/.ATTRIBUTES/VARIABLE_VALUE',\n",
              "  [3, 3, 16, 32]),\n",
              " ('agent/_target_q_network/_encoder/_postprocessing_layers/3/bias/.ATTRIBUTES/VARIABLE_VALUE',\n",
              "  [64]),\n",
              " ('agent/_target_q_network/_encoder/_postprocessing_layers/3/kernel/.ATTRIBUTES/VARIABLE_VALUE',\n",
              "  [960, 64]),\n",
              " ('agent/_target_q_network/_q_value_layer/bias/.ATTRIBUTES/VARIABLE_VALUE',\n",
              "  [3]),\n",
              " ('agent/_target_q_network/_q_value_layer/kernel/.ATTRIBUTES/VARIABLE_VALUE',\n",
              "  [64, 3])]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qtLlCNBLdBBK",
        "colab_type": "text"
      },
      "source": [
        "# Setup model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jyivcV3jEJSn",
        "colab_type": "code",
        "outputId": "f2dc5311-4d50-43c0-a66d-a586897d6d6f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "tf.reset_default_graph()\n",
        "\n",
        "input_shape = [14, 16, 5]\n",
        "\n",
        "my_input = tf.placeholder(tf.float32, shape=[None] + input_shape, name=\"my_input\")\n",
        "q_vals = q_network.QNetwork(input_tensor_spec=TensorSpec(shape=(14, 16, 5)), action_spec=BoundedTensorSpec((), tf.int32, 0, 2), conv_layer_params = [[16, 3, 1], [32, 3, 2]], fc_layer_params = [64])(my_input)\n",
        "\n",
        "[tensor for op in tf.get_default_graph().get_operations() for tensor in op.values()]"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Entity <bound method QNetwork.call of <tf_agents.networks.q_network.QNetwork object at 0x7fc262998898>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Entity <bound method QNetwork.call of <tf_agents.networks.q_network.QNetwork object at 0x7fc262998898>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING: Entity <bound method QNetwork.call of <tf_agents.networks.q_network.QNetwork object at 0x7fc262998898>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Bad argument number for Name: 3, expecting 4\n",
            "WARNING:tensorflow:Entity <bound method EncodingNetwork.call of <tf_agents.networks.encoding_network.EncodingNetwork object at 0x7fc2629989e8>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Entity <bound method EncodingNetwork.call of <tf_agents.networks.encoding_network.EncodingNetwork object at 0x7fc2629989e8>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING: Entity <bound method EncodingNetwork.call of <tf_agents.networks.encoding_network.EncodingNetwork object at 0x7fc2629989e8>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'\n",
            "WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "If using Keras pass *_constraint arguments to layers.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "If using Keras pass *_constraint arguments to layers.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<tf.Tensor 'my_input:0' shape=(?, 14, 16, 5) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d/kernel/Initializer/truncated_normal/shape:0' shape=(4,) dtype=int32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d/kernel/Initializer/truncated_normal/mean:0' shape=() dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d/kernel/Initializer/truncated_normal/stddev:0' shape=() dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d/kernel/Initializer/truncated_normal/TruncatedNormal:0' shape=(3, 3, 5, 16) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d/kernel/Initializer/truncated_normal/mul:0' shape=(3, 3, 5, 16) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d/kernel/Initializer/truncated_normal:0' shape=(3, 3, 5, 16) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d/kernel:0' shape=() dtype=resource>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d/kernel/IsInitialized/VarIsInitializedOp:0' shape=() dtype=bool>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d/kernel/Read/ReadVariableOp:0' shape=(3, 3, 5, 16) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d/bias/Initializer/zeros:0' shape=(16,) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d/bias:0' shape=() dtype=resource>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d/bias/IsInitialized/VarIsInitializedOp:0' shape=() dtype=bool>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d/bias/Read/ReadVariableOp:0' shape=(16,) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d/dilation_rate:0' shape=(2,) dtype=int32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d/Conv2D/ReadVariableOp:0' shape=(3, 3, 5, 16) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d/Conv2D:0' shape=(?, 12, 14, 16) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d/BiasAdd/ReadVariableOp:0' shape=(16,) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d/BiasAdd:0' shape=(?, 12, 14, 16) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d/Relu:0' shape=(?, 12, 14, 16) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d_1/kernel/Initializer/truncated_normal/shape:0' shape=(4,) dtype=int32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d_1/kernel/Initializer/truncated_normal/mean:0' shape=() dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d_1/kernel/Initializer/truncated_normal/stddev:0' shape=() dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d_1/kernel/Initializer/truncated_normal/TruncatedNormal:0' shape=(3, 3, 16, 32) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d_1/kernel/Initializer/truncated_normal/mul:0' shape=(3, 3, 16, 32) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d_1/kernel/Initializer/truncated_normal:0' shape=(3, 3, 16, 32) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d_1/kernel:0' shape=() dtype=resource>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d_1/kernel/IsInitialized/VarIsInitializedOp:0' shape=() dtype=bool>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d_1/kernel/Read/ReadVariableOp:0' shape=(3, 3, 16, 32) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d_1/bias/Initializer/zeros:0' shape=(32,) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d_1/bias:0' shape=() dtype=resource>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d_1/bias/IsInitialized/VarIsInitializedOp:0' shape=() dtype=bool>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d_1/bias/Read/ReadVariableOp:0' shape=(32,) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d_1/dilation_rate:0' shape=(2,) dtype=int32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d_1/Conv2D/ReadVariableOp:0' shape=(3, 3, 16, 32) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d_1/Conv2D:0' shape=(?, 5, 6, 32) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d_1/BiasAdd/ReadVariableOp:0' shape=(32,) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d_1/BiasAdd:0' shape=(?, 5, 6, 32) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d_1/Relu:0' shape=(?, 5, 6, 32) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/flatten/Shape:0' shape=(4,) dtype=int32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/flatten/strided_slice/stack:0' shape=(1,) dtype=int32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/flatten/strided_slice/stack_1:0' shape=(1,) dtype=int32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/flatten/strided_slice/stack_2:0' shape=(1,) dtype=int32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/flatten/strided_slice:0' shape=() dtype=int32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/flatten/Reshape/shape/1:0' shape=() dtype=int32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/flatten/Reshape/shape:0' shape=(2,) dtype=int32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/flatten/Reshape:0' shape=(?, 960) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/dense/kernel/Initializer/truncated_normal/shape:0' shape=(2,) dtype=int32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/dense/kernel/Initializer/truncated_normal/mean:0' shape=() dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/dense/kernel/Initializer/truncated_normal/stddev:0' shape=() dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/dense/kernel/Initializer/truncated_normal/TruncatedNormal:0' shape=(960, 64) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/dense/kernel/Initializer/truncated_normal/mul:0' shape=(960, 64) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/dense/kernel/Initializer/truncated_normal:0' shape=(960, 64) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/dense/kernel:0' shape=() dtype=resource>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/dense/kernel/IsInitialized/VarIsInitializedOp:0' shape=() dtype=bool>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/dense/kernel/Read/ReadVariableOp:0' shape=(960, 64) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/dense/bias/Initializer/zeros:0' shape=(64,) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/dense/bias:0' shape=() dtype=resource>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/dense/bias/IsInitialized/VarIsInitializedOp:0' shape=() dtype=bool>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/dense/bias/Read/ReadVariableOp:0' shape=(64,) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/dense/MatMul/ReadVariableOp:0' shape=(960, 64) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/dense/MatMul:0' shape=(?, 64) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/dense/BiasAdd/ReadVariableOp:0' shape=(64,) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/dense/BiasAdd:0' shape=(?, 64) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/EncodingNetwork/EncodingNetwork/dense/Relu:0' shape=(?, 64) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/dense/kernel/Initializer/random_uniform/shape:0' shape=(2,) dtype=int32>,\n",
              " <tf.Tensor 'QNetwork/dense/kernel/Initializer/random_uniform/min:0' shape=() dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/dense/kernel/Initializer/random_uniform/max:0' shape=() dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/dense/kernel/Initializer/random_uniform/RandomUniform:0' shape=(64, 3) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/dense/kernel/Initializer/random_uniform/sub:0' shape=() dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/dense/kernel/Initializer/random_uniform/mul:0' shape=(64, 3) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/dense/kernel/Initializer/random_uniform:0' shape=(64, 3) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/dense/kernel:0' shape=() dtype=resource>,\n",
              " <tf.Tensor 'QNetwork/dense/kernel/IsInitialized/VarIsInitializedOp:0' shape=() dtype=bool>,\n",
              " <tf.Tensor 'QNetwork/dense/kernel/Read/ReadVariableOp:0' shape=(64, 3) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/dense/bias/Initializer/Const:0' shape=(3,) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/dense/bias:0' shape=() dtype=resource>,\n",
              " <tf.Tensor 'QNetwork/dense/bias/IsInitialized/VarIsInitializedOp:0' shape=() dtype=bool>,\n",
              " <tf.Tensor 'QNetwork/dense/bias/Read/ReadVariableOp:0' shape=(3,) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/dense/MatMul/ReadVariableOp:0' shape=(64, 3) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/dense/MatMul:0' shape=(?, 3) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/dense/BiasAdd/ReadVariableOp:0' shape=(3,) dtype=float32>,\n",
              " <tf.Tensor 'QNetwork/dense/BiasAdd:0' shape=(?, 3) dtype=float32>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2IBJo4kTdH58",
        "colab_type": "text"
      },
      "source": [
        "# Get the var_name_to_prev_var_name mapping by matching tensors with the same shape."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oHJd9f3EEMVX",
        "colab_type": "code",
        "outputId": "1a9ad9e3-52a2-45d8-bc23-54c36fb0f07a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 272
        }
      },
      "source": [
        "cpt_var_info = tf.compat.v1.train.list_variables(cpt_name)\n",
        "cpt_var_info = [var for var in cpt_var_info if ((\"bias\" in var[0]) or (\"kernel\" in var[0])) and not (\"OPTIMIZER\" in var[0]) and not (\"_target_q_network\" in var[0])]\n",
        "\n",
        "shape_to_cpt_var_name = {tuple(var[1]): var[0] for var in cpt_var_info}\n",
        "shape_to_cpt_var_name"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{(3,): 'agent/_q_network/_q_value_layer/bias/.ATTRIBUTES/VARIABLE_VALUE',\n",
              " (3,\n",
              "  3,\n",
              "  5,\n",
              "  16): 'agent/_q_network/_encoder/_postprocessing_layers/0/kernel/.ATTRIBUTES/VARIABLE_VALUE',\n",
              " (3,\n",
              "  3,\n",
              "  16,\n",
              "  32): 'agent/_q_network/_encoder/_postprocessing_layers/1/kernel/.ATTRIBUTES/VARIABLE_VALUE',\n",
              " (16,): 'agent/_q_network/_encoder/_postprocessing_layers/0/bias/.ATTRIBUTES/VARIABLE_VALUE',\n",
              " (32,): 'agent/_q_network/_encoder/_postprocessing_layers/1/bias/.ATTRIBUTES/VARIABLE_VALUE',\n",
              " (64,): 'agent/_q_network/_encoder/_postprocessing_layers/3/bias/.ATTRIBUTES/VARIABLE_VALUE',\n",
              " (64, 3): 'agent/_q_network/_q_value_layer/kernel/.ATTRIBUTES/VARIABLE_VALUE',\n",
              " (960,\n",
              "  64): 'agent/_q_network/_encoder/_postprocessing_layers/3/kernel/.ATTRIBUTES/VARIABLE_VALUE'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YSSE_JK0EV33",
        "colab_type": "code",
        "outputId": "3aeba579-9bb8-492b-cda9-4978871936e2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 275
        }
      },
      "source": [
        "current_vars = tf.get_collection(tf.GraphKeys.VARIABLES)\n",
        "current_vars\n",
        "\n",
        "shape_to_current_var_name = {tuple(var.get_shape().as_list()): var.name[:-2] for var in current_vars}\n",
        "shape_to_current_var_name"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/util/decorator_utils.py:145: GraphKeys.VARIABLES (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.GraphKeys.GLOBAL_VARIABLES` instead.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/util/decorator_utils.py:145: GraphKeys.VARIABLES (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.GraphKeys.GLOBAL_VARIABLES` instead.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{(3,): 'QNetwork/dense/bias',\n",
              " (3, 3, 5, 16): 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d/kernel',\n",
              " (3, 3, 16, 32): 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d_1/kernel',\n",
              " (16,): 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d/bias',\n",
              " (32,): 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d_1/bias',\n",
              " (64,): 'QNetwork/EncodingNetwork/EncodingNetwork/dense/bias',\n",
              " (64, 3): 'QNetwork/dense/kernel',\n",
              " (960, 64): 'QNetwork/EncodingNetwork/EncodingNetwork/dense/kernel'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BzVYSz64EYLQ",
        "colab_type": "code",
        "outputId": "5d43495d-d528-4b24-cdbe-88a488c2d78f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "source": [
        "var_name_to_prev_var_name = {}\n",
        "\n",
        "for shape in shape_to_current_var_name:\n",
        "  var_name_to_prev_var_name[shape_to_current_var_name[shape]] = shape_to_cpt_var_name[shape]\n",
        "\n",
        "var_name_to_prev_var_name"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'QNetwork/EncodingNetwork/EncodingNetwork/conv2d/bias': 'agent/_q_network/_encoder/_postprocessing_layers/0/bias/.ATTRIBUTES/VARIABLE_VALUE',\n",
              " 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d/kernel': 'agent/_q_network/_encoder/_postprocessing_layers/0/kernel/.ATTRIBUTES/VARIABLE_VALUE',\n",
              " 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d_1/bias': 'agent/_q_network/_encoder/_postprocessing_layers/1/bias/.ATTRIBUTES/VARIABLE_VALUE',\n",
              " 'QNetwork/EncodingNetwork/EncodingNetwork/conv2d_1/kernel': 'agent/_q_network/_encoder/_postprocessing_layers/1/kernel/.ATTRIBUTES/VARIABLE_VALUE',\n",
              " 'QNetwork/EncodingNetwork/EncodingNetwork/dense/bias': 'agent/_q_network/_encoder/_postprocessing_layers/3/bias/.ATTRIBUTES/VARIABLE_VALUE',\n",
              " 'QNetwork/EncodingNetwork/EncodingNetwork/dense/kernel': 'agent/_q_network/_encoder/_postprocessing_layers/3/kernel/.ATTRIBUTES/VARIABLE_VALUE',\n",
              " 'QNetwork/dense/bias': 'agent/_q_network/_q_value_layer/bias/.ATTRIBUTES/VARIABLE_VALUE',\n",
              " 'QNetwork/dense/kernel': 'agent/_q_network/_q_value_layer/kernel/.ATTRIBUTES/VARIABLE_VALUE'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b8Wjnp3ydqaS",
        "colab_type": "text"
      },
      "source": [
        "# Read data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y5DtdGrSHqnz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Trajectory(\n",
        "    collections.namedtuple('Trajectory', [\n",
        "        'step_type',\n",
        "        'observation',\n",
        "        'action',\n",
        "        'policy_info',\n",
        "        'next_step_type',\n",
        "        'reward',\n",
        "        'discount',\n",
        "    ])):\n",
        "  \"\"\"Stores the observation the agent saw and the action it took.\n",
        "      The rest of the attributes aren't used in this code.\"\"\"\n",
        "  __slots__ = ()\n",
        "\n",
        "class ListWrapper(object):\n",
        "  def __init__(self, list_to_wrap):\n",
        "    self._list = list_to_wrap\n",
        "\n",
        "  def as_list(self):\n",
        "    return self._list\n",
        "\n",
        "class RenameUnpickler(pickle.Unpickler):\n",
        "    def find_class(self, module, name):\n",
        "      if name == \"Trajectory\":\n",
        "        return Trajectory\n",
        "      if name == \"ListWrapper\":\n",
        "        return ListWrapper\n",
        "\n",
        "      return super(RenameUnpickler, self).find_class(module, name)\n",
        "\n",
        "def rename_load(s):\n",
        "    \"\"\"Helper function analogous to pickle.loads().\"\"\"\n",
        "    return RenameUnpickler(s, encoding='latin1').load()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6Nt_g4fjIHzz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Modified read trajectories functions to read files from local storage\n",
        "\n",
        "def load_file(full_path):\n",
        "    try:\n",
        "        with open(full_path, 'rb') as f:\n",
        "            data = rename_load(f)\n",
        "            return data\n",
        "    except:\n",
        "        return None\n",
        "    \n",
        "def all_load_data(base_path):\n",
        "    \n",
        "    executor = concurrent.futures.ThreadPoolExecutor(max_workers=100)\n",
        "    \n",
        "    futures = []\n",
        "    for i in range(5000):\n",
        "        full_path = os.path.join(base_path, \"ts\"+str(i)+\".pickle\")\n",
        "        future = executor.submit(load_file, full_path)\n",
        "        futures.append(future)\n",
        "    \n",
        "    raw_data = []\n",
        "    for future in concurrent.futures.as_completed(futures):\n",
        "        result = future.result()\n",
        "        if result:\n",
        "            raw_data.append(result)\n",
        "    \n",
        "    return raw_data\n",
        "\n",
        "\n",
        "all_raw_data = all_load_data(\"Preference_Extraction/data/simple_env_1/\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SmIRId7SdRO8",
        "colab_type": "text"
      },
      "source": [
        "# Do the warmstart and verify it worked"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WbPnCcoDESau",
        "colab_type": "code",
        "outputId": "3e06e290-0812-4961-c298-fd743bc1fb5c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        }
      },
      "source": [
        "tf.train.warm_start(cpt_name, var_name_to_prev_var_name=var_name_to_prev_var_name)\n",
        "init_op = tf.global_variables_initializer()"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Warm-starting from: Preference_Extraction/model_ckpt\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Warm-starting from: Preference_Extraction/model_ckpt\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Warm-starting variables only in TRAINABLE_VARIABLES.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Warm-starting variables only in TRAINABLE_VARIABLES.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Warm-started 8 variables.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Warm-started 8 variables.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ml_Ivqa7IceG",
        "colab_type": "code",
        "outputId": "987fc4e8-3c63-45e2-fce6-3937808ed639",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "activation_tensor = tf.get_default_graph().get_tensor_by_name(\"QNetwork/EncodingNetwork/EncodingNetwork/dense/Relu:0\")\n",
        "weights = tf.get_default_graph().get_tensor_by_name(\"QNetwork/dense/kernel/Read/ReadVariableOp:0\")\n",
        "bias = tf.get_default_graph().get_tensor_by_name(\"QNetwork/dense/bias/Read/ReadVariableOp:0\")\n",
        "\n",
        "with tf.Session() as sess:\n",
        "  sess.run(init_op)\n",
        "\n",
        "  for i in range(len(all_raw_data[0].observation)):\n",
        "    single_observation = np.array([all_raw_data[0].observation[i]])\n",
        "\n",
        "    restored_activations = sess.run(activation_tensor, {my_input: single_observation})[0]\n",
        "    old_activations = all_raw_data[0].policy_info[\"activations\"][i]\n",
        "\n",
        "    if i < 3:\n",
        "      print(\"restored_activations\", restored_activations, \"old_activations\", old_activations)\n",
        "\n",
        "    np.testing.assert_allclose(restored_activations, old_activations, rtol=.1)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "restored_activations [  0.         0.         0.         0.       145.40762    0.\n",
            "   0.        36.025948   0.         0.         0.        37.78504\n",
            "   0.         0.         0.         0.         0.         0.\n",
            "   0.         0.         0.       122.21044    0.         0.\n",
            "   0.         0.         0.        21.95644    0.         0.\n",
            "   0.         0.         0.         0.       159.66725    0.\n",
            "  53.03794    0.         0.         0.         0.         0.\n",
            "   0.         0.       136.03       0.         0.         0.\n",
            "   0.         0.         0.         0.         0.         0.\n",
            "   0.         0.         0.         0.         0.         0.\n",
            "   0.        73.01822    0.         0.      ] old_activations [  0.         0.         0.         0.       145.40771    0.\n",
            "   0.        36.025852   0.         0.         0.        37.785103\n",
            "   0.         0.         0.         0.         0.         0.\n",
            "   0.         0.         0.       122.21045    0.         0.\n",
            "   0.         0.         0.        21.956379   0.         0.\n",
            "   0.         0.         0.         0.       159.6672     0.\n",
            "  53.037895   0.         0.         0.         0.         0.\n",
            "   0.         0.       136.03014    0.         0.         0.\n",
            "   0.         0.         0.         0.         0.         0.\n",
            "   0.         0.         0.         0.         0.         0.\n",
            "   0.        73.018265   0.         0.      ]\n",
            "restored_activations [  0.         0.         0.         0.        43.183945   0.\n",
            "   0.         0.         0.         0.         0.        66.72314\n",
            "   0.         0.         0.         0.         0.         0.\n",
            "   0.         0.         0.        95.09497    0.         0.\n",
            "   0.         0.         0.        62.341713   0.         0.\n",
            "   0.         0.         0.         0.         0.         0.\n",
            "  47.15466    0.         0.         0.         0.         0.\n",
            "   0.         0.       130.17538    0.         0.         0.\n",
            "   0.         0.         0.         0.         0.         0.\n",
            "   0.         0.         0.         0.         0.         0.\n",
            "   0.        24.441383   0.         0.      ] old_activations [  0.         0.         0.         0.        43.184002   0.\n",
            "   0.         0.         0.         0.         0.        66.72322\n",
            "   0.         0.         0.         0.         0.         0.\n",
            "   0.         0.         0.        95.09497    0.         0.\n",
            "   0.         0.         0.        62.34176    0.         0.\n",
            "   0.         0.         0.         0.         0.         0.\n",
            "  47.15465    0.         0.         0.         0.         0.\n",
            "   0.         0.       130.17545    0.         0.         0.\n",
            "   0.         0.         0.         0.         0.         0.\n",
            "   0.         0.         0.         0.         0.         0.\n",
            "   0.        24.441444   0.         0.      ]\n",
            "restored_activations [ 0.         0.         0.         0.        99.8912     0.\n",
            "  0.        41.43648    0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         3.0421734  0.\n",
            " 80.19898    0.         0.         0.         0.         0.\n",
            "  0.         0.        11.183752   0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.       ] old_activations [ 0.         0.         0.         0.        99.890915   0.\n",
            "  0.        41.436596   0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         3.0421162  0.\n",
            " 80.19898    0.         0.         0.         0.         0.\n",
            "  0.         0.        11.1835785  0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.         0.         0.\n",
            "  0.         0.         0.         0.       ]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z4wVcF_42IVK",
        "colab_type": "text"
      },
      "source": [
        "# Save Lucid model."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NZEAP5rojiQc",
        "colab_type": "code",
        "outputId": "a9cee355-03f6-4031-9190-47f0d018270f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 887
        }
      },
      "source": [
        "with tf.Session() as sess:\n",
        "  sess.run(init_op)\n",
        "\n",
        "  models.Model.suggest_save_args()\n",
        "  \n",
        "  models.Model.save(\n",
        "    input_name='my_input',\n",
        "    image_shape=input_shape,\n",
        "    output_names=[\"QNetwork/dense/BiasAdd\"],\n",
        "    image_value_range=[0,1],\n",
        "    save_url=\"lucid_save_model.pb\"\n",
        "  )"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Inferred: input_name = my_input (because it was the only Placeholder in the graph_def)\n",
            "Inferred: image_shape = [14, 16, 5]\n",
            "# Please sanity check all inferred values before using this code.\n",
            "# Incorrect `image_value_range` is the most common cause of feature visualization bugs! Most methods will fail silently with incorrect visualizations!\n",
            "Model.save(\n",
            "    input_name='my_input',\n",
            "    image_shape=[14, 16, 5],\n",
            "    output_names=_,                   # TODO (eg. ['logits'])\n",
            "    image_value_range=_,                   # TODO (eg. '[-1, 1], [0, 1], [0, 255], or [-117, 138]')\n",
            "  )\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/lucid/modelzoo/util.py:64: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/lucid/modelzoo/vision_base.py:283: UserWarning: Could not infer output_names.\n",
            "  warnings.warn(\"Could not infer output_names.\")\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/lucid/modelzoo/util.py:64: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/lucid/modelzoo/util.py:73: remove_training_nodes (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.compat.v1.graph_util.remove_training_nodes`\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/lucid/modelzoo/util.py:73: remove_training_nodes (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.compat.v1.graph_util.remove_training_nodes`\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/lucid/modelzoo/util.py:75: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.compat.v1.graph_util.extract_sub_graph`\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/lucid/modelzoo/util.py:75: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.compat.v1.graph_util.extract_sub_graph`\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/lucid/modelzoo/util.py:81: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/lucid/modelzoo/util.py:81: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/lucid/modelzoo/util.py:86: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/lucid/modelzoo/util.py:86: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Froze 8 variables.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Froze 8 variables.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Converted 8 variables to const ops.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Converted 8 variables to const ops.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/lucid/misc/io/writing.py:69: The name tf.gfile.MakeDirs is deprecated. Please use tf.io.gfile.makedirs instead.\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/lucid/misc/io/writing.py:69: The name tf.gfile.MakeDirs is deprecated. Please use tf.io.gfile.makedirs instead.\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/lucid/misc/io/writing.py:77: The name tf.gfile.Open is deprecated. Please use tf.io.gfile.GFile instead.\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/lucid/misc/io/writing.py:77: The name tf.gfile.Open is deprecated. Please use tf.io.gfile.GFile instead.\n",
            "\n"
          ],
          "name": "stderr"
        }
      ]
    }
  ]
}