alpha_feature_display_code.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "from torchtyping import TensorType\n",
    "\n",
    "# class TiedSAE(nn.Module):\n",
    "#     def __init__(self, activation_size, n_dict_components):\n",
    "#         super().__init__()\n",
    "#         self.encoder = nn.Parameter(torch.empty((n_dict_components, activation_size)))\n",
    "#         nn.init.xavier_uniform_(self.encoder)\n",
    "#         self.encoder_bias = nn.Parameter(torch.zeros((n_dict_components,)))\n",
    "\n",
    "#     def get_learned_dict(self):\n",
    "#         norms = torch.norm(self.encoder, 2, dim=-1)\n",
    "#         return self.encoder / torch.clamp(norms, 1e-8)[:, None]\n",
    "\n",
    "#     def encode(self, batch):\n",
    "#         c = torch.einsum(\"nd,bd->bn\", self.encoder, batch)\n",
    "#         c = c + self.encoder_bias\n",
    "#         c = torch.clamp(c, min=0.0)\n",
    "#         return c\n",
    "\n",
    "#     def decode(self, code: TensorType[\"_batch_size\", \"_n_dict_components\"]) -> TensorType[\"_batch_size\", \"_activation_size\"]:\n",
    "#         learned_dict = self.get_learned_dict()\n",
    "#         x_hat = torch.einsum(\"nd,bn->bd\", learned_dict, code)\n",
    "#         return x_hat\n",
    "\n",
    "#     def forward(self, batch: TensorType[\"_batch_size\", \"_activation_size\"]) -> TensorType[\"_batch_size\", \"_activation_size\"]:\n",
    "#         c = self.encode(batch)\n",
    "#         x_hat = self.decode(c)\n",
    "#         return x_hat, c\n",
    "\n",
    "#     def n_dict_components(self):\n",
    "#         return self.get_learned_dict().shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/logan/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from transformer_lens import HookedTransformer\n",
    "import numpy as np\n",
    "from torch.utils.data import DataLoader\n",
    "from datasets import Dataset, load_dataset\n",
    "from tqdm.auto import tqdm\n",
    "from functools import partial\n",
    "from einops import rearrange\n",
    "from huggingface_hub import hf_hub_download\n",
    "from transformers import AutoTokenizer, AutoModelForCausalLM, AutoModelForSequenceClassification\n",
    "\n",
    "layer = 1\n",
    "# dataset_name = \"NeelNanda/pile-10k\"\n",
    "# dataset_name = \"BlueSunflower/chess_games_base\"\n",
    "dataset_name = \"stas/openwebtext-10k\"\n",
    "device = \"cuda:0\"\n",
    "model_name = \"EleutherAI/pythia-70m-deduped\"\n",
    "# model_name=\"usvsnsp/pythia-6.9b-rm-full-hh-rlhf\"\n",
    "# model_name = \"BlueSunflower/Pythia-160M-chess\"\n",
    "# model_name=\"reciprocate/dahoas-gptj-rm-static\"\n",
    "\n",
    "max_seq_length=30\n",
    "input_setting = \"input_only\"\n",
    "# input_setting = \"both\"\n",
    "# model_type=\"reward_model\"\n",
    "model_type=\"causal\"\n",
    "ablate_context = True\n",
    "\n",
    "# model_id = \"Elriggs/pythia-70m-deduped\"\n",
    "# model_id = \"Elriggs/pythia-6.9-rm\"\n",
    "# model_id = \"Elriggs/dahoas-gptj-rm-sae\"\n",
    "# model_id = \"jbrinkma/Pythia-160M-chess-sp76-r4-gpt-neox-layers-4\"\n",
    "# filename = f\"dahoas-gptj-rm-static_r4_transformer.h.{layer}.pt\"\n",
    "# filename = f\"tied_residual_l{layer}_r6/_63/learned_dicts.pt\" \n",
    "# ae_download_location = hf_hub_download(repo_id=model_id, filename=filename)\n",
    "# all_autoencoders = torch.load(ae_download_location)\n",
    "# num_l1s = len(all_autoencoders)\n",
    "# all_l1s = [hyperparams[\"l1_alpha\"] for autoencoder, hyperparams in all_autoencoders]\n",
    "# print(all_l1s)\n",
    "# auto_num = 5\n",
    "# autoencoder, hyperparams = all_autoencoders[auto_num]\n",
    "# autoencoder = torch.load(ae_download_location)\n",
    "# autoencoder.to_device(device)\n",
    "# autoencoder.to(device)\n",
    "# cache_name = f\"gpt_neox.layers.{layer}\"\n",
    "\n",
    "\n",
    "# cache_name=f\"transformer.h.{layer}\"\n",
    "\n",
    "\n",
    "if(model_type == \"reward_model\"):\n",
    "    model = AutoModelForSequenceClassification.from_pretrained(model_name).to(device)\n",
    "else:\n",
    "    model = AutoModelForCausalLM.from_pretrained(model_name).to(device)\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from autoencoders import *\n",
    "# ae_model_id = [\"jbrinkma/Pythia-70M-chess_sp51_r4_gpt_neox.layers.1\", \"jbrinkma/Pythia-70M-chess_sp51_r4_gpt_neox.layers.2.mlp\"]\n",
    "model_id = \"jbrinkma/Pythia-70M-deduped-SAEs\"\n",
    "filename = [\"Pythia-70M-deduped-1.pt\", \"Pythia-70M-deduped-mlp-2.pt\"]\n",
    "autoencoders = []\n",
    "for filen in filename:\n",
    "    ae_download_location = hf_hub_download(repo_id=model_id, filename=filen)\n",
    "    autoencoder = torch.load(ae_download_location)\n",
    "    autoencoder.to_device(device)\n",
    "    autoencoders.append(autoencoder)\n",
    "cache_names = [\"gpt_neox.layers.1\", \"gpt_neox.layers.2.mlp\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import json\n",
    "# from datasets import Dataset, load_from_disk\n",
    "\n",
    "# # Create a Hugging Face dataset\n",
    "# dataset = Dataset.from_json(\"test_stockfish_5000.json\").map(\n",
    "#     lambda x: tokenizer(x[\"moves\"]),\n",
    "#     batched=True,\n",
    "# ).filter(\n",
    "#     lambda x: len(x['input_ids']) > max_seq_length\n",
    "# ).map(\n",
    "#     lambda x: {'input_ids': x['input_ids'][:max_seq_length]}\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading NeelNanda/pile-10k\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset parquet (/root/.cache/huggingface/datasets/NeelNanda___parquet/NeelNanda--pile-10k-72f566e9f7c464ab/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec)\n",
      "Loading cached processed dataset at /root/.cache/huggingface/datasets/NeelNanda___parquet/NeelNanda--pile-10k-72f566e9f7c464ab/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec/cache-5794dc4c32dc1e47.arrow\n",
      "Loading cached processed dataset at /root/.cache/huggingface/datasets/NeelNanda___parquet/NeelNanda--pile-10k-72f566e9f7c464ab/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec/cache-225a77b9775e981c.arrow\n",
      "Loading cached processed dataset at /root/.cache/huggingface/datasets/NeelNanda___parquet/NeelNanda--pile-10k-72f566e9f7c464ab/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec/cache-db79252a408aab7b.arrow\n"
     ]
    }
   ],
   "source": [
    "def download_dataset(dataset_name, tokenizer, max_length=256, num_datapoints=None):\n",
    "    if(num_datapoints):\n",
    "        split_text = f\"train[:{num_datapoints}]\"\n",
    "    else:\n",
    "        split_text = \"train\"\n",
    "    dataset = load_dataset(dataset_name, split=split_text).map(\n",
    "        lambda x: tokenizer(x['text']),\n",
    "        batched=True,\n",
    "    ).filter(\n",
    "        lambda x: len(x['input_ids']) > max_length\n",
    "    ).map(\n",
    "        lambda x: {'input_ids': x['input_ids'][:max_length]}\n",
    "    )\n",
    "    return dataset\n",
    "\n",
    "print(f\"Downloading {dataset_name}\")\n",
    "dataset = download_dataset(dataset_name, tokenizer=tokenizer, max_length=max_seq_length, num_datapoints=None) # num_datapoints grabs all of them if None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['gpt_neox.layers.1', 'gpt_neox.layers.2.mlp'],\n",
       " [<autoencoders.learned_dict.TiedSAE at 0x7fbfc2b37c40>,\n",
       "  <autoencoders.learned_dict.TiedSAE at 0x7fbf78629540>])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cache_names, autoencoders "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 310/310 [00:10<00:00, 29.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature 203 is alive\n",
      "Loading page (1/2)\n",
      "Rendering (2/2)                                                    \n",
      "Done                                                               \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<body style=\"background-color: black; color: white;\">\n",
       "Token Activations: <span style=\"background-color:rgba(255,255,255,1);color:rgb(0,0,0)\">&nbsp0.0&nbsp</span><span style=\"background-color:rgba(187,187,255,1);color:rgb(0,0,0)\">&nbsp2.3&nbsp</span><span style=\"background-color:rgba(125,125,255,1);color:rgb(0,0,0)\">&nbsp4.6&nbsp</span><span style=\"background-color:rgba(62,62,255,1);color:rgb(255,255,255)\">&nbsp7.0&nbsp</span><span style=\"background-color:rgba(0,0,255,1);color:rgb(255,255,255)\">&nbsp9.3&nbsp</span><div style=\"margin-top: 0.1em;\"></div>Logit Diff: <span style=\"background-color:rgba(255, 0,0,1); color:rgb(255,255,255)\">&nbsp-2.6&nbsp</span><span style=\"background-color:rgba(255, 62,62,1); color:rgb(255,255,255)\">&nbsp-2.0&nbsp</span><span style=\"background-color:rgba(255, 125,125,1); color:rgb(0,0,0)\">&nbsp-1.3&nbsp</span><span style=\"background-color:rgba(255, 187,187,1); color:rgb(0,0,0)\">&nbsp-0.7&nbsp</span><span style=\"background-color:rgba(255,255,255,1);color:rgb(0,0,0)\">&nbsp0.0&nbsp</span><span style=\"background-color:rgba(187,187,255,1);color:rgb(0,0,0)\">&nbsp0.1&nbsp</span><span style=\"background-color:rgba(125,125,255,1);color:rgb(0,0,0)\">&nbsp0.2&nbsp</span><span style=\"background-color:rgba(62,62,255,1);color:rgb(255,255,255)\">&nbsp0.2&nbsp</span><span style=\"background-color:rgba(0,0,255,1);color:rgb(255,255,255)\">&nbsp0.3&nbsp</span><div style=\"margin-top: 0.5em;\"></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">The</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(222,222,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspinvention</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(156,156,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsprelates</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 248,248,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspto</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspa</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 248,248,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(230,230,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspprocess</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspfor</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(227,227,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspblending</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspliquid</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(231,231,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspflows</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">,</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(196,196,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(218,218,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspparticularly</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 244,244,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspin</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(240,240,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspthe</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 247,247,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspapproach</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 246,246,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspsystem</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 247,247,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspto</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(190,190,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspa</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsppaper</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(231,231,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspmachine</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 247,247,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(244,244,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">,</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 247,247,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspas</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 244,244,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(0,0,255,1);margin-right: 0.00em; color:rgb(255,255,255)\">&nbspwell</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(236,236,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspas</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspa</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(210,210,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspdevice</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 240,240,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspfor</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspimplementing</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(157,157,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspthe</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 247,247,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspprocess</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(241,241,255,1); text-align: center;\"></div></div><div style=\"margin-top: 0.2em;\"></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(197,197,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">Dis</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">closed</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(93,93,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspherein</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 247,247,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">,</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspin</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(241,241,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspvarious</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(241,241,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(172,172,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspembodiments</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 248,248,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(246,246,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">,</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 237,237,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(249,249,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspare</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 245,245,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspstable</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 236,236,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">,</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(241,241,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(235,235,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsphigh</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 246,246,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspperforming</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 246,246,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspnanoparticle</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(205,205,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(234,234,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspcompositions</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(237,237,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(245,245,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspsuitable</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(242,242,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspfor</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspprinting</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">,</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspsuch</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspas</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspby</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 248,248,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspink</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(195,195,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">jet</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspprinting</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 247,247,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">,</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspas</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 248,248,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(9,9,255,1);margin-right: 0.00em; color:rgb(255,255,255)\">&nbspwell</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspas</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 248,248,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(245,245,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspprocesses</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(47,47,255,1); text-align: center;\"></div></div><div style=\"margin-top: 0.2em;\"></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(204,204,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">//</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(212,212,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspCopyright</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(224,224,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(244,244,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp(</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(200,200,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">C</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(232,232,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">)</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(211,211,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp2005</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">-</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 242,242,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">2006</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 243,243,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(244,244,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspThe</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 248,248,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(215,215,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspTrustees</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 240,240,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspof</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 218,218,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(230,230,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspIndiana</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(81,81,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspUniversity</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(237,237,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">.</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(179,179,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">\r\\n\r</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(217,217,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">\\n</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">//</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 248,248,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspUse</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 240,240,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">,</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(227,227,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspmodification</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(204,204,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(237,237,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspand</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(246,246,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspdistribution</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(230,230,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspis</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 247,247,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(21,21,255,1);margin-right: 0.00em; color:rgb(255,255,255)\">&nbspsubject</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspto</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 248,248,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspthe</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 248,248,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(183,183,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspBoost</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 240,240,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(188,188,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspSoftware</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(248,248,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">\r</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 143,143,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(226,226,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">\\n</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"margin-top: 0.2em;\"></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(153,153,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">Introduction</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">\\n</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 239,239,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(236,236,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">============</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">\\n</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(248,248,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">\\n</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">D</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(239,239,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">iet</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(241,241,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">ary</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(209,209,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspguidance</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 244,244,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspis</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(211,211,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(161,161,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspconsistent</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 245,245,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspin</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 237,237,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsprecommending</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 244,244,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspgreater</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 245,245,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspconsumption</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(160,160,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspof</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspfruit</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(228,228,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(241,241,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspand</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(232,232,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspvegetables</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspto</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 248,248,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsppromote</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(241,241,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsphealth</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">.</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 248,248,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(26,26,255,1);margin-right: 0.00em; color:rgb(255,255,255)\">&nbspIndeed</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(233,233,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">,</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 245,245,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspthe</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 244,244,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp2015</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(90,90,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspDiet</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 248,248,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(239,239,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">ary</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(233,233,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspGuidelines</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 244,244,1); text-align: center;\"></div></div><div style=\"margin-top: 0.2em;\"></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(204,204,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">//</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(212,212,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspCopyright</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(224,224,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(203,203,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp2012</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 245,245,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(200,200,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspCit</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(154,154,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(177,177,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">rix</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 233,233,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(209,209,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspSystems</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 243,243,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(194,194,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">,</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 244,244,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(164,164,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspInc</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">.</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(242,242,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(183,183,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspLicensed</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(227,227,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(180,180,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspunder</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspthe</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 244,244,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">\\n</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(204,204,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">//</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 197,197,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(191,191,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspApache</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 237,237,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(160,160,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspLicense</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(217,217,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">,</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(173,173,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(200,200,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspVersion</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(204,204,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp2</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(248,248,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">.</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">0</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(200,200,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp(</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(114,114,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">the</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(176,176,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp\"</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(125,125,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">License</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(47,47,255,1);margin-right: 0.00em; color:rgb(255,255,255)\">\");</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 248,248,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(29,29,255,1);margin-right: 0.00em; color:rgb(255,255,255)\">&nbspyou</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 214,214,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(115,115,255,1);margin-right: 0.00em; color:rgb(255,255,255)\">&nbspmay</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(125,125,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspnot</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(169,169,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspuse</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 248,248,1); text-align: center;\"></div></div><div style=\"margin-top: 0.2em;\"></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(231,231,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">Part</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(238,238,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">indo</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(237,237,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(247,247,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspda</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 247,247,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(212,212,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspide</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(242,242,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">ia</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 248,248,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspde</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspque</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspo</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspsistema</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspde</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(221,221,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspJust</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(213,213,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">i</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(224,224,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(205,205,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">ça</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 238,238,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(222,222,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsppode</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 239,239,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(200,200,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsptanto</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 241,241,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(117,117,255,1);margin-right: 0.00em; color:rgb(255,255,255)\">&nbspfav</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 247,247,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(151,151,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">ore</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 226,226,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">cer</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 238,238,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspo</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(236,236,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(69,69,255,1);margin-right: 0.00em; color:rgb(255,255,255)\">&nbspapro</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(234,234,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(126,126,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">fund</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 225,225,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">amento</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(142,142,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(113,113,255,1);margin-right: 0.00em; color:rgb(255,255,255)\">&nbspdemocr</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(234,234,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(151,151,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">á</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">tico</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(178,178,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspquanto</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(187,187,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(104,104,255,1);margin-right: 0.00em; color:rgb(255,255,255)\">&nbspcri</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(82,82,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">ar</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(76,76,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(79,79,255,1);margin-right: 0.00em; color:rgb(255,255,255)\">&nbspobst</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(65,65,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(30,30,255,1);margin-right: 0.00em; color:rgb(255,255,255)\">á</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 0,0,1); text-align: center;\"></div></div><div style=\"margin-top: 0.2em;\"></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(204,204,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">//</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(212,212,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspCopyright</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(224,224,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(244,244,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp(</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(200,200,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">C</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(232,232,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">)</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(221,221,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp2017</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(217,217,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">-</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 237,237,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(246,246,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">2019</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspJonathan</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 245,245,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspMüller</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 248,248,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp<</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(195,195,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">jon</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 248,248,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">athan</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(167,167,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">mu</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(225,225,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">eller</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">.</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(210,210,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">dev</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">@</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(169,169,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">gmail</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(241,241,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(194,194,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">.</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">com</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(230,230,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">></span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(158,158,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">\\n</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">//</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(226,226,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspThis</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(181,181,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(216,216,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspfile</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 244,244,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(218,218,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspis</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(242,242,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(31,31,255,1);margin-right: 0.00em; color:rgb(255,255,255)\">&nbspsubject</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 222,222,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspto</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 244,244,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspthe</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(226,226,255,1); text-align: center;\"></div></div><div style=\"margin-top: 0.2em;\"></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(227,227,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">1</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">.</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(207,207,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(184,184,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspIntroduction</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(219,219,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp{#</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(214,214,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(222,222,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">sec</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">1</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 246,246,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">-</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(237,237,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">be</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(205,205,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(224,224,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">hav</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(238,238,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">sci</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">-</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">10</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 248,248,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">-</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(241,241,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">00</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">114</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(224,224,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(199,199,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">}</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(201,201,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">\\n</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(220,220,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">===============</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 247,247,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(32,32,255,1);margin-right: 0.00em; color:rgb(255,255,255)\">\\n</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(228,228,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">\\n</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(222,222,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">Health</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(207,207,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">y</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 248,248,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspparent</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(205,205,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">--</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 240,240,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(231,231,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">child</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(185,185,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(226,226,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspinteractions</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 247,247,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspprovide</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 241,241,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(167,167,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspopportunities</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(233,233,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspfor</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 245,245,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspchildren</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(231,231,255,1); text-align: center;\"></div></div><div style=\"margin-top: 0.2em;\"></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(246,246,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">﻿</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(234,234,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">#</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(239,239,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspCopyright</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(238,238,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">(</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(227,227,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">c</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 248,248,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(202,202,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">)</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp2017</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(177,177,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspGoogle</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(194,194,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspInc</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">.</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(231,231,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(154,154,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">\\n</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">#</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(224,224,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(219,219,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">\\n</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(92,92,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">#</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(172,172,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspLicensed</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(239,239,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(174,174,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspunder</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspthe</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 248,248,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(171,171,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspApache</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 246,246,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(160,160,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspLicense</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(192,192,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">,</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(181,181,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspVersion</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(196,196,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp2</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(237,237,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">.</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">0</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(199,199,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp(</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(237,237,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">the</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp\"</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(102,102,255,1);margin-right: 0.00em; color:rgb(255,255,255)\">License</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(61,61,255,1);margin-right: 0.00em; color:rgb(255,255,255)\">\");</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 248,248,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(33,33,255,1);margin-right: 0.00em; color:rgb(255,255,255)\">&nbspyou</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 185,185,1); text-align: center;\"></div></div><div style=\"margin-top: 0.2em;\"></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp235</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">.</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(90,90,255,1);margin-right: 0.00em; color:rgb(255,255,255)\">&nbspLet</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspu</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 232,232,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp=</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(212,212,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp138</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(228,228,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp-</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspq</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 246,246,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(241,241,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">.</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspSolve</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(0,0,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp-</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">10</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(228,228,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp+</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspu</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 248,248,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp=</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 249,249,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspk</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(176,176,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspfor</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspk</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">.</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(212,212,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">\\n</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">-</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 246,246,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">3</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(166,166,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">\\n</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(34,34,255,1);margin-right: 0.00em; color:rgb(255,255,255)\">Let</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255,255,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspr</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 247,247,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(239,239,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspbe</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 220,220,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp(</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(217,217,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">399</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(179,179,255,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">/</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 246,246,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">14</span><div style=\"display: block; margin-right: 0.00em; height: 10px; background-color:rgba(255, 248,248,1); text-align: center;\"></div></div><div style=\"margin-top: 0.2em;\"></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading page (1/2)\n",
      "Rendering (2/2)                                                    \n",
      "Done                                                               \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<body style=\"background-color: black; color: white;\">\n",
       "Token Activations: <span style=\"background-color:rgba(255, 0,0,1); color:rgb(255,255,255)\">&nbsp-9.3&nbsp</span><span style=\"background-color:rgba(255, 62,62,1); color:rgb(255,255,255)\">&nbsp-7.0&nbsp</span><span style=\"background-color:rgba(255, 125,125,1); color:rgb(0,0,0)\">&nbsp-4.6&nbsp</span><span style=\"background-color:rgba(255, 187,187,1); color:rgb(0,0,0)\">&nbsp-2.3&nbsp</span><span style=\"background-color:rgba(255,255,255,1);color:rgb(0,0,0)\">&nbsp0.0&nbsp</span><span style=\"background-color:rgba(187,187,255,1);color:rgb(0,0,0)\">&nbsp0.1&nbsp</span><span style=\"background-color:rgba(125,125,255,1);color:rgb(0,0,0)\">&nbsp0.2&nbsp</span><span style=\"background-color:rgba(62,62,255,1);color:rgb(255,255,255)\">&nbsp0.3&nbsp</span><span style=\"background-color:rgba(0,0,255,1);color:rgb(255,255,255)\">&nbsp0.4&nbsp</span><div style=\"margin-top: 0.5em;\"></div><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">The</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspinvention</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsprelates</span><span style=\"background-color:rgba(255, 248,248,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspto</span><span style=\"background-color:rgba(189,189,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspa</span><span style=\"background-color:rgba(255, 247,247,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspprocess</span><span style=\"background-color:rgba(255, 249,249,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspfor</span><span style=\"background-color:rgba(255, 244,244,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspblending</span><span style=\"background-color:rgba(255, 244,244,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspliquid</span><span style=\"background-color:rgba(255, 246,246,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspflows</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">,</span><span style=\"background-color:rgba(255, 241,241,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspparticularly</span><span style=\"background-color:rgba(196,196,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspin</span><span style=\"background-color:rgba(255, 243,243,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspthe</span><span style=\"background-color:rgba(255, 243,243,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspapproach</span><span style=\"background-color:rgba(255, 242,242,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspsystem</span><span style=\"background-color:rgba(255, 248,248,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspto</span><span style=\"background-color:rgba(255, 244,244,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspa</span><span style=\"background-color:rgba(255, 242,242,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsppaper</span><span style=\"background-color:rgba(255, 241,241,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspmachine</span><span style=\"background-color:rgba(255, 175,175,1);margin-right: 0.00em; color:rgb(0,0,0)\">,</span><span style=\"background-color:rgba(255, 40,40,1);margin-right: 0.00em; color:rgb(255,255,255)\">&nbspas</span><span style=\"background-color:rgba(255, 0,0,1);margin-right: 0.00em; color:rgb(255,255,255)\">&nbspwell</span><div style=\"margin-top: 0.2em;\"></div><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">Dis</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">closed</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspherein</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">,</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspin</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspvarious</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspembodiments</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">,</span><span style=\"background-color:rgba(255, 249,249,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspare</span><span style=\"background-color:rgba(217,217,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspstable</span><span style=\"background-color:rgba(240,240,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">,</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsphigh</span><span style=\"background-color:rgba(255, 243,243,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspperforming</span><span style=\"background-color:rgba(229,229,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspnanoparticle</span><span style=\"background-color:rgba(255, 248,248,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspcompositions</span><span style=\"background-color:rgba(255, 245,245,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspsuitable</span><span style=\"background-color:rgba(255, 249,249,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspfor</span><span style=\"background-color:rgba(255, 247,247,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspprinting</span><span style=\"background-color:rgba(129,129,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">,</span><span style=\"background-color:rgba(34,34,255,1);margin-right: 0.00em; color:rgb(255,255,255)\">&nbspsuch</span><span style=\"background-color:rgba(255, 244,244,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspas</span><span style=\"background-color:rgba(255, 247,247,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspby</span><span style=\"background-color:rgba(255, 245,245,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspink</span><span style=\"background-color:rgba(255, 245,245,1);margin-right: 0.00em; color:rgb(0,0,0)\">jet</span><span style=\"background-color:rgba(255, 242,242,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspprinting</span><span style=\"background-color:rgba(255, 164,164,1);margin-right: 0.00em; color:rgb(0,0,0)\">,</span><span style=\"background-color:rgba(255, 69,69,1);margin-right: 0.00em; color:rgb(255,255,255)\">&nbspas</span><span style=\"background-color:rgba(255, 9,9,1);margin-right: 0.00em; color:rgb(255,255,255)\">&nbspwell</span><div style=\"margin-top: 0.2em;\"></div><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">//</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspCopyright</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp(</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">C</span><span style=\"background-color:rgba(255, 249,249,1);margin-right: 0.00em; color:rgb(0,0,0)\">)</span><span style=\"background-color:rgba(255, 249,249,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp2005</span><span style=\"background-color:rgba(212,212,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">-</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">2006</span><span style=\"background-color:rgba(185,185,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspThe</span><span style=\"background-color:rgba(255, 245,245,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspTrustees</span><span style=\"background-color:rgba(255, 249,249,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspof</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspIndiana</span><span style=\"background-color:rgba(255, 249,249,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspUniversity</span><span style=\"background-color:rgba(227,227,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">.</span><span style=\"background-color:rgba(109,109,255,1);margin-right: 0.00em; color:rgb(255,255,255)\">\r\\n\r</span><span style=\"background-color:rgba(233,233,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">\\n</span><span style=\"background-color:rgba(165,165,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">//</span><span style=\"background-color:rgba(255, 247,247,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspUse</span><span style=\"background-color:rgba(219,219,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">,</span><span style=\"background-color:rgba(255, 237,237,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspmodification</span><span style=\"background-color:rgba(148,148,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspand</span><span style=\"background-color:rgba(255, 239,239,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspdistribution</span><span style=\"background-color:rgba(255, 158,158,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspis</span><span style=\"background-color:rgba(255, 21,21,1);margin-right: 0.00em; color:rgb(255,255,255)\">&nbspsubject</span><div style=\"margin-top: 0.2em;\"></div><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">Introduction</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">\\n</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">============</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">\\n</span><span style=\"background-color:rgba(255, 244,244,1);margin-right: 0.00em; color:rgb(0,0,0)\">\\n</span><span style=\"background-color:rgba(210,210,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">D</span><span style=\"background-color:rgba(255, 249,249,1);margin-right: 0.00em; color:rgb(0,0,0)\">iet</span><span style=\"background-color:rgba(255, 249,249,1);margin-right: 0.00em; color:rgb(0,0,0)\">ary</span><span style=\"background-color:rgba(255, 247,247,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspguidance</span><span style=\"background-color:rgba(255, 246,246,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspis</span><span style=\"background-color:rgba(255, 237,237,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspconsistent</span><span style=\"background-color:rgba(185,185,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspin</span><span style=\"background-color:rgba(255, 246,246,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsprecommending</span><span style=\"background-color:rgba(184,184,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspgreater</span><span style=\"background-color:rgba(222,222,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspconsumption</span><span style=\"background-color:rgba(255, 246,246,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspof</span><span style=\"background-color:rgba(255, 247,247,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspfruit</span><span style=\"background-color:rgba(197,197,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspand</span><span style=\"background-color:rgba(255, 245,245,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspvegetables</span><span style=\"background-color:rgba(172,172,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspto</span><span style=\"background-color:rgba(255, 241,241,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsppromote</span><span style=\"background-color:rgba(255, 241,241,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsphealth</span><span style=\"background-color:rgba(255, 112,112,1);margin-right: 0.00em; color:rgb(255,255,255)\">.</span><span style=\"background-color:rgba(255, 26,26,1);margin-right: 0.00em; color:rgb(255,255,255)\">&nbspIndeed</span><div style=\"margin-top: 0.2em;\"></div><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">//</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspCopyright</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp2012</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspCit</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">rix</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspSystems</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">,</span><span style=\"background-color:rgba(255, 246,246,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspInc</span><span style=\"background-color:rgba(255, 246,246,1);margin-right: 0.00em; color:rgb(0,0,0)\">.</span><span style=\"background-color:rgba(255, 241,241,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspLicensed</span><span style=\"background-color:rgba(255, 246,246,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspunder</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspthe</span><span style=\"background-color:rgba(213,213,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">\\n</span><span style=\"background-color:rgba(255, 249,249,1);margin-right: 0.00em; color:rgb(0,0,0)\">//</span><span style=\"background-color:rgba(255, 247,247,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspApache</span><span style=\"background-color:rgba(255, 224,224,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspLicense</span><span style=\"background-color:rgba(255, 247,247,1);margin-right: 0.00em; color:rgb(0,0,0)\">,</span><span style=\"background-color:rgba(255, 231,231,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspVersion</span><span style=\"background-color:rgba(255, 246,246,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp2</span><span style=\"background-color:rgba(255, 242,242,1);margin-right: 0.00em; color:rgb(0,0,0)\">.</span><span style=\"background-color:rgba(213,213,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">0</span><span style=\"background-color:rgba(203,203,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp(</span><span style=\"background-color:rgba(255, 248,248,1);margin-right: 0.00em; color:rgb(0,0,0)\">the</span><span style=\"background-color:rgba(255, 243,243,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp\"</span><span style=\"background-color:rgba(255, 89,89,1);margin-right: 0.00em; color:rgb(255,255,255)\">License</span><span style=\"background-color:rgba(255, 194,194,1);margin-right: 0.00em; color:rgb(0,0,0)\">\");</span><span style=\"background-color:rgba(255, 29,29,1);margin-right: 0.00em; color:rgb(255,255,255)\">&nbspyou</span><div style=\"margin-top: 0.2em;\"></div><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">Part</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">indo</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspda</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspide</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">ia</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspde</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspque</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspo</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspsistema</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspde</span><span style=\"background-color:rgba(255, 249,249,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspJust</span><span style=\"background-color:rgba(197,197,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">i</span><span style=\"background-color:rgba(255, 247,247,1);margin-right: 0.00em; color:rgb(0,0,0)\">ça</span><span style=\"background-color:rgba(255, 248,248,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsppode</span><span style=\"background-color:rgba(255, 249,249,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsptanto</span><span style=\"background-color:rgba(223,223,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspfav</span><span style=\"background-color:rgba(222,222,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">ore</span><span style=\"background-color:rgba(218,218,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">cer</span><span style=\"background-color:rgba(171,171,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspo</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspapro</span><span style=\"background-color:rgba(215,215,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">fund</span><span style=\"background-color:rgba(227,227,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">amento</span><span style=\"background-color:rgba(103,103,255,1);margin-right: 0.00em; color:rgb(255,255,255)\">&nbspdemocr</span><span style=\"background-color:rgba(181,181,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">á</span><span style=\"background-color:rgba(255, 242,242,1);margin-right: 0.00em; color:rgb(0,0,0)\">tico</span><span style=\"background-color:rgba(255, 245,245,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspquanto</span><span style=\"background-color:rgba(71,71,255,1);margin-right: 0.00em; color:rgb(255,255,255)\">&nbspcri</span><span style=\"background-color:rgba(174,174,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">ar</span><span style=\"background-color:rgba(255, 92,92,1);margin-right: 0.00em; color:rgb(255,255,255)\">&nbspobst</span><span style=\"background-color:rgba(255, 30,30,1);margin-right: 0.00em; color:rgb(255,255,255)\">á</span><div style=\"margin-top: 0.2em;\"></div><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">//</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspCopyright</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp(</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">C</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">)</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp2017</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">-</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">2019</span><span style=\"background-color:rgba(255, 248,248,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspJonathan</span><span style=\"background-color:rgba(255, 248,248,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspMüller</span><span style=\"background-color:rgba(255, 247,247,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp<</span><span style=\"background-color:rgba(255, 248,248,1);margin-right: 0.00em; color:rgb(0,0,0)\">jon</span><span style=\"background-color:rgba(255, 248,248,1);margin-right: 0.00em; color:rgb(0,0,0)\">athan</span><span style=\"background-color:rgba(255, 249,249,1);margin-right: 0.00em; color:rgb(0,0,0)\">mu</span><span style=\"background-color:rgba(255, 248,248,1);margin-right: 0.00em; color:rgb(0,0,0)\">eller</span><span style=\"background-color:rgba(255, 243,243,1);margin-right: 0.00em; color:rgb(0,0,0)\">.</span><span style=\"background-color:rgba(255, 245,245,1);margin-right: 0.00em; color:rgb(0,0,0)\">dev</span><span style=\"background-color:rgba(203,203,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">@</span><span style=\"background-color:rgba(224,224,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">gmail</span><span style=\"background-color:rgba(196,196,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">.</span><span style=\"background-color:rgba(255, 245,245,1);margin-right: 0.00em; color:rgb(0,0,0)\">com</span><span style=\"background-color:rgba(234,234,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">></span><span style=\"background-color:rgba(111,111,255,1);margin-right: 0.00em; color:rgb(255,255,255)\">\\n</span><span style=\"background-color:rgba(189,189,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">//</span><span style=\"background-color:rgba(255, 239,239,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspThis</span><span style=\"background-color:rgba(156,156,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspfile</span><span style=\"background-color:rgba(255, 140,140,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspis</span><span style=\"background-color:rgba(255, 31,31,1);margin-right: 0.00em; color:rgb(255,255,255)\">&nbspsubject</span><div style=\"margin-top: 0.2em;\"></div><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">1</span><span style=\"background-color:rgba(255, 196,196,1);margin-right: 0.00em; color:rgb(0,0,0)\">.</span><span style=\"background-color:rgba(255, 247,247,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspIntroduction</span><span style=\"background-color:rgba(0,0,255,1);margin-right: 0.00em; color:rgb(255,255,255)\">&nbsp{#</span><span style=\"background-color:rgba(255, 245,245,1);margin-right: 0.00em; color:rgb(0,0,0)\">sec</span><span style=\"background-color:rgba(255, 249,249,1);margin-right: 0.00em; color:rgb(0,0,0)\">1</span><span style=\"background-color:rgba(255, 246,246,1);margin-right: 0.00em; color:rgb(0,0,0)\">-</span><span style=\"background-color:rgba(255, 245,245,1);margin-right: 0.00em; color:rgb(0,0,0)\">be</span><span style=\"background-color:rgba(255, 245,245,1);margin-right: 0.00em; color:rgb(0,0,0)\">hav</span><span style=\"background-color:rgba(255, 245,245,1);margin-right: 0.00em; color:rgb(0,0,0)\">sci</span><span style=\"background-color:rgba(255, 246,246,1);margin-right: 0.00em; color:rgb(0,0,0)\">-</span><span style=\"background-color:rgba(255, 247,247,1);margin-right: 0.00em; color:rgb(0,0,0)\">10</span><span style=\"background-color:rgba(255, 247,247,1);margin-right: 0.00em; color:rgb(0,0,0)\">-</span><span style=\"background-color:rgba(255, 248,248,1);margin-right: 0.00em; color:rgb(0,0,0)\">00</span><span style=\"background-color:rgba(255, 248,248,1);margin-right: 0.00em; color:rgb(0,0,0)\">114</span><span style=\"background-color:rgba(255, 239,239,1);margin-right: 0.00em; color:rgb(0,0,0)\">}</span><span style=\"background-color:rgba(255, 224,224,1);margin-right: 0.00em; color:rgb(0,0,0)\">\\n</span><span style=\"background-color:rgba(255, 32,32,1);margin-right: 0.00em; color:rgb(255,255,255)\">===============</span><span style=\"background-color:rgba(255, 32,32,1);margin-right: 0.00em; color:rgb(255,255,255)\">\\n</span><div style=\"margin-top: 0.2em;\"></div><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">﻿</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">#</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspCopyright</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">(</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">c</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">)</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp2017</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspGoogle</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspInc</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">.</span><span style=\"background-color:rgba(228,228,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">\\n</span><span style=\"background-color:rgba(255, 249,249,1);margin-right: 0.00em; color:rgb(0,0,0)\">#</span><span style=\"background-color:rgba(215,215,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">\\n</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">#</span><span style=\"background-color:rgba(255, 242,242,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspLicensed</span><span style=\"background-color:rgba(255, 246,246,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspunder</span><span style=\"background-color:rgba(229,229,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspthe</span><span style=\"background-color:rgba(255, 247,247,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspApache</span><span style=\"background-color:rgba(255, 234,234,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspLicense</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">,</span><span style=\"background-color:rgba(255, 230,230,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspVersion</span><span style=\"background-color:rgba(255, 248,248,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp2</span><span style=\"background-color:rgba(255, 245,245,1);margin-right: 0.00em; color:rgb(0,0,0)\">.</span><span style=\"background-color:rgba(177,177,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">0</span><span style=\"background-color:rgba(162,162,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp(</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">the</span><span style=\"background-color:rgba(255, 243,243,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp\"</span><span style=\"background-color:rgba(255, 95,95,1);margin-right: 0.00em; color:rgb(255,255,255)\">License</span><span style=\"background-color:rgba(255, 198,198,1);margin-right: 0.00em; color:rgb(0,0,0)\">\");</span><span style=\"background-color:rgba(255, 33,33,1);margin-right: 0.00em; color:rgb(255,255,255)\">&nbspyou</span><div style=\"margin-top: 0.2em;\"></div><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp235</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">.</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspLet</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspu</span><span style=\"background-color:rgba(255, 244,244,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp=</span><span style=\"background-color:rgba(239,239,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp138</span><span style=\"background-color:rgba(237,237,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp-</span><span style=\"background-color:rgba(220,220,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspq</span><span style=\"background-color:rgba(255, 248,248,1);margin-right: 0.00em; color:rgb(0,0,0)\">.</span><span style=\"background-color:rgba(255, 224,224,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspSolve</span><span style=\"background-color:rgba(255, 248,248,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp-</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">10</span><span style=\"background-color:rgba(255, 247,247,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp+</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspu</span><span style=\"background-color:rgba(216,216,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbsp=</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspk</span><span style=\"background-color:rgba(255, 248,248,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspfor</span><span style=\"background-color:rgba(255,255,255,1);margin-right: 0.00em; color:rgb(0,0,0)\">&nbspk</span><span style=\"background-color:rgba(255, 247,247,1);margin-right: 0.00em; color:rgb(0,0,0)\">.</span><span style=\"background-color:rgba(255, 244,244,1);margin-right: 0.00em; color:rgb(0,0,0)\">\\n</span><span style=\"background-color:rgba(255, 248,248,1);margin-right: 0.00em; color:rgb(0,0,0)\">-</span><span style=\"background-color:rgba(255, 245,245,1);margin-right: 0.00em; color:rgb(0,0,0)\">3</span><span style=\"background-color:rgba(255, 247,247,1);margin-right: 0.00em; color:rgb(0,0,0)\">\\n</span><span style=\"background-color:rgba(255, 34,34,1);margin-right: 0.00em; color:rgb(255,255,255)\">Let</span><div style=\"margin-top: 0.2em;\"></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from alpha_utils_interp import *\n",
    "import os\n",
    "# make features/ dir if not exist\n",
    "save_path = \"features/\"\n",
    "if not os.path.exists(save_path):\n",
    "    os.makedirs(save_path)\n",
    "\n",
    "# features_all = [[2863, 1224, 1582, 2863, 2390, 1224, 2390, 2863, 2863, 1582],[3028,  887, 2917, 1904, 3028, 1404,  887, 1404,  887,  887]]\n",
    "features_all = [[3028,  887, 2917, 1904, 3028, 1404,  887, 1404,  887,  887],[2863, 1224, 1582, 2863, 2390, 1224, 2390, 2863, 2863, 1582]]\n",
    "# features_all = [[1224, 2390, 2863, 1582],[3028,  887, 2917, 1904, 3028, 1404,  887, 1404,  887,  887]]\n",
    "features_all = [[596, 1184, 1495, 1864, 2320],[11]] # positive\n",
    "features_all = [[46, 70, 74, 97, 146],[29]] # negative\n",
    "features_all = [[14, 98, 302, 489],[203]] # negative\n",
    "\n",
    "ind_num = 1\n",
    "features = features_all[ind_num]\n",
    "cache_name = cache_names[ind_num]\n",
    "autoencoder = autoencoders[ind_num]\n",
    "\n",
    "num_feature_datapoints = 10\n",
    "dictionary_activations, tokens_for_each_datapoint = get_dictionary_activations(model, dataset, cache_name, max_seq_length, autoencoder, batch_size=32)\n",
    "\n",
    "# features = [1,2,3,4,5,6,7,8,9,10]\n",
    "# features = [5,6]\n",
    "# features = None\n",
    "num_features = 30\n",
    "# feature = 0\n",
    "input_setting = \"input_only\"\n",
    "for feature in features:\n",
    "    # Check if feature is dead (<10 activations)\n",
    "    dead_threshold = 1\n",
    "    # if(dictionary_activations[:, current_feature].count_nonzero() < dead_threshold):\n",
    "    if(dictionary_activations[:, feature].count_nonzero() < dead_threshold):\n",
    "        print(f\"Feature {feature} is dead\")\n",
    "        # Go to next feature\n",
    "        continue\n",
    "    print(f\"Feature {feature} is alive\")\n",
    "    uniform_indices = get_feature_indices(feature, dictionary_activations, k=num_feature_datapoints, setting=\"uniform\")\n",
    "    # uniform_indices = get_feature_indices(feature, dictionary_activations, k=num_feature_datapoints, setting=\"max\")\n",
    "    text_list, full_text, token_list, full_token_list, partial_activations, full_activations = get_feature_datapoints(uniform_indices, dictionary_activations[:, feature], tokenizer, max_seq_length, dataset)\n",
    "    # get_token_statistics(feature, dictionary_activations[:, feature], dataset, tokenizer, max_seq_length, tokens_for_each_datapoint, save_location = save_path, num_unique_tokens=10)\n",
    "    if(input_setting == \"input_only\"):\n",
    "        # Calculate logit diffs on this feature for the full_token_list\n",
    "        logit_diffs = ablate_feature_direction(model, full_token_list, cache_name, max_seq_length, autoencoder, feature = feature, batch_size=32, setting=\"sentences\", model_type=model_type)\n",
    "        # save_token_display(full_token_list, full_activations, tokenizer, path =f\"{save_path}uniform_{feature}.png\", logit_diffs = logit_diffs, model_type=model_type)\n",
    "        save_token_display(full_token_list, full_activations, tokenizer, path =f\"{save_path}uniform_{feature}.png\", logit_diffs = logit_diffs, model_type=model_type, show=True)\n",
    "        all_changed_activations = ablate_context_one_token_at_a_time(model, token_list, cache_name, autoencoder, feature, max_ablation_length=30)\n",
    "        save_token_display(token_list, all_changed_activations, tokenizer, path =f\"{save_path}ablate_context_{feature}.png\", model_type=model_type, show=True)\n",
    "    else:\n",
    "        logit_diffs = ablate_feature_direction(model, dataset, cache_name, max_seq_length, autoencoder, feature = feature, batch_size=32, setting=\"dataset\")\n",
    "        _, _, _, full_token_list_ablated, _, full_activations_ablated = get_feature_datapoints(uniform_indices, logit_diffs, tokenizer, max_seq_length, dataset)\n",
    "        get_token_statistics(feature, logit_diffs, dataset, tokenizer, max_seq_length, tokens_for_each_datapoint, save_location = save_path, setting=\"output\", num_unique_tokens=10)\n",
    "        save_token_display(full_token_list_ablated, full_activations, tokenizer, path =f\"{save_path}uniform_{feature}.png\", logit_diffs = full_activations_ablated)\n",
    "    # if(ablate_context ==True):\n",
    "    #     all_changed_activations = ablate_context_one_token_at_a_time(model, dataset, cache_name, autoencoder, feature, max_ablation_length=20)\n",
    "    #     save_token_display(token_list, all_changed_activations, tokenizer, path =f\"{save_path}ablate_context_{feature}.png\", model_type=model_type, show=True)\n",
    "    # combine_images(feature, setting=input_setting, ablate_context=ablate_context)\n",
    "    feature += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sparsity = dictionary_activations[0:100].count_nonzero(dim=1).float().mean().item()\n",
    "print(f\"Sparsity: {sparsity}\")\n",
    "dead_features = dictionary_activations[:100000].sum(0).count_nonzero().item()\n",
    "print(f\"Alive features: {dead_features}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dictionary_activations.sum(0).count_nonzero().item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dictionary_activations.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get_token_statistics(feature, dictionary_activations[:, feature], dataset, tokenizer, max_seq_length, tokens_for_each_datapoint, save_location = save_path, num_unique_tokens=10)\n",
    "setting = \"input\"\n",
    "feature_activation = dictionary_activations[:, feature]\n",
    "save_location=\"\"\n",
    "num_unique_tokens=10\n",
    "negative_threshold=-0.01\n",
    "\n",
    "if(setting==\"input\"):\n",
    "    nonzero_indices = feature_activation.nonzero()[:, 0]  # Get the nonzero indices\n",
    "else:\n",
    "    nonzero_indices = (feature_activation < negative_threshold).nonzero()[:, 0]\n",
    "nonzero_values = feature_activation[nonzero_indices].abs()  # Get the nonzero values\n",
    "\n",
    "# Unravel the indices to get the token IDs\n",
    "datapoint_indices = [np.unravel_index(i, (dataset.num_rows, max_seq_length)) for i in nonzero_indices]\n",
    "all_tokens = [dataset[int(md)][\"input_ids\"][int(s_ind)] for md, s_ind in datapoint_indices]\n",
    "\n",
    "# Find the max value for each unique token\n",
    "token_value_dict = defaultdict(int)\n",
    "for token, value in zip(all_tokens, nonzero_values):\n",
    "    token_value_dict[token] = max(token_value_dict[token], value)\n",
    "# if(setting==\"input\"):\n",
    "sorted_tokens = sorted(token_value_dict.keys(), key=lambda x: -token_value_dict[x])\n",
    "# else:\n",
    "#     sorted_tokens = sorted(token_value_dict.keys(), key=lambda x: token_value_dict[x])\n",
    "# Take the top 10 (or fewer if there aren't 10)\n",
    "max_tokens = sorted_tokens[:min(num_unique_tokens, len(sorted_tokens))]\n",
    "total_sums = nonzero_values.abs().sum()\n",
    "max_token_sums = []\n",
    "token_activations = []\n",
    "assert len(max_tokens) > 0, \"poo poo pee pee\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_activation.count_nonzero()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dictionary_activations.count_nonzero(dim=0)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.stack(full_token_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run example text through model\n",
    "text = \"I like to eat pizza.\"\n",
    "tokens = tokenizer(text, return_tensors=\"pt\").input_ids.to(device)\n",
    "# outputs = model(tokens)\n",
    "# outputs.logits\n",
    "\n",
    "device = model.device\n",
    "def less_than_rank_1_ablate(value, hook):\n",
    "    # Only ablate the feature direction up to the negative bias\n",
    "    # ie Only subtract when it activates above that negative bias.\n",
    "\n",
    "    # Rearrange to fit autoencoder\n",
    "    int_val = rearrange(value, 'b s h -> (b s) h')\n",
    "    # Run through the autoencoder\n",
    "    act = autoencoder.encode(int_val)\n",
    "    dictionary_for_this_autoencoder = autoencoder.get_learned_dict()\n",
    "    feature_direction = torch.outer(act[:, feature].squeeze(), dictionary_for_this_autoencoder[feature].squeeze())\n",
    "    batch, seq_len, hidden_size = value.shape\n",
    "    feature_direction = rearrange(feature_direction, '(b s) h -> b s h', b=batch, s=seq_len)\n",
    "    value -= feature_direction\n",
    "    return value\n",
    "\n",
    "# with Trace(model, cache_name, edit_output=less_than_rank_1_ablate) as ret:\n",
    "#     _ = model(batch).logits\n",
    "#     representation = ret.output\n",
    "#     # check if instance tuple\n",
    "#     if(isinstance(representation, tuple)):\n",
    "#         representation = representation[0]\n",
    "\n",
    "\n",
    "batch = tokens\n",
    "original_logits = model(batch).logits.log_softmax(dim=-1)\n",
    "# with Trace(model, cache_name, edit_output=less_than_rank_1_ablate) as ret:\n",
    "with Trace(model, cache_name, edit_output=less_than_rank_1_ablate) as ret:\n",
    "    ablated_logits = model(batch).logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# def ablate_feature_direction(model, dataset, cache_name, max_seq_length, autoencoder, feature, batch_size=32):\n",
    "\n",
    "device = model.cfg.device\n",
    "def less_than_rank_1_ablate(value, hook):\n",
    "    # Only ablate the feature direction up to the negative bias\n",
    "    # ie Only subtract when it activates above that negative bias.\n",
    "    # Rearrange to fit autoencoder\n",
    "    int_val = rearrange(value, 'b s h -> (b s) h')\n",
    "    # Run through the autoencoder\n",
    "    act = autoencoder.encode(int_val)\n",
    "    dictionary_for_this_autoencoder = autoencoder.get_learned_dict()\n",
    "    feature_direction = torch.outer(act[:, feature].squeeze(), dictionary_for_this_autoencoder[feature].squeeze())\n",
    "    batch, seq_len, hidden_size = value.shape\n",
    "    feature_direction = rearrange(feature_direction, '(b s) h -> b s h', b=batch, s=seq_len)\n",
    "    value -= feature_direction\n",
    "    return value\n",
    "\n",
    "# tokens_batched = full_tokens_list\n",
    "tokens_batched = torch.stack(full_token_list)\n",
    "logit_diffs = torch.zeros((tokens_batched.shape[0], tokens_batched.shape[1]-1))\n",
    "\n",
    "with torch.no_grad(), dataset.formatted_as(\"pt\"):\n",
    "    original_logits = model(tokens_batched.to(device)).log_softmax(dim=-1)\n",
    "    ablated_logits = model.run_with_hooks(tokens_batched.to(device), fwd_hooks=[(cache_name, less_than_rank_1_ablate)]).log_softmax(dim=-1)\n",
    "    diff_logits = ablated_logits  - original_logits# ablated > original -> negative diff\n",
    "    gather_tokens = rearrange(tokens_batched[:,1:].to(device), \"b s -> b s 1\")\n",
    "    gathered = diff_logits[:, :-1].gather(-1,gather_tokens)\n",
    "    # append all 0's to the beggining of gathered\n",
    "    gathered = torch.cat([torch.zeros((gathered.shape[0],1,1)).to(device), gathered], dim=1)\n",
    "    logit_diffs = rearrange(gathered, \"b s n -> (b s n)\").cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_colorbar(min_value, max_value, white = 255, red_blue_ness = 250, positive_threshold = 0.01, negative_threshold = 0.01):\n",
    "    # Add color bar\n",
    "    colorbar = \"\"\n",
    "    num_colors = 4\n",
    "    if(min_value < -negative_threshold):\n",
    "        for i in range(num_colors, 0, -1):\n",
    "            ratio = i / (num_colors)\n",
    "            value = round((min_value*ratio),1)\n",
    "            text_color = \"255,255,255\" if ratio > 0.5 else \"0,0,0\"\n",
    "            colorbar += f'<span style=\"background-color:rgba(255, {int(red_blue_ness-(red_blue_ness*ratio))},{int(red_blue_ness-(red_blue_ness*ratio))},1); color:rgb({text_color})\">&nbsp{value}&nbsp</span>'\n",
    "    # Do zero\n",
    "    colorbar += f'<span style=\"background-color:rgba({white},{white},{white},1);color:rgb(0,0,0)\">&nbsp0.0&nbsp</span>'\n",
    "    # Do positive\n",
    "    if(max_value > positive_threshold):\n",
    "        for i in range(1, num_colors+1):\n",
    "            ratio = i / (num_colors)\n",
    "            value = round((max_value*ratio),1)\n",
    "            text_color = \"255,255,255\" if ratio > 0.5 else \"0,0,0\"\n",
    "            colorbar += f'<span style=\"background-color:rgba({int(red_blue_ness-(red_blue_ness*ratio))},{int(red_blue_ness-(red_blue_ness*ratio))},255,1);color:rgb({text_color})\">&nbsp{value}&nbsp</span>'\n",
    "    return colorbar\n",
    "\n",
    "def value_to_color(activation, max_value, min_value, white = 255, red_blue_ness = 250, positive_threshold = 0.01, negative_threshold = 0.01):\n",
    "    if activation > positive_threshold:\n",
    "        ratio = activation/max_value\n",
    "        text_color = \"0,0,0\" if ratio <= 0.5 else \"255,255,255\"  \n",
    "        background_color = f'rgba({int(red_blue_ness-(red_blue_ness*ratio))},{int(red_blue_ness-(red_blue_ness*ratio))},255,1)'\n",
    "    elif activation < -negative_threshold:\n",
    "        ratio = activation/min_value\n",
    "        text_color = \"0,0,0\" if ratio <= 0.5 else \"255,255,255\"  \n",
    "        background_color = f'rgba(255, {int(red_blue_ness-(red_blue_ness*ratio))},{int(red_blue_ness-(red_blue_ness*ratio))},1)'\n",
    "    else:\n",
    "        text_color = \"0,0,0\"\n",
    "        background_color = f'rgba({white},{white},{white},1)'\n",
    "    return text_color, background_color\n",
    "\n",
    "def convert_token_array_to_list(array):\n",
    "    if isinstance(array, torch.Tensor):\n",
    "        if array.dim() == 1:\n",
    "            array = [array.tolist()]\n",
    "        elif array.dim()==2:\n",
    "            array = array.tolist()\n",
    "        else: \n",
    "            raise NotImplementedError(\"tokens must be 1 or 2 dimensional\")\n",
    "    elif isinstance(array, list):\n",
    "        # ensure it's a list of lists\n",
    "        if isinstance(array[0], int):\n",
    "            array = [array]\n",
    "    return array\n",
    "\n",
    "def tokens_and_activations_to_html(toks, activations, tokenizer, logit_diffs=None):\n",
    "    # text_spacing = \"0.07em\"\n",
    "    text_spacing = \"0.00em\"\n",
    "    toks = convert_token_array_to_list(toks)\n",
    "    activations = convert_token_array_to_list(activations)\n",
    "    # toks = [[tokenizer.decode(t).replace('Ġ', '&nbsp').replace('\\n', '↵') for t in tok] for tok in toks]\n",
    "    toks = [[tokenizer.decode(t).replace('Ġ', '&nbsp').replace('\\n', '\\\\n') for t in tok] for tok in toks]\n",
    "    highlighted_text = []\n",
    "    # Make background black\n",
    "    # highlighted_text.append('<body style=\"background-color:black; color: white;\">')\n",
    "    highlighted_text.append(\"\"\"\n",
    "<body style=\"background-color: black; color: white;\">\n",
    "\"\"\")\n",
    "    max_value = max([max(activ) for activ in activations])\n",
    "    min_value = min([min(activ) for activ in activations])\n",
    "    if(logit_diffs):\n",
    "        logit_max_value = max([max(activ) for activ in logit_diffs])\n",
    "        logit_min_value = min([min(activ) for activ in logit_diffs])\n",
    "\n",
    "    # Add color bar\n",
    "    highlighted_text.append(\"Token Activations: \" + make_colorbar(min_value, max_value))\n",
    "    if(logit_diffs):\n",
    "        highlighted_text.append('<div style=\"margin-top: 0.1em;\"></div>')\n",
    "        highlighted_text.append(\"Logit Diff: \" + make_colorbar(logit_min_value, logit_max_value))\n",
    "    \n",
    "    highlighted_text.append('<div style=\"margin-top: 0.5em;\"></div>')\n",
    "    for seq_ind, (act, tok) in enumerate(zip(activations, toks)):\n",
    "        for act_ind, (a, t) in enumerate(zip(act, tok)):\n",
    "            if(logit_diffs):\n",
    "                highlighted_text.append('<div style=\"display: inline-block;\">')\n",
    "            text_color, background_color = value_to_color(a, max_value, min_value)\n",
    "            highlighted_text.append(f'<span style=\"background-color:{background_color};margin-right: {text_spacing}; color:rgb({text_color})\">{t.replace(\" \", \"&nbsp\")}</span>')\n",
    "            if(logit_diffs):\n",
    "                logit_diffs_act = logit_diffs[seq_ind][act_ind]\n",
    "                _, logit_background_color = value_to_color(logit_diffs_act, logit_max_value, logit_min_value)\n",
    "                highlighted_text.append(f'<div style=\"display: block; margin-right: {text_spacing}; height: 10px; background-color:{logit_background_color}; text-align: center;\"></div></div>')\n",
    "        highlighted_text.append('<div style=\"margin-top: 0.2em;\"></div>')\n",
    "        # highlighted_text.append('<br><br>')\n",
    "    # highlighted_text.append('</body>')\n",
    "    highlighted_text = ''.join(highlighted_text)\n",
    "    return highlighted_text\n",
    "\n",
    "def display_tokens(tokens, activations, tokenizer, logit_diffs=None):\n",
    "    return display(HTML(tokens_and_activations_to_html(tokens, activations, tokenizer, logit_diffs)))\n",
    "\n",
    "def save_token_display(tokens, activations, tokenizer, path, logit_diffs=None):\n",
    "    html = tokens_and_activations_to_html(tokens, activations, tokenizer, logit_diffs)\n",
    "    imgkit.from_string(html, path)\n",
    "    # print(f\"Saved to {path}\")\n",
    "    return\n",
    "save_token_display(full_token_list_ablated, full_activations, model.tokenizer, path =f\"{save_path}uniform_black_{feature}.png\", logit_diffs = full_activations_ablated)\n",
    "# display_tokens(full_token_list_ablated, full_activations, model.tokenizer, logit_diffs = full_activations_ablated)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image, ImageDraw, ImageFont\n",
    "\n",
    "# Load an existing image\n",
    "image = Image.open(f\"concatenated_image.png\")\n",
    "\n",
    "# Create a drawing object\n",
    "draw = ImageDraw.Draw(image)\n",
    "\n",
    "# Define the position for the text to be placed (this example puts it at the top-center)\n",
    "text = \"Feature {feature}\"\n",
    "text_position = (image.width // 2, 10)\n",
    "\n",
    "# You can use a truetype or opentype font file if you want to customize\n",
    "# Otherwise, PIL provides a basic built-in font\n",
    "try:\n",
    "    font = ImageFont.truetype(\"arial.ttf\", size=30)\n",
    "except IOError:\n",
    "    font = ImageFont.load_default()\n",
    "\n",
    "# Calculate text size and position to center-align the text\n",
    "text_width, text_height = draw.textsize(text, font=font)\n",
    "text_position = ((image.width - text_width) // 2, 100)\n",
    "\n",
    "# Add the text to image\n",
    "draw.text(text_position, text, font=font, fill=\"white\")\n",
    "\n",
    "# Save or show the image\n",
    "image.save(\"image_with_title.png\")\n",
    "image.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "font.font.getsize(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_position, text, font"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.core.display import display, HTML\n",
    "\n",
    "# The initial HTML and CSS to set the background to black\n",
    "initial_html = \"\"\"\n",
    "<!DOCTYPE html>\n",
    "<html>\n",
    "<head>\n",
    "    <title>My Page</title>\n",
    "</head>\n",
    "<body style=\"background-color: blue;\">\n",
    "\"\"\"\n",
    "\n",
    "# The HTML for the text spans that you'll append later\n",
    "span_text = \"\"\"\n",
    "    <span style=\"color: white;\">This is some text.</span>\n",
    "    <span style=\"color: green;\">This is some more text.</span>\n",
    "\"\"\"\n",
    "\n",
    "# Closing tags for the HTML\n",
    "closing_html = \"\"\"\n",
    "</body>\n",
    "</html>\n",
    "\"\"\"\n",
    "\n",
    "# Combine everything\n",
    "full_html = initial_html + span_text + closing_html\n",
    "\n",
    "# Display the HTML\n",
    "display(HTML(full_html))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.cfg.device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "d_model = model.cfg.d_model\n",
    "assert (d_model == autoencoder.encoder.shape[-1]), f\"Model and autoencoder must have same hidden size. Model: {d_model}, Autoencoder: {autoencoder.encoder.shape[-1]}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now we can use the model to get the activations\n",
    "from torch.utils.data import DataLoader\n",
    "from tqdm.auto import tqdm\n",
    "from einops import rearrange\n",
    "def get_dictionary_activations(model, dataset, cache_name, autoencoder, batch_size=32):\n",
    "    num_features, d_model = autoencoder.encoder.shape\n",
    "    datapoints = dataset.num_rows\n",
    "    dictionary_activations = torch.zeros((datapoints*max_seq_length, num_features))\n",
    "    token_list = torch.zeros((datapoints*max_seq_length), dtype=torch.int64)\n",
    "    with torch.no_grad(), dataset.formatted_as(\"pt\"):\n",
    "        dl = DataLoader(dataset[\"input_ids\"], batch_size=batch_size)\n",
    "        for i, batch in enumerate(tqdm(dl)):\n",
    "            token_list[i*batch_size*max_seq_length:(i+1)*batch_size*max_seq_length] = rearrange(batch, \"b s -> (b s)\")\n",
    "            _, cache = model.run_with_cache(batch.to(device))\n",
    "            batched_neuron_activations = rearrange(cache[cache_name], \"b s n -> (b s) n\" )\n",
    "            batched_dictionary_activations = autoencoder.encode(batched_neuron_activations)\n",
    "            dictionary_activations[i*batch_size*max_seq_length:(i+1)*batch_size*max_seq_length,:] = batched_dictionary_activations.cpu()\n",
    "    return dictionary_activations, token_list\n",
    "\n",
    "print(\"Getting dictionary activations\")\n",
    "dictionary_activations, tokens_for_each_datapoint = get_dictionary_activations(model, dataset, cache_name, autoencoder, batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def ablate_feature_direction(model, dataset, cache_name, autoencoder, feature, batch_size=32):\n",
    "    def less_than_rank_1_ablate(value, hook):\n",
    "        # Only ablate the feature direction up to the negative bias\n",
    "        # ie Only subtract when it activates above that negative bias.\n",
    "\n",
    "        # Rearrange to fit autoencoder\n",
    "        int_val = rearrange(value, 'b s h -> (b s) h')\n",
    "        # Run through the autoencoder\n",
    "        act = autoencoder.encode(int_val)\n",
    "        dictionary_for_this_autoencoder = autoencoder.get_learned_dict()\n",
    "        feature_direction = torch.outer(act[:, feature].squeeze(), dictionary_for_this_autoencoder[feature].squeeze())\n",
    "        batch, seq_len, hidden_size = value.shape\n",
    "        feature_direction = rearrange(feature_direction, '(b s) h -> b s h', b=batch, s=seq_len)\n",
    "        value -= feature_direction\n",
    "        return value\n",
    "\n",
    "    datapoints = dataset.num_rows\n",
    "    logit_diffs = torch.zeros((datapoints*max_seq_length))\n",
    "    with torch.no_grad(), dataset.formatted_as(\"pt\"):\n",
    "        dl = DataLoader(dataset[\"input_ids\"], batch_size=batch_size)\n",
    "        for i, batch in enumerate(tqdm(dl)):\n",
    "            original_logits = model(batch.to(device)).log_softmax(dim=-1)\n",
    "            ablated_logits = model.run_with_hooks(batch.to(device), fwd_hooks=[(cache_name, less_than_rank_1_ablate)]).log_softmax(dim=-1)\n",
    "            diff_logits = ablated_logits  - original_logits# ablated > original -> negative diff\n",
    "            gather_tokens = rearrange(batch[:,1:].to(device), \"b s -> b s 1\")\n",
    "            gathered = diff_logits[:, :-1].gather(-1,gather_tokens)\n",
    "            # append all 0's to the beggining of gathered\n",
    "            gathered = torch.cat([torch.zeros((gathered.shape[0],1,1)).to(device), gathered], dim=1)\n",
    "            diff = rearrange(gathered, \"b s n -> (b s n)\")\n",
    "            # Add one to the first position of logit diff, so we're always skipping over the first token (since it's not predicted)\n",
    "            logit_diffs[i*batch_size*max_seq_length:(i+1)*batch_size*max_seq_length] = diff.cpu()\n",
    "    return logit_diffs\n",
    "feature = 1\n",
    "logit_diffs = ablate_feature_direction(model, dataset, cache_name, autoencoder, feature = feature, batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # from interp_utils import *\n",
    "# if isinstance(features, int):\n",
    "#     features = [features]\n",
    "# for feature in features:\n",
    "#     text_list, full_text, token_list, full_token_list = get_feature_datapoints(feature, dictionary_activations, model.tokenizer, max_seq_length, dataset, setting=\"uniform\")\n",
    "#     # text_list, full_text, token_list, full_token_list = get_feature_datapoints(feature, dictionary_activations, dataset, setting=\"max\")\n",
    "#     # visualize_text(full_text, feature, model, autoencoder, layer)\n",
    "# l = visualize_text(text_list, feature, model, autoencoder, layer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import display, HTML\n",
    "import imgkit\n",
    "\n",
    "def make_colorbar(min_value, max_value, white = 245, red_blue_ness = 250, positive_threshold = 0.01, negative_threshold = 0.01):\n",
    "    # Add color bar\n",
    "    colorbar = \"\"\n",
    "    num_colors = 4\n",
    "    if(min_value < -negative_threshold):\n",
    "        for i in range(num_colors, 0, -1):\n",
    "            ratio = i / (num_colors)\n",
    "            value = round((min_value*ratio),1)\n",
    "            text_color = \"255,255,255\" if ratio > 0.5 else \"0,0,0\"\n",
    "            colorbar += f'<span style=\"background-color:rgba(255, {int(red_blue_ness-(red_blue_ness*ratio))},{int(red_blue_ness-(red_blue_ness*ratio))},1); color:rgb({text_color})\">&nbsp{value}&nbsp</span>'\n",
    "    # Do zero\n",
    "    colorbar += f'<span style=\"background-color:rgba({white},{white},{white},1);color:rgb(0,0,0)\">&nbsp0.0&nbsp</span>'\n",
    "    # Do positive\n",
    "    if(max_value > positive_threshold):\n",
    "        for i in range(1, num_colors+1):\n",
    "            ratio = i / (num_colors)\n",
    "            value = round((max_value*ratio),1)\n",
    "            text_color = \"255,255,255\" if ratio > 0.5 else \"0,0,0\"\n",
    "            colorbar += f'<span style=\"background-color:rgba({int(red_blue_ness-(red_blue_ness*ratio))},{int(red_blue_ness-(red_blue_ness*ratio))},255,1);color:rgb({text_color})\">&nbsp{value}&nbsp</span>'\n",
    "    return colorbar\n",
    "\n",
    "def value_to_color(activation, max_value, min_value, white = 245, red_blue_ness = 250, positive_threshold = 0.01, negative_threshold = 0.01):\n",
    "    if activation > positive_threshold:\n",
    "        ratio = activation/max_value\n",
    "        text_color = \"0,0,0\" if ratio <= 0.5 else \"255,255,255\"  \n",
    "        background_color = f'rgba({int(red_blue_ness-(red_blue_ness*ratio))},{int(red_blue_ness-(red_blue_ness*ratio))},255,1)'\n",
    "    elif activation < -negative_threshold:\n",
    "        ratio = activation/min_value\n",
    "        text_color = \"0,0,0\" if ratio <= 0.5 else \"255,255,255\"  \n",
    "        background_color = f'rgba(255, {int(red_blue_ness-(red_blue_ness*ratio))},{int(red_blue_ness-(red_blue_ness*ratio))},1)'\n",
    "    else:\n",
    "        text_color = \"0,0,0\"\n",
    "        background_color = f'rgba({white},{white},{white},1)'\n",
    "    return text_color, background_color\n",
    "\n",
    "def convert_token_array_to_list(array):\n",
    "    if isinstance(array, torch.Tensor):\n",
    "        if array.dim() == 1:\n",
    "            array = [array.tolist()]\n",
    "        elif array.dim()==2:\n",
    "            array = array.tolist()\n",
    "        else: \n",
    "            raise NotImplementedError(\"tokens must be 1 or 2 dimensional\")\n",
    "    elif isinstance(array, list):\n",
    "        # ensure it's a list of lists\n",
    "        if isinstance(array[0], int):\n",
    "            array = [array]\n",
    "    return array\n",
    "\n",
    "def tokens_and_activations_to_html(toks, activations, tokenizer, logit_diffs=None):\n",
    "    toks = convert_token_array_to_list(toks)\n",
    "    activations = convert_token_array_to_list(activations)\n",
    "    # toks = [[tokenizer.decode(t).replace('Ġ', '&nbsp').replace('\\n', '↵') for t in tok] for tok in toks]\n",
    "    toks = [[tokenizer.decode(t).replace('Ġ', '&nbsp').replace('\\n', '\\\\n') for t in tok] for tok in toks]\n",
    "    highlighted_text = []\n",
    "    max_value = max([max(activ) for activ in activations])\n",
    "    min_value = min([min(activ) for activ in activations])\n",
    "    if(logit_diffs):\n",
    "        logit_max_value = max([max(activ) for activ in logit_diffs])\n",
    "        logit_min_value = min([min(activ) for activ in logit_diffs])\n",
    "\n",
    "    # Add color bar\n",
    "    highlighted_text.append(\"Token Activations: \" + make_colorbar(min_value, max_value))\n",
    "    if(logit_diffs):\n",
    "        highlighted_text.append('<br><br>')\n",
    "        highlighted_text.append(\"Logit Diff: \" + make_colorbar(logit_min_value, logit_max_value))\n",
    "        \n",
    "    highlighted_text.append('<br><br>')\n",
    "    for seq_ind, (act, tok) in enumerate(zip(activations, toks)):\n",
    "        for act_ind, (a, t) in enumerate(zip(act, tok)):\n",
    "            if(logit_diffs):\n",
    "                highlighted_text.append('<div style=\"display: inline-block;\">')\n",
    "            text_color, background_color = value_to_color(a, max_value, min_value)\n",
    "            highlighted_text.append(f'<span style=\"background-color:{background_color};color:rgb({text_color})\">{t.replace(\" \", \"&nbsp\")}</span>')\n",
    "            if(logit_diffs):\n",
    "                logit_diffs_act = logit_diffs[seq_ind][act_ind]\n",
    "                _, logit_background_color = value_to_color(logit_diffs_act, logit_max_value, logit_min_value)\n",
    "                highlighted_text.append(f'<div style=\"display: block; height: 10px; background-color:{logit_background_color}; text-align: center;\"></div></div>')\n",
    "        highlighted_text.append('<br><br>')\n",
    "    highlighted_text = ''.join(highlighted_text)\n",
    "    return highlighted_text\n",
    "\n",
    "def display_tokens(tokens, activations, tokenizer, logit_diffs=None):\n",
    "    return display(HTML(tokens_and_activations_to_html(tokens, activations, tokenizer, logit_diffs)))\n",
    "\n",
    "def save_token_display(tokens, activations, tokenizer, path, logit_diffs=None):\n",
    "    html = tokens_and_activations_to_html(tokens, activations, tokenizer, logit_diffs)\n",
    "    imgkit.from_string(html, path)\n",
    "    # print(f\"Saved to {path}\")\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_feature_indices(feature_index, dictionary_activations, tokenizer, token_amount, dataset, k=10, setting=\"max\"):\n",
    "    best_feature_activations = dictionary_activations[:, feature_index]\n",
    "    # Sort the features by activation, get the indices\n",
    "    if setting==\"max\":\n",
    "        found_indices = torch.argsort(best_feature_activations, descending=True)[:k]\n",
    "    elif setting==\"uniform\":\n",
    "        # min_value = torch.min(best_feature_activations)\n",
    "        min_value = torch.min(best_feature_activations)\n",
    "        max_value = torch.max(best_feature_activations)\n",
    "\n",
    "        # Define the number of bins\n",
    "        num_bins = k\n",
    "\n",
    "        # Calculate the bin boundaries as linear interpolation between min and max\n",
    "        bin_boundaries = torch.linspace(min_value, max_value, num_bins + 1)\n",
    "\n",
    "        # Assign each activation to its respective bin\n",
    "        bins = torch.bucketize(best_feature_activations, bin_boundaries)\n",
    "\n",
    "        # Initialize a list to store the sampled indices\n",
    "        sampled_indices = []\n",
    "\n",
    "        # Sample from each bin\n",
    "        for bin_idx in torch.unique(bins):\n",
    "            if(bin_idx==0): # Skip the first one. This is below the median\n",
    "                continue\n",
    "            # Get the indices corresponding to the current bin\n",
    "            bin_indices = torch.nonzero(bins == bin_idx, as_tuple=False).squeeze(dim=1)\n",
    "            \n",
    "            # Randomly sample from the current bin\n",
    "            sampled_indices.extend(np.random.choice(bin_indices, size=1, replace=False))\n",
    "\n",
    "        # Convert the sampled indices to a PyTorch tensor & reverse order\n",
    "        found_indices = torch.tensor(sampled_indices).long().flip(dims=[0])\n",
    "    else: # random\n",
    "        # get nonzero indices\n",
    "        nonzero_indices = torch.nonzero(best_feature_activations)[:, 0]\n",
    "        # shuffle\n",
    "        shuffled_indices = nonzero_indices[torch.randperm(nonzero_indices.shape[0])]\n",
    "        found_indices = shuffled_indices[:k]\n",
    "    return found_indices\n",
    "def get_feature_datapoints(found_indices, best_feature_activations, tokenizer, token_amount, dataset):\n",
    "    num_datapoints = dataset.num_rows\n",
    "    datapoint_indices =[np.unravel_index(i, (num_datapoints, token_amount)) for i in found_indices]\n",
    "    all_activations = best_feature_activations.reshape(num_datapoints, token_amount).tolist()\n",
    "    full_activations = []\n",
    "    partial_activations = []\n",
    "    text_list = []\n",
    "    full_text = []\n",
    "    token_list = []\n",
    "    full_token_list = []\n",
    "    for i, (md, s_ind) in enumerate(datapoint_indices):\n",
    "        md = int(md)\n",
    "        s_ind = int(s_ind)\n",
    "        full_tok = torch.tensor(dataset[md][\"input_ids\"])\n",
    "        full_text.append(tokenizer.decode(full_tok))\n",
    "        tok = dataset[md][\"input_ids\"][:s_ind+1]\n",
    "        full_activations.append(all_activations[md])\n",
    "        partial_activations.append(all_activations[md][:s_ind+1])\n",
    "        text = tokenizer.decode(tok)\n",
    "        text_list.append(text)\n",
    "        token_list.append(tok)\n",
    "        full_token_list.append(full_tok)\n",
    "    return text_list, full_text, token_list, full_token_list, partial_activations, full_activations\n",
    "\n",
    "uniform_indices = get_feature_indices(feature, dictionary_activations, model.tokenizer, max_seq_length, dataset, k=10, setting=\"uniform\")\n",
    "text_list, full_text, token_list, full_token_list, partial_activations, full_activations = get_feature_datapoints(uniform_indices, dictionary_activations[:, feature], model.tokenizer, max_seq_length, dataset)\n",
    "display_tokens(token_list, partial_activations, model.tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def get_token_statistics(feature, feature_activation, dataset, max_seq_length, save_location=\"\", num_unique_tokens=10, setting=\"input\", negative_threshold=-0.01):\n",
    "    if(setting==\"input\"):\n",
    "        nonzero_indices = feature_activation.nonzero()[:, 0]  # Get the nonzero indices\n",
    "    else:\n",
    "        nonzero_indices = (feature_activation < negative_threshold).nonzero()[:, 0]\n",
    "    nonzero_values = feature_activation[nonzero_indices].abs()  # Get the nonzero values\n",
    "\n",
    "    # Unravel the indices to get the token IDs\n",
    "    datapoint_indices = [np.unravel_index(i, (dataset.num_rows, max_seq_length)) for i in nonzero_indices]\n",
    "    all_tokens = [dataset[int(md)][\"input_ids\"][int(s_ind)] for md, s_ind in datapoint_indices]\n",
    "\n",
    "    # Find the max value for each unique token\n",
    "    token_value_dict = defaultdict(int)\n",
    "    for token, value in zip(all_tokens, nonzero_values):\n",
    "        token_value_dict[token] = max(token_value_dict[token], value)\n",
    "    # if(setting==\"input\"):\n",
    "    sorted_tokens = sorted(token_value_dict.keys(), key=lambda x: -token_value_dict[x])\n",
    "    # else:\n",
    "    #     sorted_tokens = sorted(token_value_dict.keys(), key=lambda x: token_value_dict[x])\n",
    "    # Take the top 10 (or fewer if there aren't 10)\n",
    "    max_tokens = sorted_tokens[:min(num_unique_tokens, len(sorted_tokens))]\n",
    "    total_sums = nonzero_values.abs().sum()\n",
    "    max_token_sums = []\n",
    "    token_activations = []\n",
    "    for max_token in max_tokens:\n",
    "        # Find ind of max token\n",
    "        max_token_indices = tokens_for_each_datapoint[nonzero_indices] == max_token\n",
    "        # Grab the values for those indices\n",
    "        max_token_values = nonzero_values[max_token_indices]\n",
    "        max_token_sum = max_token_values.abs().sum()\n",
    "        max_token_sums.append(max_token_sum)\n",
    "        token_activations.append(max_token_values)\n",
    "\n",
    "\n",
    "    if(setting==\"input\"):\n",
    "        title_text = \"Input Token Activations\"\n",
    "        save_name = \"input\"\n",
    "        y_label = \"Feature Activation\"\n",
    "    else:\n",
    "        title_text = \"Output Logit Difference\"\n",
    "        save_name = \"logit_diff\"\n",
    "        y_label = \"Logit Difference\"\n",
    "\n",
    "    # Plot a boxplot for each tensor in the list\n",
    "    fig, ax = plt.subplots(figsize=(10, 10))\n",
    "    ax.set_title(f'{title_text}: feature={feature}')\n",
    "    max_text = [model.tokenizer.decode([t]).replace(\"\\n\", \"\\\\n\").replace(\" \", \"_\") for t in max_tokens]\n",
    "    # Set x-axis label\n",
    "    ax.set_xlabel('Token')\n",
    "    #rotate x labels\n",
    "    plt.xticks(rotation=30)\n",
    "    # Set y-axis label\n",
    "    ax.set_ylabel(y_label)\n",
    "    ax.boxplot(token_activations[::-1], labels=max_text[::-1])\n",
    "    #Save it\n",
    "    plt.savefig(f'{save_location}feature_{feature}_{save_name}_boxplot.png')\n",
    "\n",
    "    #Bar graph of the percentage of total activations\n",
    "    fig, ax = plt.subplots(figsize=(10, 10))\n",
    "    ax.set_title(f'Weighted Percentage of {title_text}: feature={feature}')\n",
    "    max_text = [model.tokenizer.decode([t]).replace(\"\\n\", \"\\\\n\").replace(\" \", \"_\") for t in max_tokens]\n",
    "    # Set x-axis label\n",
    "    ax.set_xlabel('Token')\n",
    "    plt.xticks(rotation=30)\n",
    "\n",
    "    # Set y-axis label\n",
    "    ax.set_ylabel(f'Weighted Percentage of Total {y_label}')\n",
    "    ax.bar(max_text[::-1], [t/total_sums*100 for t in max_token_sums[::-1]])\n",
    "    plt.savefig(f'{save_location}feature_{feature}_{save_name}_bar.png')\n",
    "# get_token_statistics(feature, dictionary_activations[:, feature], dataset, max_seq_length, save_location = \"features/\", num_unique_tokens=10)\n",
    "get_token_statistics(feature, logit_diffs, dataset, max_seq_length, save_location = \"features/\", setting=\"output\", num_unique_tokens=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(logit_diffs < -0.01).nonzero()[:, 0], feature_activation.nonzero()[:, 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, _, _, full_token_list_ablated, _, full_activations_ablated = get_feature_datapoints(uniform_indices, logit_diffs, model.tokenizer, max_seq_length, dataset)\n",
    "display_tokens(full_token_list_ablated, full_activations, model.tokenizer, logit_diffs = full_activations_ablated)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens_and_activations_to_html(full_token_list_ablated, full_activations, model.tokenizer, logit_diffs = full_activations_ablated)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display(HTML('<div style=\"display: inline-block;\"><span style=\"background-color:rgba(245,245,245,1);color:rgb(0,0,0)\">About</span><div style=\"display: block; height: 10px; background-color:rgba(245,245,245,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(245,245,245,1);color:rgb(0,0,0)\">\\\\n</span><div style=\"display: block; height: 10px; background-color:rgba(245,245,245,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(245,245,245,1);color:rgb(0,0,0)\">\\\\n</span><div style=\"display: block; height: 10px; background-color:rgba(245,245,245,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(245,245,245,1);color:rgb(0,0,0)\">Te</span><div style=\"display: block; height: 10px; background-color:rgba(245,245,245,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(245,245,245,1);color:rgb(0,0,0)\">ams</span><div style=\"display: block; height: 10px; background-color:rgba(245,245,245,1); text-align: center;\"></div></div><div style=\"display: inline-block;\"><span style=\"background-color:rgba(245,245,245,1);color:rgb(0,0,0)\">&nbspare</span><div style=\"display: block; height: 10px; background-color:rgba(245,245,245,1); text-align: center;\"></div></div>'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#def ablate_context_one_token_at_a_time(model, dataset, cache_name, autoencoder, feature, batch_size=32):\n",
    "all_changed_activations = []\n",
    "for token_ind, token_l in enumerate(token_list):\n",
    "# for token_ind, token_l in enumerate(full_token_list):\n",
    "    seq_size = len(token_l)\n",
    "    original_activation = partial_activations[token_ind][-1]\n",
    "    # Run through the model for each seq length\n",
    "    if(seq_size==1):\n",
    "        continue # Size 1 sequences don't have any context to ablate\n",
    "    # changed_activations = torch.zeros(seq_size).cpu() + original_activation\n",
    "    changed_activations = torch.zeros(seq_size).cpu() \n",
    "    for i in range(seq_size-1):\n",
    "        # ablated_tokens = token_l[:i+1] + token_l[i+1:]\n",
    "        ablated_tokens = token_l\n",
    "        ablated_tokens = torch.tensor(ablated_tokens).unsqueeze(0)\n",
    "        with torch.no_grad():\n",
    "            _, cache = model.run_with_cache(ablated_tokens.to(device))\n",
    "            neuron_activations = rearrange(cache[cache_name], \"b s n -> (b s) n\" )\n",
    "            dictionary_activations = autoencoder.encode(neuron_activations)\n",
    "\n",
    "            changed_activations[i] += dictionary_activations[-1,feature].item()\n",
    "    changed_activations -= original_activation\n",
    "    all_changed_activations.append(changed_activations.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display_tokens(token_list, all_changed_activations, model.tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_changed_activations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display_tokens(token_list, partial_activations, model.tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ablate_feature_direction_display(full_text, autoencoder, model, layer, features=feature)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "logan",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}