04_Attack_LISA.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "mY_xKZiEnud9"
   },
   "source": [
    "### **Adversarial Attacks on LISA Dataset**\n",
    "\n",
    "Parameters for the **inconspicuous** attack on LISA data:\n",
    "* mask threshold 0.1\n",
    "* mask penalty for using colors (beta1) 0.1\n",
    "* mask regularization for noise (beta2) 0.01\n",
    "* mask deviation penalty (beta3) 4.0\n",
    "* attack LR 0.01\n",
    "* attack penalty for using colors (beta1) 1.0\n",
    "* attack deviation penalty (beta3) 4.0\n",
    "* 20000 attack training epochs\n",
    "\n",
    "\n",
    "Parameters for the **visible** attack on LISA data:\n",
    "* mask threshold 0.01\n",
    "* mask penalty for using colors (beta1) 0.0\n",
    "* mask regularization for noise (beta2) 0.0\n",
    "* mask deviation penalty (beta3) 1.0\n",
    "* attack LR 0.0001\n",
    "* attack penalty for using colors (beta1) 0.0\n",
    "* attack deviation penalty (beta3) 1.0\n",
    "* 50000 attack training epochs\n",
    "\n",
    "\n",
    "For both attacks:\n",
    "* 1000 mask training epochs\n",
    "* mask initialization LR 0.01\n",
    "\n",
    "* attack batch size 8\n",
    "\n",
    "* augmentation is not used for initialization, but used for attack\n",
    "\n",
    "For the attack on class 12 (stop), the target class is 11 (speed limit 45). For the attack on class 9 (speed limit 30), the target class is 12 (stop).\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **Invonspicuous Attack**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EfficientNet\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import os\n",
    "import json\n",
    "from models import *\n",
    "from utils import *\n",
    "from torchvision.models import mobilenet_v2, efficientnet_b0, densenet121, shufflenet_v2_x1_0\n",
    "\n",
    "# Parameters of LISA dataset (you can leave this as it is)\n",
    "with open('classes.json', 'r') as config:\n",
    "    params = json.load(config)\n",
    "    params['CLASS_N'] = params['LISA']['class_n']\n",
    "    params['DEVICE'] = params['device']\n",
    "    params['LABELS'] = params['LISA']['labels']\n",
    "\n",
    "    \n",
    "model_types = ['CNNsmall', 'CNNlarge', 'Transformer', 'ResNet18', 'MobileNetv2', 'DenseNet121', 'ShuffleNetv2x1', \"EfficientNet\"]\n",
    "\n",
    "\n",
    "model_type = \"EfficientNet\"\n",
    "print(model_type)\n",
    "\n",
    "params['MODEL_TYPE'] = model_type\n",
    "# Directory where output files will be stored\n",
    "params['OUTPUT_DIR'] = f\"attacks/inconspicuous{params['MODEL_TYPE']}LISA_ATTACK\"\n",
    "\n",
    "# --- PATHS INPUT---\n",
    "# Path to weights of baseline model to attack\n",
    "params['PATH_MODEL'] = f\"models/{model_type}LISA.pth\"\n",
    "# Set your hyperparameters (we add them to the params dictionary)\n",
    "\n",
    "# --- GENERAL PARAMETERS ---\n",
    "# Target class that should be recognized instead of orginal class of sign\n",
    "params['TARGET_CLASS'] = 2\n",
    "\n",
    "# Path of original sign to pertubate (should be in original printable size)\n",
    "params['PATH_SIGN'] = 'attack_utils/lisa_30.jpg'\n",
    "# Path to the sign mask (defines area of sign)\n",
    "params['PATH_SIGN_MASK'] = 'attack_utils/lisa_30_mask.jpg'\n",
    "\n",
    "# --- PATHS OUTPUT --- (ONLY CHANGE FILENAMES)\n",
    "# Path where to store the pertubated sign (printing size)\n",
    "params['PATH_PERT_SIGN_SMALL'] = os.path.join(params['OUTPUT_DIR'], 'pert_sign_small.png')\n",
    "# Path where to store the pertubated sign (printing size)\n",
    "params['PATH_PERT_SIGN_LARGE'] = os.path.join(params['OUTPUT_DIR'], 'pert_sign_large.png')\n",
    "# Path where to store the sign in the size of the model input\n",
    "params['PATH_ORIG_SIGN_SMALL'] = os.path.join(params['OUTPUT_DIR'], 'orig_sign_small_.png')\n",
    "# Path where to store the last augmented sign\n",
    "params['PATH_AUGMENTED_SIGN_SMALL'] = os.path.join(params['OUTPUT_DIR'], 'augmented_sign_small.png')\n",
    "# Path where to store the noise in small size\n",
    "params['PATH_NOISE_SMALL'] = os.path.join(params['OUTPUT_DIR'], 'noise_small.png')\n",
    "# Path where to store the noise in printable size\n",
    "params['PATH_NOISE_LARGE'] = os.path.join(params['OUTPUT_DIR'], 'noise_large.png')\n",
    "# Path where to store the noise mask\n",
    "params['PATH_NOISE_MASK_SMALL'] = os.path.join(params['OUTPUT_DIR'], 'noise_mask_small.png')\n",
    "# Path where to store the noise mask\n",
    "params['PATH_NOISE_MASK_LARGE'] = os.path.join(params['OUTPUT_DIR'], 'noise_mask_large.png')\n",
    "# Paht where to store noise tensor\n",
    "params['PATH_NOISE_TENSOR'] = os.path.join(params['OUTPUT_DIR'], 'noise_tensor.pt')\n",
    "\n",
    "# --- PARAMETERS FOR INITIALIZATION ---\n",
    "params['INIT_MASK_THRESHOLD'] = 0.1\n",
    "# How much you want to penalty using colors\n",
    "params['INIT_MASK_BETA_1'] = 0.1\n",
    "# How much you want to regularize the ammount of noise\n",
    "params['INIT_MASK_BETA_2'] = 0.01\n",
    "# How much you want to penalty the deviation from to target class\n",
    "params['INIT_MASK_BETA_3'] = 4\n",
    "# Learning rate of adam for the mask initialization\n",
    "params['INIT_MASK_LEARNING_RATE'] = 0.01\n",
    "# How many epochs you want to train the initialization of the mask\n",
    "params['INIT_MASK_EPOCHS'] = 1000\n",
    "# Turn augmentation for initialization of mask on/off\n",
    "params['INIT_MASK_AUGMENTATION'] = False\n",
    "\n",
    "\n",
    "# PARAMETERS FOR ATTACK\n",
    "# Learning rate of adam for the attack\n",
    "params['ATTACK_LEARNING_RATE'] = 0.01\n",
    "# Batch size for one optimization step\n",
    "params['ATTACK_BATCH_SIZE'] = 8\n",
    "# How much you want to penalty using colors in the attack\n",
    "params['ATTACK_BETA_1'] = 1\n",
    "# How much you want to regularize the ammount of noise in the attack\n",
    "params['ATTACK_BETA_2'] = 0.0\n",
    "# How much you want to penalty the deviation from to target class\n",
    "params['ATTACK_BETA_3'] = 4\n",
    "# How many epochs you want to train the attack\n",
    "params['ATTACK_EPOCHS'] = 20000\n",
    "# Turn augmentation on/off for attack\n",
    "params['ATTACK_AUGMENTATION'] = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction on the original sign:\n",
      "Class with the max confidence is  9: Speed Limit 30\n",
      "Predict: 9 (9: Speed Limit 30) Confidence: 62.22227215766907%\n"
     ]
    }
   ],
   "source": [
    "transform_to_tensor = transforms.Compose([transforms.Resize((32, 32)),\n",
    "                                              transforms.ToTensor()])\n",
    "\n",
    "orig_sign = torch.clamp(transform_to_tensor(Image.open(params['PATH_SIGN'])), 0, 1)\n",
    "\n",
    "model = load_model(params)\n",
    "\n",
    "print(\"Prediction on the original sign:\")\n",
    "test_on_tensor(model=model, tensor=orig_sign, params=params, labels=params['LABELS'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "executionInfo": {
     "elapsed": 3946537,
     "status": "ok",
     "timestamp": 1689196348531,
     "user": {
      "displayName": "Leopold Müller",
      "userId": "15867647540029964791"
     },
     "user_tz": -120
    },
    "id": "qLb1y5mfPhKO",
    "outputId": "084009a4-0c6c-4257-92d5-784164c41866"
   },
   "outputs": [],
   "source": [
    "# Create output directory\n",
    "if not os.path.exists(params['OUTPUT_DIR']):\n",
    "  os.makedirs(params['OUTPUT_DIR'])\n",
    "\n",
    "print(f\"-----------------{model_type}----------------------\")\n",
    "\n",
    "# Train attack\n",
    "train_attack(params)\n",
    "# Test attack\n",
    "test_attack(params)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AZmrtin0PhKP"
   },
   "source": [
    "## **Visible Attack**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000,
     "output_embedded_package_id": "1XhUEn0OvaR4shxC7SPPN41qpMkQ-0J17"
    },
    "executionInfo": {
     "elapsed": 4346224,
     "status": "ok",
     "timestamp": 1689167895347,
     "user": {
      "displayName": "Leopold Müller",
      "userId": "15867647540029964791"
     },
     "user_tz": -120
    },
    "id": "IuJg7Qf0PhKP",
    "outputId": "2a85a714-aa80-43b3-fdb7-542ca045affc"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "# Parameters of LISA dataset (you can leave this as it is)\n",
    "with open('classes.json', 'r') as config:\n",
    "    params = json.load(config)\n",
    "    params['CLASS_N'] = params['LISA']['class_n']\n",
    "    params['DEVICE'] = params['device']\n",
    "    params['LABELS'] = params['LISA']['labels']\n",
    "\n",
    "model_types = ['CNNsmall', 'CNNlarge', 'Transformer', 'ResNet18', 'MobileNetv2', 'DenseNet121', 'ShuffleNetv2x1', \"EfficientNet\"]\n",
    "\n",
    "\n",
    "model_type = \"EfficientNet\"\n",
    "print(model_type)\n",
    "\n",
    "\n",
    "# Set your hyperparameters (we add them to the params dictionary)\n",
    "\n",
    "# --- GENERAL PARAMETERS ---\n",
    "# Target class that should be recognized instead of orginal class of sign\n",
    "params['TARGET_CLASS'] = 12\n",
    "params['MODEL_TYPE'] = model_type\n",
    "# Directory where output files will be stored\n",
    "params['OUTPUT_DIR'] = f\"attacks/visible{params['MODEL_TYPE']}LISA_ATTACK\"\n",
    "\n",
    "# --- PATHS INPUT---\n",
    "# Path to weights of baseline model to attack\n",
    "params['PATH_MODEL'] = f\"models/{model_type}LISA.pth\"\n",
    "# Path of original sign to pertubate (should be in original printable size)\n",
    "params['PATH_SIGN'] = 'attack_utils/lisa_30.jpg'\n",
    "# Path to the sign mask (defines area of sign)\n",
    "params['PATH_SIGN_MASK'] = 'attack_utils/lisa_30_mask.jpg'\n",
    "\n",
    "\n",
    "# --- PATHS OUTPUT --- (ONLY CHANGE FILENAMES)\n",
    "# Path where to store the pertubated sign (printing size)\n",
    "params['PATH_PERT_SIGN_SMALL'] = os.path.join(params['OUTPUT_DIR'], 'pert_sign_small.png')\n",
    "# Path where to store the pertubated sign (printing size)\n",
    "params['PATH_PERT_SIGN_LARGE'] = os.path.join(params['OUTPUT_DIR'], 'pert_sign_large.png')\n",
    "# Path where to store the sign in the size of the model input\n",
    "params['PATH_ORIG_SIGN_SMALL'] = os.path.join(params['OUTPUT_DIR'], 'orig_sign_small_.png')\n",
    "# Path where to store the last augmented sign\n",
    "params['PATH_AUGMENTED_SIGN_SMALL'] = os.path.join(params['OUTPUT_DIR'], 'augmented_sign_small.png')\n",
    "# Path where to store the noise in small size\n",
    "params['PATH_NOISE_SMALL'] = os.path.join(params['OUTPUT_DIR'], 'noise_small.png')\n",
    "# Path where to store the noise in printable size\n",
    "params['PATH_NOISE_LARGE'] = os.path.join(params['OUTPUT_DIR'], 'noise_large.png')\n",
    "# Path where to store the noise mask\n",
    "params['PATH_NOISE_MASK_SMALL'] = os.path.join(params['OUTPUT_DIR'], 'noise_mask_small.png')\n",
    "# Path where to store the noise mask\n",
    "params['PATH_NOISE_MASK_LARGE'] = os.path.join(params['OUTPUT_DIR'], 'noise_mask_large.png')\n",
    "# Paht where to store noise tensor\n",
    "params['PATH_NOISE_TENSOR'] = os.path.join(params['OUTPUT_DIR'], 'noise_tensor.pt')\n",
    "\n",
    "# --- PARAMETERS FOR INITIALIZATION ---\n",
    "params['INIT_MASK_THRESHOLD'] = 0.01\n",
    "# How much you want to penalty using colors\n",
    "params['INIT_MASK_BETA_1'] = 0\n",
    "# How much you want to regularize the ammount of noise\n",
    "params['INIT_MASK_BETA_2'] = 0\n",
    "# How much you want to penalty the deviation from to target class\n",
    "params['INIT_MASK_BETA_3'] = 1\n",
    "# Learning rate of adam for the mask initialization\n",
    "params['INIT_MASK_LEARNING_RATE'] = 0.01\n",
    "# How many epochs you want to train the initialization of the mask\n",
    "params['INIT_MASK_EPOCHS'] = 1000\n",
    "# Turn augmentation for initialization of mask on/off\n",
    "params['INIT_MASK_AUGMENTATION'] = False\n",
    "\n",
    "\n",
    "# PARAMETERS FOR ATTACK\n",
    "# Learning rate of adam for the attack\n",
    "params['ATTACK_LEARNING_RATE'] = 0.0001\n",
    "# Batch size for one optimization step\n",
    "params['ATTACK_BATCH_SIZE'] = 8\n",
    "# How much you want to penalty using colors in the attack\n",
    "params['ATTACK_BETA_1'] = 0.0\n",
    "# How much you want to regularize the ammount of noise in the attack\n",
    "params['ATTACK_BETA_2'] = 0.01\n",
    "# How much you want to penalty the deviation from to target class\n",
    "params['ATTACK_BETA_3'] = 1.0\n",
    "# How many epochs you want to train the attack\n",
    "params['ATTACK_EPOCHS'] = 20000\n",
    "# Turn augmentation on/off for attack\n",
    "params['ATTACK_AUGMENTATION'] = True\n",
    "\n",
    "# Create output directory\n",
    "if not os.path.exists(params['OUTPUT_DIR']):\n",
    "  os.makedirs(params['OUTPUT_DIR'])\n",
    "\n",
    "print(f\"-----------------{model_type}----------------------\")\n",
    "\n",
    "# Train attack\n",
    "train_attack(params)\n",
    "# Test attack\n",
    "test_attack(params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [
    "d8d_BzabIAXt",
    "q2khGChjH6dW",
    "QgiIk-8LnueC"
   ],
   "gpuType": "T4",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}