build_and_run_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-14 17:53:18.199489: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-05-14 17:53:26,460 ; DEBUG ; Creating converter from 7 to 5\n",
      "2023-05-14 17:53:26,461 ; DEBUG ; Creating converter from 5 to 7\n",
      "2023-05-14 17:53:26,462 ; DEBUG ; Creating converter from 7 to 5\n",
      "2023-05-14 17:53:26,463 ; DEBUG ; Creating converter from 5 to 7\n"
     ]
    }
   ],
   "source": [
    "import logging\n",
    "\n",
    "logging.basicConfig(format=\"%(asctime)s ; %(levelname)s ; %(message)s\", level=logging.DEBUG)\n",
    "logging.getLogger(\"scapy\").setLevel(logging.CRITICAL)\n",
    "logger = logging.getLogger(\"adAPT\")\n",
    "\n",
    "from typing import Tuple\n",
    "from pathlib import Path\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras.regularizers import L2\n",
    "from tensorflow.keras import losses\n",
    "from tensorflow.keras import metrics\n",
    "from tensorflow.keras import optimizers\n",
    "from tensorflow import keras\n",
    "from sklearn.model_selection import train_test_split\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "b_pkl = \"./data/benign_features.pkl\"\n",
    "m_pkl = \"./data/malicious_features.pkl\"\n",
    "\n",
    "b_df = pd.read_pickle(b_pkl)\n",
    "m_df = pd.read_pickle(m_pkl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_df = pd.concat([b_df, m_df])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>protocol</th>\n",
       "      <th>app_layer</th>\n",
       "      <th>source_addr</th>\n",
       "      <th>dest_addr</th>\n",
       "      <th>source_port</th>\n",
       "      <th>dest_port</th>\n",
       "      <th>proto_packet_length</th>\n",
       "      <th>proto_packet_cache</th>\n",
       "      <th>ip_packet_length</th>\n",
       "      <th>ip_packet_cache</th>\n",
       "      <th>...</th>\n",
       "      <th>base_domain_entropy</th>\n",
       "      <th>host_length</th>\n",
       "      <th>proto_packet_entropy</th>\n",
       "      <th>source_ip_class_a</th>\n",
       "      <th>source_ip_class_b</th>\n",
       "      <th>source_ip_class_c</th>\n",
       "      <th>dest_ip_class_a</th>\n",
       "      <th>dest_ip_class_b</th>\n",
       "      <th>dest_ip_class_c</th>\n",
       "      <th>malware</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>14395</th>\n",
       "      <td>IP</td>\n",
       "      <td>HTTPResponse</td>\n",
       "      <td>10.1.11.101</td>\n",
       "      <td>51.195.169.87</td>\n",
       "      <td>64823</td>\n",
       "      <td>8080</td>\n",
       "      <td>120</td>\n",
       "      <td>b'E\\x00\\x00x\\xc81@\\x00\\x80\\x06?\\xce\\n\\x01\\x0be...</td>\n",
       "      <td>120</td>\n",
       "      <td>b'E\\x00\\x00x\\xc81@\\x00\\x80\\x06?\\xce\\n\\x01\\x0be...</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4.084184</td>\n",
       "      <td>10</td>\n",
       "      <td>10.1</td>\n",
       "      <td>10.1.11</td>\n",
       "      <td>51</td>\n",
       "      <td>51.195</td>\n",
       "      <td>51.195.169</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59183</th>\n",
       "      <td>IP</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>69.49.241.55</td>\n",
       "      <td>172.16.1.137</td>\n",
       "      <td>587</td>\n",
       "      <td>64195</td>\n",
       "      <td>40</td>\n",
       "      <td>b'E\\x00\\x00(*\\xaa\\x00\\x00\\x80\\x06,$E1\\xf17\\xac...</td>\n",
       "      <td>40</td>\n",
       "      <td>b'E\\x00\\x00(*\\xaa\\x00\\x00\\x80\\x06,$E1\\xf17\\xac...</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3.821928</td>\n",
       "      <td>69</td>\n",
       "      <td>69.49</td>\n",
       "      <td>69.49.241</td>\n",
       "      <td>172</td>\n",
       "      <td>172.16</td>\n",
       "      <td>172.16.1</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21426</th>\n",
       "      <td>IP</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>162.246.19.18</td>\n",
       "      <td>172.16.1.137</td>\n",
       "      <td>465</td>\n",
       "      <td>64088</td>\n",
       "      <td>40</td>\n",
       "      <td>b'E\\x00\\x00(\\xcf\\xe2\\x00\\x00\\x80\\x06\\x07L\\xa2\\...</td>\n",
       "      <td>40</td>\n",
       "      <td>b'E\\x00\\x00(\\xcf\\xe2\\x00\\x00\\x80\\x06\\x07L\\xa2\\...</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3.921928</td>\n",
       "      <td>162</td>\n",
       "      <td>162.246</td>\n",
       "      <td>162.246.19</td>\n",
       "      <td>172</td>\n",
       "      <td>172.16</td>\n",
       "      <td>172.16.1</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4368</th>\n",
       "      <td>IP</td>\n",
       "      <td>HTTPResponse</td>\n",
       "      <td>104.21.50.34</td>\n",
       "      <td>10.9.20.144</td>\n",
       "      <td>80</td>\n",
       "      <td>49790</td>\n",
       "      <td>1280</td>\n",
       "      <td>b'E\\x00\\x05\\x00\\xdd\\r\\x00\\x00\\x80\\x06\\xa0\\x1ah...</td>\n",
       "      <td>1280</td>\n",
       "      <td>b'E\\x00\\x05\\x00\\xdd\\r\\x00\\x00\\x80\\x06\\xa0\\x1ah...</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3.921928</td>\n",
       "      <td>104</td>\n",
       "      <td>104.21</td>\n",
       "      <td>104.21.50</td>\n",
       "      <td>10</td>\n",
       "      <td>10.9</td>\n",
       "      <td>10.9.20</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5375</th>\n",
       "      <td>IP</td>\n",
       "      <td>HTTPResponse</td>\n",
       "      <td>10.1.11.101</td>\n",
       "      <td>51.195.169.87</td>\n",
       "      <td>64823</td>\n",
       "      <td>8080</td>\n",
       "      <td>1070</td>\n",
       "      <td>b'E\\x00\\x04.\\xb73@\\x00\\x80\\x06M\\x16\\n\\x01\\x0be...</td>\n",
       "      <td>1070</td>\n",
       "      <td>b'E\\x00\\x04.\\xb73@\\x00\\x80\\x06M\\x16\\n\\x01\\x0be...</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4.121928</td>\n",
       "      <td>10</td>\n",
       "      <td>10.1</td>\n",
       "      <td>10.1.11</td>\n",
       "      <td>51</td>\n",
       "      <td>51.195</td>\n",
       "      <td>51.195.169</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      protocol     app_layer    source_addr      dest_addr  source_port   \n",
       "14395       IP  HTTPResponse    10.1.11.101  51.195.169.87        64823  \\\n",
       "59183       IP       Unknown   69.49.241.55   172.16.1.137          587   \n",
       "21426       IP       Unknown  162.246.19.18   172.16.1.137          465   \n",
       "4368        IP  HTTPResponse   104.21.50.34    10.9.20.144           80   \n",
       "5375        IP  HTTPResponse    10.1.11.101  51.195.169.87        64823   \n",
       "\n",
       "       dest_port  proto_packet_length   \n",
       "14395       8080                  120  \\\n",
       "59183      64195                   40   \n",
       "21426      64088                   40   \n",
       "4368       49790                 1280   \n",
       "5375        8080                 1070   \n",
       "\n",
       "                                      proto_packet_cache  ip_packet_length   \n",
       "14395  b'E\\x00\\x00x\\xc81@\\x00\\x80\\x06?\\xce\\n\\x01\\x0be...               120  \\\n",
       "59183  b'E\\x00\\x00(*\\xaa\\x00\\x00\\x80\\x06,$E1\\xf17\\xac...                40   \n",
       "21426  b'E\\x00\\x00(\\xcf\\xe2\\x00\\x00\\x80\\x06\\x07L\\xa2\\...                40   \n",
       "4368   b'E\\x00\\x05\\x00\\xdd\\r\\x00\\x00\\x80\\x06\\xa0\\x1ah...              1280   \n",
       "5375   b'E\\x00\\x04.\\xb73@\\x00\\x80\\x06M\\x16\\n\\x01\\x0be...              1070   \n",
       "\n",
       "                                         ip_packet_cache  ...   \n",
       "14395  b'E\\x00\\x00x\\xc81@\\x00\\x80\\x06?\\xce\\n\\x01\\x0be...  ...  \\\n",
       "59183  b'E\\x00\\x00(*\\xaa\\x00\\x00\\x80\\x06,$E1\\xf17\\xac...  ...   \n",
       "21426  b'E\\x00\\x00(\\xcf\\xe2\\x00\\x00\\x80\\x06\\x07L\\xa2\\...  ...   \n",
       "4368   b'E\\x00\\x05\\x00\\xdd\\r\\x00\\x00\\x80\\x06\\xa0\\x1ah...  ...   \n",
       "5375   b'E\\x00\\x04.\\xb73@\\x00\\x80\\x06M\\x16\\n\\x01\\x0be...  ...   \n",
       "\n",
       "      base_domain_entropy host_length proto_packet_entropy source_ip_class_a   \n",
       "14395                 0.0           0             4.084184                10  \\\n",
       "59183                 0.0           0             3.821928                69   \n",
       "21426                 0.0           0             3.921928               162   \n",
       "4368                  0.0           0             3.921928               104   \n",
       "5375                  0.0           0             4.121928                10   \n",
       "\n",
       "      source_ip_class_b  source_ip_class_c  dest_ip_class_a  dest_ip_class_b   \n",
       "14395              10.1            10.1.11               51           51.195  \\\n",
       "59183             69.49          69.49.241              172           172.16   \n",
       "21426           162.246         162.246.19              172           172.16   \n",
       "4368             104.21          104.21.50               10             10.9   \n",
       "5375               10.1            10.1.11               51           51.195   \n",
       "\n",
       "       dest_ip_class_c  malware  \n",
       "14395       51.195.169      1.0  \n",
       "59183         172.16.1      1.0  \n",
       "21426         172.16.1      1.0  \n",
       "4368           10.9.20      1.0  \n",
       "5375        51.195.169      1.0  \n",
       "\n",
       "[5 rows x 27 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_df_for_ml(df: pd.DataFrame) -> pd.DataFrame:\n",
    "    df = df.copy()\n",
    "    fields = [\n",
    "        \"protocol\",\n",
    "        \"app_layer\",\n",
    "        \"source_port\",\n",
    "        \"dest_port\",\n",
    "        \"proto_packet_length\",\n",
    "        \"ip_packet_length\",\n",
    "        # \"url\",\n",
    "        \"base_domain\",\n",
    "        \"tld\",\n",
    "        \"url_entropy\",\n",
    "        \"host_entropy\",\n",
    "        \"base_domain_entropy\",\n",
    "        \"host_length\",\n",
    "        \"proto_packet_entropy\",\n",
    "        \"source_ip_class_a\",\n",
    "        \"source_ip_class_b\",\n",
    "        \"source_ip_class_c\",\n",
    "        \"dest_ip_class_a\",\n",
    "        \"dest_ip_class_b\",\n",
    "        \"dest_ip_class_c\",\n",
    "    ]\n",
    "    new_df = df[fields]\n",
    "    new_df.loc[:, [\"source_port\", \"dest_port\"]] = new_df[\n",
    "        [\n",
    "            \"source_port\",\n",
    "            \"dest_port\",\n",
    "        ]\n",
    "    ].astype(str)\n",
    "    new_df.loc[:, [\"ip_packet_length\", \"source_port\"]] = new_df[\n",
    "        [\n",
    "            \"ip_packet_length\",\n",
    "            \"source_port\",\n",
    "        ]\n",
    "    ].astype(float)\n",
    "\n",
    "    return pd.get_dummies(new_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = all_df[\"malware\"].values\n",
    "input_values = all_df.drop(\"malware\", axis=1)\n",
    "prepped = prepare_df_for_ml(input_values)\n",
    "X = np.asarray(prepped.values).astype(\"float64\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(196981,)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(196981, 6867)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, Y_train, Y_test = train_test_split(X, y, test_size=0.40, random_state=234)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test, X_cv, Y_test, Y_cv = train_test_split(X_test, Y_test, test_size=0.50, random_state=33)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train.shape: (118188, 6867)\n",
      "X_test.shape: (39396, 6867)\n",
      "X_cv.shape: (39397, 6867)\n",
      "Y_train.shape: (118188,)\n",
      "Y_test.shape: (39396,)\n",
      "Y_cv.shape: (39397,)\n"
     ]
    }
   ],
   "source": [
    "print(f\"X_train.shape: {X_train.shape}\")\n",
    "print(f\"X_test.shape: {X_test.shape}\")\n",
    "print(f\"X_cv.shape: {X_cv.shape}\")\n",
    "print(f\"Y_train.shape: {Y_train.shape}\")\n",
    "print(f\"Y_test.shape: {Y_test.shape}\")\n",
    "print(f\"Y_cv.shape: {Y_cv.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "normalizer = tf.keras.layers.Normalization(axis=-1)\n",
    "\n",
    "layer_1 = layers.Dense(units=9, input_shape=(X_train.shape[-1],), activation=\"relu\", kernel_regularizer=L2(0.01))\n",
    "layer_2 = layers.Dense(units=15, activation=\"relu\", kernel_regularizer=L2(0.01))\n",
    "layer_3 = layers.Dense(units=1, activation=\"sigmoid\", kernel_regularizer=L2(0.01))\n",
    "\n",
    "model = keras.Sequential([normalizer, layer_1, layer_2, layer_3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(\n",
    "    optimizer=\"adam\",  # Optimizer\n",
    "    # Loss function to minimize\n",
    "    loss=losses.BinaryCrossentropy(),\n",
    "    metrics=[tf.keras.metrics.BinaryAccuracy(), tf.keras.metrics.FalseNegatives()],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "3694/3694 [==============================] - 11s 3ms/step - loss: 0.1896 - binary_accuracy: 0.9766 - false_negatives: 190.0000\n",
      "Epoch 2/10\n",
      "3694/3694 [==============================] - 9s 3ms/step - loss: 0.1501 - binary_accuracy: 0.9880 - false_negatives: 239.0000\n",
      "Epoch 3/10\n",
      "3694/3694 [==============================] - 11s 3ms/step - loss: 0.1455 - binary_accuracy: 0.9888 - false_negatives: 201.0000\n",
      "Epoch 4/10\n",
      "3694/3694 [==============================] - 11s 3ms/step - loss: 0.1372 - binary_accuracy: 0.9895 - false_negatives: 158.0000\n",
      "Epoch 5/10\n",
      "3694/3694 [==============================] - 10s 3ms/step - loss: 0.1296 - binary_accuracy: 0.9904 - false_negatives: 147.0000\n",
      "Epoch 6/10\n",
      "3694/3694 [==============================] - 10s 3ms/step - loss: 0.1279 - binary_accuracy: 0.9905 - false_negatives: 137.0000\n",
      "Epoch 7/10\n",
      "3694/3694 [==============================] - 10s 3ms/step - loss: 0.1279 - binary_accuracy: 0.9906 - false_negatives: 135.0000\n",
      "Epoch 8/10\n",
      "3694/3694 [==============================] - 10s 3ms/step - loss: 0.1258 - binary_accuracy: 0.9909 - false_negatives: 107.0000\n",
      "Epoch 9/10\n",
      "3694/3694 [==============================] - 10s 3ms/step - loss: 0.1256 - binary_accuracy: 0.9905 - false_negatives: 137.0000\n",
      "Epoch 10/10\n",
      "3694/3694 [==============================] - 10s 3ms/step - loss: 0.1250 - binary_accuracy: 0.9905 - false_negatives: 140.0000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x16efd41f0>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, Y_train, epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate on test data\n",
      "1232/1232 [==============================] - 2s 2ms/step - loss: 0.1152 - binary_accuracy: 0.9937 - false_negatives: 25.0000\n",
      "test loss, test acc: [0.11515635251998901, 0.9936795830726624, 25.0]\n"
     ]
    }
   ],
   "source": [
    "print(\"Evaluate on test data\")\n",
    "results = model.evaluate(X_test, Y_test)\n",
    "print(\"test loss, test acc:\", results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate on Cross Validation data\n",
      "1232/1232 [==============================] - 2s 2ms/step - loss: 0.1146 - binary_accuracy: 0.9939 - false_negatives: 22.0000\n",
      "cv loss and acc: [0.11457856744527817, 0.9938573837280273, 22.0]\n"
     ]
    }
   ],
   "source": [
    "print(\"Evaluate on Cross Validation data\")\n",
    "results_cv = model.evaluate(X_cv, Y_cv)\n",
    "print(f\"cv loss and acc: {results_cv}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate on full data set\n",
      "6156/6156 [==============================] - 12s 2ms/step - loss: 0.1152 - binary_accuracy: 0.9935 - false_negatives: 137.0000\n",
      "cv loss and acc: [0.11520779132843018, 0.993496835231781, 137.0]\n"
     ]
    }
   ],
   "source": [
    "print(\"Evaluate on full data set\")\n",
    "results_full = model.evaluate(X, y)\n",
    "print(f\"cv loss and acc: {results_full}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}